From 37fe3c148f6edcb7ad8567ed2490f8c67208c50f Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Mon, 7 Oct 2019 19:13:06 +0300 Subject: [PATCH] * whois: use whois.arin.net + robust redirect mechanism * decrease timeout 30sec -> 5sec * faster response parsing * don't use likexian/whois-go package --- go.mod | 2 +- go.sum | 27 ++++-------- home/dns.go | 3 +- home/helpers.go | 15 +++++++ home/helpers_test.go | 8 ++++ home/whois.go | 99 +++++++++++++++++++++++++++++++++++++------- home/whois_test.go | 8 +--- 7 files changed, 122 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index 69072309..9e61efc6 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/joomcode/errorx v1.0.0 github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b + github.com/kr/pretty v0.1.0 // indirect github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414 - github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49 github.com/miekg/dns v1.1.19 github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 github.com/stretchr/testify v1.4.0 diff --git a/go.sum b/go.sum index 875e3dd3..42415841 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,16 @@ -github.com/AdguardTeam/dnsproxy v0.19.4 h1:rZb40VUr/yN8RG4j3+NuGqODmPvte7acPfSDl0j2wiU= -github.com/AdguardTeam/dnsproxy v0.19.4/go.mod h1:NaulY9i279jZwN8QBbvbZnn5HkrjBgJi4hbFY5nW+Kc= github.com/AdguardTeam/dnsproxy v0.19.5 h1:QAKWa2+rTp7GAeOFLMPqIYPS7eglLVEkVLH4kHRbnCQ= github.com/AdguardTeam/dnsproxy v0.19.5/go.mod h1:qEiDndktnVJYYzHiQGKUl8Zm0b7HGpPmYWShAxmqjtw= github.com/AdguardTeam/golibs v0.1.3 h1:hmapdTtMtIk3T8eQDwTOLdqZLGDKNKk9325uC8z12xg= github.com/AdguardTeam/golibs v0.1.3/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko= github.com/AdguardTeam/golibs v0.2.1 h1:jGCnbM5UOUq/GrG+8eLN7Y+OTfEo5F/8L0wq3ur2h4E= github.com/AdguardTeam/golibs v0.2.1/go.mod h1:caAJ5knSHbR6vV6qfRDgAfXVia4hHgLqeztAY4UX0fw= -github.com/AdguardTeam/urlfilter v0.5.0 h1:ATzs2Er0BMt7NbZnFJ4UEzt3uIV+rydbQCYqBXNRbJc= -github.com/AdguardTeam/urlfilter v0.5.0/go.mod h1:6YehXZ8e0Hx2MvqeQWLFom6IkPinm04tNhO1CkwAxmg= github.com/AdguardTeam/urlfilter v0.6.0 h1:HVPfAsGcHW47HasmqcLNA/VJ41GaR/SzUufuIj70ouA= github.com/AdguardTeam/urlfilter v0.6.0/go.mod h1:y+XdxBdbRG9v7pfjznlvv4Ufi2HTG8D0YMqR22OVy0Y= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= @@ -67,18 +64,13 @@ github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/ github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b h1:vfiqKno48aUndBMjTeWFpCExNnTf2Xnd6d228L4EfTQ= github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b/go.mod h1:10UU/bEkzh2iEN6aYzbevY7J6p03KO5siTxQWXMEerg= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414 h1:6wnYc2S/lVM7BvR32BM74ph7bPgqMztWopMYKgVyEho= github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414/go.mod h1:0AqAH3ZogsCrvrtUpvc6EtVKbc3w6xwZhkvGLuqyi3o= -github.com/likexian/gokit v0.0.0-20190309162924-0a377eecf7aa/go.mod h1:QdfYv6y6qPA9pbBA2qXtoT8BMKha6UyNbxWGWl/9Jfk= -github.com/likexian/gokit v0.0.0-20190418170008-ace88ad0983b/go.mod h1:KKqSnk/VVSW8kEyO2vVCXoanzEutKdlBAPohmGXkxCk= -github.com/likexian/gokit v0.0.0-20190501133040-e77ea8b19cdc/go.mod h1:3kvONayqCaj+UgrRZGpgfXzHdMYCAO0KAt4/8n0L57Y= -github.com/likexian/gokit v0.0.0-20190604165112-68b8a4ba758c h1:KByA4IxKqqYwpqzk/P+w1DBpkPbvy3DArTP/U3LSxTQ= -github.com/likexian/gokit v0.0.0-20190604165112-68b8a4ba758c/go.mod h1:kn+nTv3tqh6yhor9BC4Lfiu58SmH8NmQ2PmEl+uM6nU= -github.com/likexian/simplejson-go v0.0.0-20190409170913-40473a74d76d/go.mod h1:Typ1BfnATYtZ/+/shXfFYLrovhFyuKvzwrdOnIDHlmg= -github.com/likexian/simplejson-go v0.0.0-20190419151922-c1f9f0b4f084/go.mod h1:U4O1vIJvIKwbMZKUJ62lppfdvkCdVd2nfMimHK81eec= -github.com/likexian/simplejson-go v0.0.0-20190502021454-d8787b4bfa0b/go.mod h1:3BWwtmKP9cXWwYCr5bkoVDEfLywacOv0s06OBEDpyt8= -github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49 h1:xGa+flE6p2UnMgxIS8bm7Q9JSt47HRuYVtwneDVnfLk= -github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49/go.mod h1:oR3bJMzrOb55cqTAn14DEzYFLDpSPTXJ3ORe7go9Hc8= github.com/markbates/oncer v0.0.0-20181014194634-05fccaae8fc4 h1:Mlji5gkcpzkqTROyE4ZxZ8hN7osunMb2RuGVrbvMvCc= github.com/markbates/oncer v0.0.0-20181014194634-05fccaae8fc4/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI= @@ -95,6 +87,7 @@ github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYe github.com/rogpeppe/go-charset v0.0.0-20190617161244-0dc95cdf6f31/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v2.19.9+incompatible h1:IrPVlK4nfwW10DF7pW+7YJKws9NkgNzWozwwWv9FsgY= github.com/shirou/gopsutil v2.19.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= @@ -120,8 +113,6 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191001170739-f9e2070545dc h1:KyTYo8xkh/2WdbFLUyQwBS0Jfn3qfZ9QmuPbok2oENE= golang.org/x/crypto v0.0.0-20191001170739-f9e2070545dc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -142,14 +133,13 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTm golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190122071731-054c452bb702 h1:Lk4tbZFnlyPgV+sLgTw5yGfzrlOn9kx4vSombi2FFlY= golang.org/x/sys v0.0.0-20190122071731-054c452bb702/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002091554-b397fe3ad8ed h1:5TJcLJn2a55mJjzYk0yOoqN8X1OdvBDUnaZaKKyQtkY= @@ -164,6 +154,7 @@ golang.org/x/tools v0.0.0-20191001184121-329c8d646ebe/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI= diff --git a/home/dns.go b/home/dns.go index 3e58ffb1..3bf96185 100644 --- a/home/dns.go +++ b/home/dns.go @@ -61,7 +61,8 @@ func initDNSServer() { config.dnsctx.rdns = InitRDNS(&config.clients) config.dnsctx.whois = initWhois(&config.clients) - topClients := config.stats.GetTopData(30) + const topClientsNumber = 30 // the number of clients to get + topClients := config.stats.GetTopClientsIP(topClientsNumber) for _, ip := range topClients { ipAddr := net.ParseIP(ip) if !ipAddr.IsLoopback() { diff --git a/home/helpers.go b/home/helpers.go index c71aa5ff..756d6b97 100644 --- a/home/helpers.go +++ b/home/helpers.go @@ -377,3 +377,18 @@ func parseIPv4(s string) net.IP { return ip.To4() } + +// SplitNext - split string by a byte and return the first chunk +// Whitespace is trimmed +func SplitNext(str *string, splitBy byte) string { + i := strings.IndexByte(*str, splitBy) + s := "" + if i != -1 { + s = (*str)[0:i] + *str = (*str)[i+1:] + } else { + s = *str + *str = "" + } + return strings.TrimSpace(s) +} diff --git a/home/helpers_test.go b/home/helpers_test.go index 03243346..c2095966 100644 --- a/home/helpers_test.go +++ b/home/helpers_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/AdguardTeam/golibs/log" + "github.com/stretchr/testify/assert" ) func TestGetValidNetInterfacesForWeb(t *testing.T) { @@ -23,3 +24,10 @@ func TestGetValidNetInterfacesForWeb(t *testing.T) { log.Printf("%v", iface) } } + +func TestSplitNext(t *testing.T) { + s := " a,b , c " + assert.True(t, SplitNext(&s, ',') == "a") + assert.True(t, SplitNext(&s, ',') == "b") + assert.True(t, SplitNext(&s, ',') == "c" && len(s) == 0) +} diff --git a/home/whois.go b/home/whois.go index ccd5e51e..ced0b6e2 100644 --- a/home/whois.go +++ b/home/whois.go @@ -1,26 +1,35 @@ package home import ( + "fmt" + "io/ioutil" + "net" "strings" "sync" + "time" "github.com/AdguardTeam/golibs/log" - whois "github.com/likexian/whois-go" ) -const maxValueLength = 250 +const ( + defaultServer = "whois.arin.net" + defaultPort = "43" + maxValueLength = 250 +) // Whois - module context type Whois struct { - clients *clientsContainer - ips map[string]bool - lock sync.Mutex - ipChan chan string + clients *clientsContainer + ips map[string]bool + lock sync.Mutex + ipChan chan string + timeoutMsec uint } // Create module context func initWhois(clients *clientsContainer) *Whois { w := Whois{} + w.timeoutMsec = 5000 w.clients = clients w.ips = make(map[string]bool) w.ipChan = make(chan string, 255) @@ -41,11 +50,9 @@ func whoisParse(data string) map[string]string { m := map[string]string{} descr := "" netname := "" - lines := strings.Split(data, "\n") - for _, ln := range lines { - ln = strings.TrimSpace(ln) - - if len(ln) == 0 || ln[0] == '#' { + for len(data) != 0 { + ln := SplitNext(&data, '\n') + if len(ln) == 0 || ln[0] == '#' || ln[0] == '%' { continue } @@ -71,6 +78,14 @@ func whoisParse(data string) map[string]string { descr = v case "netname": netname = v + + case "whois": // "whois: whois.arin.net" + m["whois"] = v + + case "referralserver": // "ReferralServer: whois://whois.ripe.net" + if strings.HasPrefix(v, "whois://") { + m["whois"] = v[len("whois://"):] + } } } @@ -85,10 +100,66 @@ func whoisParse(data string) map[string]string { return m } +// Send request to a server and receive the response +func (w *Whois) query(target string, serverAddr string) (string, error) { + addr, _, _ := net.SplitHostPort(serverAddr) + if addr == "whois.arin.net" { + target = "n + " + target + } + conn, err := net.DialTimeout("tcp", serverAddr, time.Duration(w.timeoutMsec)*time.Millisecond) + if err != nil { + return "", err + } + defer conn.Close() + + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(w.timeoutMsec) * time.Millisecond)) + _, err = conn.Write([]byte(target + "\r\n")) + if err != nil { + return "", err + } + + data, err := ioutil.ReadAll(conn) + if err != nil { + return "", err + } + + return string(data), nil +} + +// Query WHOIS servers (handle redirects) +func (w *Whois) queryAll(target string) (string, error) { + server := net.JoinHostPort(defaultServer, defaultPort) + const maxRedirects = 5 + for i := 0; i != maxRedirects; i++ { + resp, err := w.query(target, server) + if err != nil { + return "", err + } + log.Debug("Whois: received response (%d bytes) from %s IP:%s", len(resp), server, target) + + m := whoisParse(resp) + redir, ok := m["whois"] + if !ok { + return resp, nil + } + redir = strings.ToLower(redir) + + _, _, err = net.SplitHostPort(redir) + if err != nil { + server = net.JoinHostPort(redir, defaultPort) + } else { + server = redir + } + + log.Debug("Whois: redirected to %s IP:%s", redir, target) + } + return "", fmt.Errorf("Whois: redirect loop") +} + // Request WHOIS information -func whoisProcess(ip string) [][]string { +func (w *Whois) process(ip string) [][]string { data := [][]string{} - resp, err := whois.Whois(ip) + resp, err := w.queryAll(ip) if err != nil { log.Debug("Whois: error: %s IP:%s", err, ip) return data @@ -137,7 +208,7 @@ func (w *Whois) workerLoop() { var ip string ip = <-w.ipChan - info := whoisProcess(ip) + info := w.process(ip) if len(info) == 0 { continue } diff --git a/home/whois_test.go b/home/whois_test.go index 7a841110..1c77b056 100644 --- a/home/whois_test.go +++ b/home/whois_test.go @@ -1,19 +1,15 @@ package home import ( - "strings" "testing" - whois "github.com/likexian/whois-go" "github.com/stretchr/testify/assert" ) func TestWhois(t *testing.T) { - resp, err := whois.Whois("8.8.8.8") + w := Whois{timeoutMsec: 5000} + resp, err := w.queryAll("8.8.8.8") assert.True(t, err == nil) - assert.True(t, strings.Index(resp, "OrgName: Google LLC") != -1) - assert.True(t, strings.Index(resp, "City: Mountain View") != -1) - assert.True(t, strings.Index(resp, "Country: US") != -1) m := whoisParse(resp) assert.True(t, m["orgname"] == "Google LLC") assert.True(t, m["country"] == "US")