From fd592b4dc89fb42cebca95cf645180edc225c287 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Wed, 18 Sep 2019 18:44:27 +0300 Subject: [PATCH] + whois: add WHOIS information for a client + return WHOIS info in GET /clients response --- AGHTechDoc.md | 10 ++++ go.mod | 1 + go.sum | 10 ++++ home/clients.go | 82 +++++++++++++++++++++++++----- home/clients_test.go | 30 ++++------- home/dns.go | 59 ++++++++++++++++++++++ home/dns_test.go | 2 +- home/rdns.go | 2 +- home/whois.go | 118 +++++++++++++++++++++++++++++++++++++++++++ home/whois_test.go | 21 ++++++++ 10 files changed, 300 insertions(+), 35 deletions(-) create mode 100644 home/whois.go create mode 100644 home/whois_test.go diff --git a/AGHTechDoc.md b/AGHTechDoc.md index 0522439e..60fc986f 100644 --- a/AGHTechDoc.md +++ b/AGHTechDoc.md @@ -650,6 +650,10 @@ Response: safesearch_enabled: false use_global_blocked_services: true blocked_services: [ "name1", ... ] + whois_info: { + key: "value" + ... + } } ] auto_clients: [ @@ -657,10 +661,16 @@ Response: name: "host" ip: "..." source: "etc/hosts" || "rDNS" + whois_info: { + key: "value" + ... + } } ] } +Supported keys for `whois_info`: orgname, country, city. + ### Add client diff --git a/go.mod b/go.mod index f15b6e19..49f0f47f 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414 + github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49 github.com/miekg/dns v1.1.8 github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 github.com/stretchr/testify v1.4.0 diff --git a/go.sum b/go.sum index c26688a1..bea03755 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,16 @@ github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b h1:vfiqKno48aUnd github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b/go.mod h1:10UU/bEkzh2iEN6aYzbevY7J6p03KO5siTxQWXMEerg= github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414 h1:6wnYc2S/lVM7BvR32BM74ph7bPgqMztWopMYKgVyEho= github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414/go.mod h1:0AqAH3ZogsCrvrtUpvc6EtVKbc3w6xwZhkvGLuqyi3o= +github.com/likexian/gokit v0.0.0-20190309162924-0a377eecf7aa/go.mod h1:QdfYv6y6qPA9pbBA2qXtoT8BMKha6UyNbxWGWl/9Jfk= +github.com/likexian/gokit v0.0.0-20190418170008-ace88ad0983b/go.mod h1:KKqSnk/VVSW8kEyO2vVCXoanzEutKdlBAPohmGXkxCk= +github.com/likexian/gokit v0.0.0-20190501133040-e77ea8b19cdc/go.mod h1:3kvONayqCaj+UgrRZGpgfXzHdMYCAO0KAt4/8n0L57Y= +github.com/likexian/gokit v0.0.0-20190604165112-68b8a4ba758c h1:KByA4IxKqqYwpqzk/P+w1DBpkPbvy3DArTP/U3LSxTQ= +github.com/likexian/gokit v0.0.0-20190604165112-68b8a4ba758c/go.mod h1:kn+nTv3tqh6yhor9BC4Lfiu58SmH8NmQ2PmEl+uM6nU= +github.com/likexian/simplejson-go v0.0.0-20190409170913-40473a74d76d/go.mod h1:Typ1BfnATYtZ/+/shXfFYLrovhFyuKvzwrdOnIDHlmg= +github.com/likexian/simplejson-go v0.0.0-20190419151922-c1f9f0b4f084/go.mod h1:U4O1vIJvIKwbMZKUJ62lppfdvkCdVd2nfMimHK81eec= +github.com/likexian/simplejson-go v0.0.0-20190502021454-d8787b4bfa0b/go.mod h1:3BWwtmKP9cXWwYCr5bkoVDEfLywacOv0s06OBEDpyt8= +github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49 h1:xGa+flE6p2UnMgxIS8bm7Q9JSt47HRuYVtwneDVnfLk= +github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49/go.mod h1:oR3bJMzrOb55cqTAn14DEzYFLDpSPTXJ3ORe7go9Hc8= github.com/markbates/oncer v0.0.0-20181014194634-05fccaae8fc4 h1:Mlji5gkcpzkqTROyE4ZxZ8hN7osunMb2RuGVrbvMvCc= github.com/markbates/oncer v0.0.0-20181014194634-05fccaae8fc4/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI= diff --git a/home/clients.go b/home/clients.go index e80a7a00..7fcfeed3 100644 --- a/home/clients.go +++ b/home/clients.go @@ -31,6 +31,7 @@ type Client struct { SafeSearchEnabled bool SafeBrowsingEnabled bool ParentalEnabled bool + WhoisInfo [][]string // [[key,value], ...] UseOwnBlockedServices bool // false: use global settings BlockedServices []string @@ -46,29 +47,34 @@ type clientJSON struct { SafeSearchEnabled bool `json:"safebrowsing_enabled"` SafeBrowsingEnabled bool `json:"safesearch_enabled"` + WhoisInfo map[string]interface{} `json:"whois_info"` + UseGlobalBlockedServices bool `json:"use_global_blocked_services"` BlockedServices []string `json:"blocked_services"` } type clientSource uint +// Client sources const ( - // Priority: etc/hosts > DHCP > ARP > rDNS - ClientSourceRDNS clientSource = 0 // from rDNS - ClientSourceDHCP clientSource = 1 // from DHCP - ClientSourceARP clientSource = 2 // from 'arp -a' - ClientSourceHostsFile clientSource = 3 // from /etc/hosts + // Priority: etc/hosts > DHCP > ARP > rDNS > WHOIS + ClientSourceWHOIS clientSource = iota // from WHOIS + ClientSourceRDNS // from rDNS + ClientSourceDHCP // from DHCP + ClientSourceARP // from 'arp -a' + ClientSourceHostsFile // from /etc/hosts ) // ClientHost information type ClientHost struct { - Host string - Source clientSource + Host string + Source clientSource + WhoisInfo [][]string // [[key,value], ...] } type clientsContainer struct { - list map[string]*Client - ipIndex map[string]*Client + list map[string]*Client // name -> client + ipIndex map[string]*Client // IP -> client ipHost map[string]ClientHost // IP -> Hostname lock sync.Mutex } @@ -101,7 +107,7 @@ func (clients *clientsContainer) GetList() map[string]*Client { } // Exists checks if client with this IP already exists -func (clients *clientsContainer) Exists(ip string) bool { +func (clients *clientsContainer) Exists(ip string, source clientSource) bool { clients.lock.Lock() defer clients.lock.Unlock() @@ -110,8 +116,14 @@ func (clients *clientsContainer) Exists(ip string) bool { return true } - _, ok = clients.ipHost[ip] - return ok + ch, ok := clients.ipHost[ip] + if !ok { + return false + } + if source > ch.Source { + return false // we're going to overwrite this client's info with a stronger source + } + return true } // Find searches for a client by IP @@ -266,6 +278,31 @@ func (clients *clientsContainer) Update(name string, c Client) error { return nil } +// SetWhoisInfo - associate WHOIS information with a client +func (clients *clientsContainer) SetWhoisInfo(ip string, info [][]string) { + clients.lock.Lock() + defer clients.lock.Unlock() + + c, ok := clients.ipIndex[ip] + if ok { + c.WhoisInfo = info + log.Debug("Clients: set WHOIS info for client %s: %v", c.Name, c.WhoisInfo) + } + + ch, ok := clients.ipHost[ip] + if ok { + ch.WhoisInfo = info + log.Debug("Clients: set WHOIS info for auto-client %s: %v", ch.Host, ch.WhoisInfo) + } + + ch = ClientHost{ + Source: ClientSourceWHOIS, + } + ch.WhoisInfo = info + clients.ipHost[ip] = ch + log.Debug("Clients: set WHOIS info for auto-client with IP %s: %v", ip, ch.WhoisInfo) +} + // AddHost adds new IP -> Host pair // Use priority of the source (etc/hosts > ARP > rDNS) // so we overwrite existing entries with an equal or higher priority @@ -280,8 +317,9 @@ func (clients *clientsContainer) AddHost(ip, host string, source clientSource) ( } clients.ipHost[ip] = ClientHost{ - Host: host, - Source: source, + Host: host, + Source: source, + WhoisInfo: c.WhoisInfo, } log.Tracef("'%s' -> '%s' [%d]", ip, host, len(clients.ipHost)) return true, nil @@ -386,6 +424,8 @@ type clientHostJSON struct { IP string `json:"ip"` Name string `json:"name"` Source string `json:"source"` + + WhoisInfo map[string]interface{} `json:"whois_info"` } type clientListJSON struct { @@ -421,6 +461,11 @@ func handleGetClients(w http.ResponseWriter, r *http.Request) { } } + cj.WhoisInfo = make(map[string]interface{}) + for _, wi := range c.WhoisInfo { + cj.WhoisInfo[wi[0]] = wi[1] + } + data.Clients = append(data.Clients, cj) } for ip, ch := range config.clients.ipHost { @@ -428,6 +473,7 @@ func handleGetClients(w http.ResponseWriter, r *http.Request) { IP: ip, Name: ch.Host, } + cj.Source = "etc/hosts" switch ch.Source { case ClientSourceDHCP: @@ -436,7 +482,15 @@ func handleGetClients(w http.ResponseWriter, r *http.Request) { cj.Source = "rDNS" case ClientSourceARP: cj.Source = "ARP" + case ClientSourceWHOIS: + cj.Source = "WHOIS" } + + cj.WhoisInfo = make(map[string]interface{}) + for _, wi := range ch.WhoisInfo { + cj.WhoisInfo[wi[0]] = wi[1] + } + data.AutoClients = append(data.AutoClients, cj) } config.clients.lock.Unlock() diff --git a/home/clients_test.go b/home/clients_test.go index d5dc5143..c4d1837e 100644 --- a/home/clients_test.go +++ b/home/clients_test.go @@ -1,6 +1,10 @@ package home -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestClients(t *testing.T) { var c Client @@ -61,15 +65,9 @@ func TestClients(t *testing.T) { } // get - if clients.Exists("1.2.3.4") { - t.Fatalf("Exists") - } - if !clients.Exists("1.1.1.1") { - t.Fatalf("Exists #1") - } - if !clients.Exists("2.2.2.2") { - t.Fatalf("Exists #2") - } + assert.True(t, !clients.Exists("1.2.3.4", ClientSourceHostsFile)) + assert.True(t, clients.Exists("1.1.1.1", ClientSourceHostsFile)) + assert.True(t, clients.Exists("2.2.2.2", ClientSourceHostsFile)) // failed update - no such name c.IP = "1.2.3.0" @@ -100,9 +98,7 @@ func TestClients(t *testing.T) { } // get after update - if clients.Exists("1.1.1.1") || !clients.Exists("1.1.1.2") { - t.Fatalf("Exists - get after update") - } + assert.True(t, !(clients.Exists("1.1.1.1", ClientSourceHostsFile) || !clients.Exists("1.1.1.2", ClientSourceHostsFile))) // failed remove - no such name if clients.Del("client3") { @@ -110,9 +106,7 @@ func TestClients(t *testing.T) { } // remove - if !clients.Del("client1") || clients.Exists("1.1.1.2") { - t.Fatalf("Del") - } + assert.True(t, !(!clients.Del("client1") || clients.Exists("1.1.1.2", ClientSourceHostsFile))) // add host client b, e = clients.AddHost("1.1.1.1", "host", ClientSourceARP) @@ -139,7 +133,5 @@ func TestClients(t *testing.T) { } // get - if !clients.Exists("1.1.1.1") { - t.Fatalf("clientAddHost") - } + assert.True(t, clients.Exists("1.1.1.1", ClientSourceHostsFile)) } diff --git a/home/dns.go b/home/dns.go index c9f199e1..3b90ddfa 100644 --- a/home/dns.go +++ b/home/dns.go @@ -25,6 +25,8 @@ type dnsContext struct { rdnsIP map[string]bool rdnsLock sync.Mutex // synchronize access to rdnsIP upstream upstream.Upstream // Upstream object for our own DNS server + + whois *Whois } // initDNSServer creates an instance of the dnsforward.Server @@ -56,6 +58,7 @@ func initDNSServer(baseDir string) { config.Users = nil initRDNS() + config.dnsctx.whois = initWhois(&config.clients) initFiltering() } @@ -63,6 +66,59 @@ func isRunning() bool { return config.dnsServer != nil && config.dnsServer.IsRunning() } +// Return TRUE if IP is within public Internet IP range +func isPublicIP(ip net.IP) bool { + ip4 := ip.To4() + if ip4 != nil { + switch ip4[0] { + case 0: + return false //software + case 10: + return false //private network + case 127: + return false //loopback + case 169: + if ip4[1] == 254 { + return false //link-local + } + case 172: + if ip4[1] >= 16 && ip4[1] <= 31 { + return false //private network + } + case 192: + if (ip4[1] == 0 && ip4[2] == 0) || //private network + (ip4[1] == 0 && ip4[2] == 2) || //documentation + (ip4[1] == 88 && ip4[2] == 99) || //reserved + (ip4[1] == 168) { //private network + return false + } + case 198: + if (ip4[1] == 18 || ip4[2] == 19) || //private network + (ip4[1] == 51 || ip4[2] == 100) { //documentation + return false + } + case 203: + if ip4[1] == 0 && ip4[2] == 113 { //documentation + return false + } + case 224: + if ip4[1] == 0 && ip4[2] == 0 { //multicast + return false + } + case 255: + if ip4[1] == 255 && ip4[2] == 255 && ip4[3] == 255 { //subnet + return false + } + } + } else { + if ip.IsLoopback() || ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast() { + return false + } + } + + return true +} + func onDNSRequest(d *proxy.DNSContext) { qType := d.Req.Question[0].Qtype if qType != dns.TypeA && qType != dns.TypeAAAA { @@ -79,6 +135,9 @@ func onDNSRequest(d *proxy.DNSContext) { if !ipAddr.IsLoopback() { beginAsyncRDNS(ip) } + if isPublicIP(ipAddr) { + config.dnsctx.whois.Begin(ip) + } } func generateServerConfig() (dnsforward.ServerConfig, error) { diff --git a/home/dns_test.go b/home/dns_test.go index dd6ba8cb..67d7b3ed 100644 --- a/home/dns_test.go +++ b/home/dns_test.go @@ -7,7 +7,7 @@ import ( func TestResolveRDNS(t *testing.T) { config.DNS.BindHost = "1.1.1.1" initDNSServer(".") - if r := resolveRDNS("1.1.1.1"); r != "one.one.one.one" { + if r := config.dnsctx.rdns.resolve("1.1.1.1"); r != "one.one.one.one" { t.Errorf("resolveRDNS(): %s", r) } } diff --git a/home/rdns.go b/home/rdns.go index 9ea11a26..048dcde1 100644 --- a/home/rdns.go +++ b/home/rdns.go @@ -39,7 +39,7 @@ func initRDNS() { // Add IP address to the rDNS queue func beginAsyncRDNS(ip string) { - if config.clients.Exists(ip) { + if config.clients.Exists(ip, ClientSourceRDNS) { return } diff --git a/home/whois.go b/home/whois.go new file mode 100644 index 00000000..f5dd2ab2 --- /dev/null +++ b/home/whois.go @@ -0,0 +1,118 @@ +package home + +import ( + "strings" + "sync" + + "github.com/AdguardTeam/golibs/log" + whois "github.com/likexian/whois-go" +) + +// Whois - module context +type Whois struct { + clients *clientsContainer + ips map[string]bool + lock sync.Mutex + ipChan chan string +} + +// Create module context +func initWhois(clients *clientsContainer) *Whois { + w := Whois{} + w.clients = clients + w.ips = make(map[string]bool) + w.ipChan = make(chan string, 255) + go w.workerLoop() + return &w +} + +// Parse plain-text data from the response +func whoisParse(data string) map[string]string { + m := map[string]string{} + lines := strings.Split(data, "\n") + for _, ln := range lines { + ln = strings.TrimSpace(ln) + + if len(ln) == 0 || ln[0] == '#' { + continue + } + + kv := strings.SplitN(ln, ":", 2) + if len(kv) != 2 { + continue + } + k := strings.TrimSpace(kv[0]) + k = strings.ToLower(k) + v := strings.TrimSpace(kv[1]) + + if k == "orgname" || k == "org-name" { + m["orgname"] = v + } else if k == "city" { + m["city"] = v + } else if k == "country" { + m["country"] = v + } + } + return m +} + +// Request WHOIS information +func whoisProcess(ip string) [][]string { + data := [][]string{} + resp, err := whois.Whois(ip) + if err != nil { + log.Debug("Whois: error: %s IP:%s", err, ip) + return data + } + + log.Debug("Whois: IP:%s response: %d bytes", ip, len(resp)) + + m := whoisParse(resp) + + keys := []string{"orgname", "country", "city"} + for _, k := range keys { + v, found := m[k] + if !found { + continue + } + pair := []string{k, v} + data = append(data, pair) + } + + return data +} + +// Begin - begin requesting WHOIS info +func (w *Whois) Begin(ip string) { + w.lock.Lock() + _, found := w.ips[ip] + if found { + w.lock.Unlock() + return + } + w.ips[ip] = true + w.lock.Unlock() + + log.Debug("Whois: adding %s", ip) + select { + case w.ipChan <- ip: + // + default: + log.Debug("Whois: queue is full") + } +} + +// Get IP address from channel; get WHOIS info; associate info with a client +func (w *Whois) workerLoop() { + for { + var ip string + ip = <-w.ipChan + + info := whoisProcess(ip) + if len(info) == 0 { + continue + } + + w.clients.SetWhoisInfo(ip, info) + } +} diff --git a/home/whois_test.go b/home/whois_test.go new file mode 100644 index 00000000..7a841110 --- /dev/null +++ b/home/whois_test.go @@ -0,0 +1,21 @@ +package home + +import ( + "strings" + "testing" + + whois "github.com/likexian/whois-go" + "github.com/stretchr/testify/assert" +) + +func TestWhois(t *testing.T) { + resp, err := whois.Whois("8.8.8.8") + assert.True(t, err == nil) + assert.True(t, strings.Index(resp, "OrgName: Google LLC") != -1) + assert.True(t, strings.Index(resp, "City: Mountain View") != -1) + assert.True(t, strings.Index(resp, "Country: US") != -1) + m := whoisParse(resp) + assert.True(t, m["orgname"] == "Google LLC") + assert.True(t, m["country"] == "US") + assert.True(t, m["city"] == "Mountain View") +}