+ whois: add WHOIS information for a client

+ return WHOIS info in GET /clients response

AGHTechDoc.md

@@ -650,6 +650,10 @@ Response:
 			safesearch_enabled: false
 			use_global_blocked_services: true
 			blocked_services: [ "name1", ... ]
+			whois_info: {
+				key: "value"
+				...
+			}
 		}
 	]
 	auto_clients: [
@@ -657,10 +661,16 @@ Response:
 			name: "host"
 			ip: "..."
 			source: "etc/hosts" || "rDNS"
+			whois_info: {
+				key: "value"
+				...
+			}
 		}
 	]
 	}
 
+Supported keys for `whois_info`: orgname, country, city.
+
 
 ### Add client
 

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
 	github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b
 	github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414
+	github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49
 	github.com/miekg/dns v1.1.8
 	github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0
 	github.com/stretchr/testify v1.4.0

go.sum

@@ -53,6 +53,16 @@ github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b h1:vfiqKno48aUnd
 github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b/go.mod h1:10UU/bEkzh2iEN6aYzbevY7J6p03KO5siTxQWXMEerg=
 github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414 h1:6wnYc2S/lVM7BvR32BM74ph7bPgqMztWopMYKgVyEho=
 github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414/go.mod h1:0AqAH3ZogsCrvrtUpvc6EtVKbc3w6xwZhkvGLuqyi3o=
+github.com/likexian/gokit v0.0.0-20190309162924-0a377eecf7aa/go.mod h1:QdfYv6y6qPA9pbBA2qXtoT8BMKha6UyNbxWGWl/9Jfk=
+github.com/likexian/gokit v0.0.0-20190418170008-ace88ad0983b/go.mod h1:KKqSnk/VVSW8kEyO2vVCXoanzEutKdlBAPohmGXkxCk=
+github.com/likexian/gokit v0.0.0-20190501133040-e77ea8b19cdc/go.mod h1:3kvONayqCaj+UgrRZGpgfXzHdMYCAO0KAt4/8n0L57Y=
+github.com/likexian/gokit v0.0.0-20190604165112-68b8a4ba758c h1:KByA4IxKqqYwpqzk/P+w1DBpkPbvy3DArTP/U3LSxTQ=
+github.com/likexian/gokit v0.0.0-20190604165112-68b8a4ba758c/go.mod h1:kn+nTv3tqh6yhor9BC4Lfiu58SmH8NmQ2PmEl+uM6nU=
+github.com/likexian/simplejson-go v0.0.0-20190409170913-40473a74d76d/go.mod h1:Typ1BfnATYtZ/+/shXfFYLrovhFyuKvzwrdOnIDHlmg=
+github.com/likexian/simplejson-go v0.0.0-20190419151922-c1f9f0b4f084/go.mod h1:U4O1vIJvIKwbMZKUJ62lppfdvkCdVd2nfMimHK81eec=
+github.com/likexian/simplejson-go v0.0.0-20190502021454-d8787b4bfa0b/go.mod h1:3BWwtmKP9cXWwYCr5bkoVDEfLywacOv0s06OBEDpyt8=
+github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49 h1:xGa+flE6p2UnMgxIS8bm7Q9JSt47HRuYVtwneDVnfLk=
+github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49/go.mod h1:oR3bJMzrOb55cqTAn14DEzYFLDpSPTXJ3ORe7go9Hc8=
 github.com/markbates/oncer v0.0.0-20181014194634-05fccaae8fc4 h1:Mlji5gkcpzkqTROyE4ZxZ8hN7osunMb2RuGVrbvMvCc=
 github.com/markbates/oncer v0.0.0-20181014194634-05fccaae8fc4/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI=

home/clients.go

@@ -31,6 +31,7 @@ type Client struct {
 	SafeSearchEnabled   bool
 	SafeBrowsingEnabled bool
 	ParentalEnabled     bool
+	WhoisInfo           [][]string // [[key,value], ...]
 
 	UseOwnBlockedServices bool // false: use global settings
 	BlockedServices       []string
@@ -46,29 +47,34 @@ type clientJSON struct {
 	SafeSearchEnabled   bool   `json:"safebrowsing_enabled"`
 	SafeBrowsingEnabled bool   `json:"safesearch_enabled"`
 
+	WhoisInfo map[string]interface{} `json:"whois_info"`
+
 	UseGlobalBlockedServices bool     `json:"use_global_blocked_services"`
 	BlockedServices          []string `json:"blocked_services"`
 }
 
 type clientSource uint
 
+// Client sources
 const (
-	// Priority: etc/hosts > DHCP > ARP > rDNS
+	// Priority: etc/hosts > DHCP > ARP > rDNS > WHOIS
-	ClientSourceRDNS      clientSource = 0 // from rDNS
+	ClientSourceWHOIS     clientSource = iota // from WHOIS
-	ClientSourceDHCP      clientSource = 1 // from DHCP
+	ClientSourceRDNS                          // from rDNS
-	ClientSourceARP       clientSource = 2 // from 'arp -a'
+	ClientSourceDHCP                          // from DHCP
-	ClientSourceHostsFile clientSource = 3 // from /etc/hosts
+	ClientSourceARP                           // from 'arp -a'
+	ClientSourceHostsFile                     // from /etc/hosts
 )
 
 // ClientHost information
 type ClientHost struct {
 	Host      string
 	Source    clientSource
+	WhoisInfo [][]string // [[key,value], ...]
 }
 
 type clientsContainer struct {
-	list    map[string]*Client
+	list    map[string]*Client    // name -> client
-	ipIndex map[string]*Client
+	ipIndex map[string]*Client    // IP -> client
 	ipHost  map[string]ClientHost // IP -> Hostname
 	lock    sync.Mutex
 }
@@ -101,7 +107,7 @@ func (clients *clientsContainer) GetList() map[string]*Client {
 }
 
 // Exists checks if client with this IP already exists
-func (clients *clientsContainer) Exists(ip string) bool {
+func (clients *clientsContainer) Exists(ip string, source clientSource) bool {
 	clients.lock.Lock()
 	defer clients.lock.Unlock()
 
@@ -110,8 +116,14 @@ func (clients *clientsContainer) Exists(ip string) bool {
 		return true
 	}
 
-	_, ok = clients.ipHost[ip]
+	ch, ok := clients.ipHost[ip]
-	return ok
+	if !ok {
+		return false
+	}
+	if source > ch.Source {
+		return false // we're going to overwrite this client's info with a stronger source
+	}
+	return true
 }
 
 // Find searches for a client by IP
@@ -266,6 +278,31 @@ func (clients *clientsContainer) Update(name string, c Client) error {
 	return nil
 }
 
+// SetWhoisInfo - associate WHOIS information with a client
+func (clients *clientsContainer) SetWhoisInfo(ip string, info [][]string) {
+	clients.lock.Lock()
+	defer clients.lock.Unlock()
+
+	c, ok := clients.ipIndex[ip]
+	if ok {
+		c.WhoisInfo = info
+		log.Debug("Clients: set WHOIS info for client %s: %v", c.Name, c.WhoisInfo)
+	}
+
+	ch, ok := clients.ipHost[ip]
+	if ok {
+		ch.WhoisInfo = info
+		log.Debug("Clients: set WHOIS info for auto-client %s: %v", ch.Host, ch.WhoisInfo)
+	}
+
+	ch = ClientHost{
+		Source: ClientSourceWHOIS,
+	}
+	ch.WhoisInfo = info
+	clients.ipHost[ip] = ch
+	log.Debug("Clients: set WHOIS info for auto-client with IP %s: %v", ip, ch.WhoisInfo)
+}
+
 // AddHost adds new IP -> Host pair
 // Use priority of the source (etc/hosts > ARP > rDNS)
 //  so we overwrite existing entries with an equal or higher priority
@@ -282,6 +319,7 @@ func (clients *clientsContainer) AddHost(ip, host string, source clientSource) (
 	clients.ipHost[ip] = ClientHost{
 		Host:      host,
 		Source:    source,
+		WhoisInfo: c.WhoisInfo,
 	}
 	log.Tracef("'%s' -> '%s' [%d]", ip, host, len(clients.ipHost))
 	return true, nil
@@ -386,6 +424,8 @@ type clientHostJSON struct {
 	IP     string `json:"ip"`
 	Name   string `json:"name"`
 	Source string `json:"source"`
+
+	WhoisInfo map[string]interface{} `json:"whois_info"`
 }
 
 type clientListJSON struct {
@@ -421,6 +461,11 @@ func handleGetClients(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
+		cj.WhoisInfo = make(map[string]interface{})
+		for _, wi := range c.WhoisInfo {
+			cj.WhoisInfo[wi[0]] = wi[1]
+		}
+
 		data.Clients = append(data.Clients, cj)
 	}
 	for ip, ch := range config.clients.ipHost {
@@ -428,6 +473,7 @@ func handleGetClients(w http.ResponseWriter, r *http.Request) {
 			IP:   ip,
 			Name: ch.Host,
 		}
+
 		cj.Source = "etc/hosts"
 		switch ch.Source {
 		case ClientSourceDHCP:
@@ -436,7 +482,15 @@ func handleGetClients(w http.ResponseWriter, r *http.Request) {
 			cj.Source = "rDNS"
 		case ClientSourceARP:
 			cj.Source = "ARP"
+		case ClientSourceWHOIS:
+			cj.Source = "WHOIS"
 		}
+
+		cj.WhoisInfo = make(map[string]interface{})
+		for _, wi := range ch.WhoisInfo {
+			cj.WhoisInfo[wi[0]] = wi[1]
+		}
+
 		data.AutoClients = append(data.AutoClients, cj)
 	}
 	config.clients.lock.Unlock()

home/clients_test.go

@@ -1,6 +1,10 @@
 package home
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
 
 func TestClients(t *testing.T) {
 	var c Client
@@ -61,15 +65,9 @@ func TestClients(t *testing.T) {
 	}
 
 	// get
-	if clients.Exists("1.2.3.4") {
+	assert.True(t, !clients.Exists("1.2.3.4", ClientSourceHostsFile))
-		t.Fatalf("Exists")
+	assert.True(t, clients.Exists("1.1.1.1", ClientSourceHostsFile))
-	}
+	assert.True(t, clients.Exists("2.2.2.2", ClientSourceHostsFile))
-	if !clients.Exists("1.1.1.1") {
-		t.Fatalf("Exists #1")
-	}
-	if !clients.Exists("2.2.2.2") {
-		t.Fatalf("Exists #2")
-	}
 
 	// failed update - no such name
 	c.IP = "1.2.3.0"
@@ -100,9 +98,7 @@ func TestClients(t *testing.T) {
 	}
 
 	// get after update
-	if clients.Exists("1.1.1.1") || !clients.Exists("1.1.1.2") {
+	assert.True(t, !(clients.Exists("1.1.1.1", ClientSourceHostsFile) || !clients.Exists("1.1.1.2", ClientSourceHostsFile)))
-		t.Fatalf("Exists - get after update")
-	}
 
 	// failed remove - no such name
 	if clients.Del("client3") {
@@ -110,9 +106,7 @@ func TestClients(t *testing.T) {
 	}
 
 	// remove
-	if !clients.Del("client1") || clients.Exists("1.1.1.2") {
+	assert.True(t, !(!clients.Del("client1") || clients.Exists("1.1.1.2", ClientSourceHostsFile)))
-		t.Fatalf("Del")
-	}
 
 	// add host client
 	b, e = clients.AddHost("1.1.1.1", "host", ClientSourceARP)
@@ -139,7 +133,5 @@ func TestClients(t *testing.T) {
 	}
 
 	// get
-	if !clients.Exists("1.1.1.1") {
+	assert.True(t, clients.Exists("1.1.1.1", ClientSourceHostsFile))
-		t.Fatalf("clientAddHost")
-	}
 }

home/dns.go

@@ -25,6 +25,8 @@ type dnsContext struct {
 	rdnsIP   map[string]bool
 	rdnsLock sync.Mutex        // synchronize access to rdnsIP
 	upstream upstream.Upstream // Upstream object for our own DNS server
+
+	whois *Whois
 }
 
 // initDNSServer creates an instance of the dnsforward.Server
@@ -56,6 +58,7 @@ func initDNSServer(baseDir string) {
 	config.Users = nil
 
 	initRDNS()
+	config.dnsctx.whois = initWhois(&config.clients)
 	initFiltering()
 }
 
@@ -63,6 +66,59 @@ func isRunning() bool {
 	return config.dnsServer != nil && config.dnsServer.IsRunning()
 }
 
+// Return TRUE if IP is within public Internet IP range
+func isPublicIP(ip net.IP) bool {
+	ip4 := ip.To4()
+	if ip4 != nil {
+		switch ip4[0] {
+		case 0:
+			return false //software
+		case 10:
+			return false //private network
+		case 127:
+			return false //loopback
+		case 169:
+			if ip4[1] == 254 {
+				return false //link-local
+			}
+		case 172:
+			if ip4[1] >= 16 && ip4[1] <= 31 {
+				return false //private network
+			}
+		case 192:
+			if (ip4[1] == 0 && ip4[2] == 0) || //private network
+				(ip4[1] == 0 && ip4[2] == 2) || //documentation
+				(ip4[1] == 88 && ip4[2] == 99) || //reserved
+				(ip4[1] == 168) { //private network
+				return false
+			}
+		case 198:
+			if (ip4[1] == 18 || ip4[2] == 19) || //private network
+				(ip4[1] == 51 || ip4[2] == 100) { //documentation
+				return false
+			}
+		case 203:
+			if ip4[1] == 0 && ip4[2] == 113 { //documentation
+				return false
+			}
+		case 224:
+			if ip4[1] == 0 && ip4[2] == 0 { //multicast
+				return false
+			}
+		case 255:
+			if ip4[1] == 255 && ip4[2] == 255 && ip4[3] == 255 { //subnet
+				return false
+			}
+		}
+	} else {
+		if ip.IsLoopback() || ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast() {
+			return false
+		}
+	}
+
+	return true
+}
+
 func onDNSRequest(d *proxy.DNSContext) {
 	qType := d.Req.Question[0].Qtype
 	if qType != dns.TypeA && qType != dns.TypeAAAA {
@@ -79,6 +135,9 @@ func onDNSRequest(d *proxy.DNSContext) {
 	if !ipAddr.IsLoopback() {
 		beginAsyncRDNS(ip)
 	}
+	if isPublicIP(ipAddr) {
+		config.dnsctx.whois.Begin(ip)
+	}
 }
 
 func generateServerConfig() (dnsforward.ServerConfig, error) {

home/dns_test.go

@@ -7,7 +7,7 @@ import (
 func TestResolveRDNS(t *testing.T) {
 	config.DNS.BindHost = "1.1.1.1"
 	initDNSServer(".")
-	if r := resolveRDNS("1.1.1.1"); r != "one.one.one.one" {
+	if r := config.dnsctx.rdns.resolve("1.1.1.1"); r != "one.one.one.one" {
 		t.Errorf("resolveRDNS(): %s", r)
 	}
 }

home/rdns.go

@@ -39,7 +39,7 @@ func initRDNS() {
 
 // Add IP address to the rDNS queue
 func beginAsyncRDNS(ip string) {
-	if config.clients.Exists(ip) {
+	if config.clients.Exists(ip, ClientSourceRDNS) {
 		return
 	}
 

home/whois.go

@@ -0,0 +1,118 @@
+package home
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/AdguardTeam/golibs/log"
+	whois "github.com/likexian/whois-go"
+)
+
+// Whois - module context
+type Whois struct {
+	clients *clientsContainer
+	ips     map[string]bool
+	lock    sync.Mutex
+	ipChan  chan string
+}
+
+// Create module context
+func initWhois(clients *clientsContainer) *Whois {
+	w := Whois{}
+	w.clients = clients
+	w.ips = make(map[string]bool)
+	w.ipChan = make(chan string, 255)
+	go w.workerLoop()
+	return &w
+}
+
+// Parse plain-text data from the response
+func whoisParse(data string) map[string]string {
+	m := map[string]string{}
+	lines := strings.Split(data, "\n")
+	for _, ln := range lines {
+		ln = strings.TrimSpace(ln)
+
+		if len(ln) == 0 || ln[0] == '#' {
+			continue
+		}
+
+		kv := strings.SplitN(ln, ":", 2)
+		if len(kv) != 2 {
+			continue
+		}
+		k := strings.TrimSpace(kv[0])
+		k = strings.ToLower(k)
+		v := strings.TrimSpace(kv[1])
+
+		if k == "orgname" || k == "org-name" {
+			m["orgname"] = v
+		} else if k == "city" {
+			m["city"] = v
+		} else if k == "country" {
+			m["country"] = v
+		}
+	}
+	return m
+}
+
+// Request WHOIS information
+func whoisProcess(ip string) [][]string {
+	data := [][]string{}
+	resp, err := whois.Whois(ip)
+	if err != nil {
+		log.Debug("Whois: error: %s  IP:%s", err, ip)
+		return data
+	}
+
+	log.Debug("Whois: IP:%s  response: %d bytes", ip, len(resp))
+
+	m := whoisParse(resp)
+
+	keys := []string{"orgname", "country", "city"}
+	for _, k := range keys {
+		v, found := m[k]
+		if !found {
+			continue
+		}
+		pair := []string{k, v}
+		data = append(data, pair)
+	}
+
+	return data
+}
+
+// Begin - begin requesting WHOIS info
+func (w *Whois) Begin(ip string) {
+	w.lock.Lock()
+	_, found := w.ips[ip]
+	if found {
+		w.lock.Unlock()
+		return
+	}
+	w.ips[ip] = true
+	w.lock.Unlock()
+
+	log.Debug("Whois: adding %s", ip)
+	select {
+	case w.ipChan <- ip:
+		//
+	default:
+		log.Debug("Whois: queue is full")
+	}
+}
+
+// Get IP address from channel; get WHOIS info; associate info with a client
+func (w *Whois) workerLoop() {
+	for {
+		var ip string
+		ip = <-w.ipChan
+
+		info := whoisProcess(ip)
+		if len(info) == 0 {
+			continue
+		}
+
+		w.clients.SetWhoisInfo(ip, info)
+	}
+}

home/whois_test.go

@@ -0,0 +1,21 @@
+package home
+
+import (
+	"strings"
+	"testing"
+
+	whois "github.com/likexian/whois-go"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWhois(t *testing.T) {
+	resp, err := whois.Whois("8.8.8.8")
+	assert.True(t, err == nil)
+	assert.True(t, strings.Index(resp, "OrgName:        Google LLC") != -1)
+	assert.True(t, strings.Index(resp, "City:           Mountain View") != -1)
+	assert.True(t, strings.Index(resp, "Country:        US") != -1)
+	m := whoisParse(resp)
+	assert.True(t, m["orgname"] == "Google LLC")
+	assert.True(t, m["country"] == "US")
+	assert.True(t, m["city"] == "Mountain View")
+}