From 06d465b0d128c1e87432a34c69b65e5f3095a1b6 Mon Sep 17 00:00:00 2001 From: Stanislav Chzhen Date: Wed, 21 Jun 2023 12:53:53 +0300 Subject: [PATCH] Pull request 1858: AG-22594-imp-whois Merge in DNS/adguard-home from AG-22594-imp-whois to master Squashed commit of the following: commit 093feed53291d02469fb1bd8d99472597ebd5015 Merge: 956d20dc4 ca313521d Author: Stanislav Chzhen Date: Wed Jun 21 12:42:40 2023 +0300 Merge branch 'master' into AG-22594-imp-whois commit 956d20dc473dcec90895b6f618fc56e96e9ff833 Author: Stanislav Chzhen Date: Tue Jun 20 18:30:48 2023 +0300 whois: imp code more commit c771fd9c5e4d90e76d079a0d25ab097ab5652a42 Author: Stanislav Chzhen Date: Tue Jun 20 15:05:45 2023 +0300 whois: imp code commit 21900fd468e10d9aee22149a6312b8596ff39810 Merge: 8dbe132c0 371261b2c Author: Stanislav Chzhen Date: Tue Jun 20 11:34:06 2023 +0300 Merge branch 'master' into AG-22594-imp-whois commit 8dbe132c08d3ad4a63b0d4bdb9d00a5bc25971f4 Author: Stanislav Chzhen Date: Tue Jun 20 11:33:26 2023 +0300 whois: imp code more commit f5e761a260237579c67cbd48f01ea90499bff6b0 Author: Stanislav Chzhen Date: Mon Jun 19 16:04:35 2023 +0300 whois: imp code commit 2780f7e16aacddad8736f83b77ef9bfa1271f8b1 Author: Stanislav Chzhen Date: Fri Jun 16 17:33:47 2023 +0300 all: imp code commit 1fc67016068b745a46b3d0d341ab14f9f5bdc9aa Author: Stanislav Chzhen Date: Fri Jun 16 17:29:19 2023 +0300 whois: imp tests commit 204761870764fb10feea20065d79dee8c321e70b Author: Stanislav Chzhen Date: Fri Jun 16 11:55:37 2023 +0300 all: upd deps commit ded4f59498c5c544277b9c8e249567626547680e Author: Stanislav Chzhen Date: Wed Jun 14 20:43:32 2023 +0300 all: imp tests commit 0eed9834ff9dd94d0788ce69d0bb0521fa725410 Author: Stanislav Chzhen Date: Wed Jun 14 19:31:49 2023 +0300 all: imp code commit 9f867587c8ad87363b8c8b061ead536c1ec59c5d Merge: 504e9484d 681c604c2 Author: Stanislav Chzhen Date: Tue Jun 13 14:20:44 2023 +0300 Merge branch 'master' into AG-22594-imp-whois commit 504e9484dd84ab9d7c84a3f8399993d6422d3b67 Author: Stanislav Chzhen Date: Tue Jun 13 14:18:06 2023 +0300 all: imp cache commit c492abe41ace7ad76fcd4e297c22b910a90fec30 Merge: db36adb9c 826b314f1 Author: Stanislav Chzhen Date: Fri Jun 9 16:06:12 2023 +0300 Merge branch 'master' into AG-22594-imp-whois commit db36adb9c14ce92b3971db0d87ec313d5bcd787e Author: Stanislav Chzhen Date: Fri Jun 9 15:53:33 2023 +0300 all: add todo commit 5cf192de9f93cd0d8521a3a6b4ded7f2bc5e0031 Author: Stanislav Chzhen Date: Thu Jun 8 14:59:26 2023 +0300 all: imp docs commit 021aa3eb5b9476a93b4af5fc90cc9ccf014ca152 Author: Stanislav Chzhen Date: Mon Jun 5 18:35:25 2023 +0300 all: imp naming commit 4626c3a7fa3f2543501806c9fa1a19531547f394 Author: Stanislav Chzhen Date: Fri Jun 2 17:41:00 2023 +0300 all: imp tests commit 1afcc9605ca176e4c7f76a03a2c996cf7d6bde13 Author: Stanislav Chzhen Date: Fri Jun 2 12:44:32 2023 +0300 all: imp docs commit cdd0544ff1a63faed5ced3dae6bfb3b783e45428 Author: Stanislav Chzhen Date: Thu Jun 1 17:21:37 2023 +0300 all: add docs ... and 2 more commits --- go.mod | 14 +- go.sum | 28 ++- internal/home/client.go | 18 +- internal/home/clients.go | 29 +-- internal/home/clients_test.go | 8 +- internal/home/clientshttp.go | 16 +- internal/home/dns.go | 77 ++++++- internal/home/home.go | 4 +- internal/home/whois.go | 259 ----------------------- internal/home/whois_test.go | 152 -------------- internal/querylog/client.go | 22 +- internal/whois/whois.go | 376 ++++++++++++++++++++++++++++++++++ internal/whois/whois_test.go | 155 ++++++++++++++ 13 files changed, 658 insertions(+), 500 deletions(-) delete mode 100644 internal/home/whois.go delete mode 100644 internal/home/whois_test.go create mode 100644 internal/whois/whois.go create mode 100644 internal/whois/whois_test.go diff --git a/go.mod b/go.mod index 2a3c0947..bf2f3e10 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,11 @@ go 1.19 require ( github.com/AdguardTeam/dnsproxy v0.50.2 - github.com/AdguardTeam/golibs v0.13.2 + github.com/AdguardTeam/golibs v0.13.3 github.com/AdguardTeam/urlfilter v0.16.1 github.com/NYTimes/gziphandler v1.1.1 github.com/ameshkov/dnscrypt/v2 v2.2.7 + github.com/bluele/gcache v0.0.2 github.com/digineo/go-ipset/v2 v2.2.1 github.com/dimfeld/httptreemux/v5 v5.5.0 github.com/fsnotify/fsnotify v1.6.0 @@ -27,13 +28,13 @@ require ( github.com/mdlayher/raw v0.1.0 github.com/miekg/dns v1.1.54 github.com/quic-go/quic-go v0.35.1 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 github.com/ti-mo/netfilter v0.5.0 go.etcd.io/bbolt v1.3.7 - golang.org/x/crypto v0.9.0 + golang.org/x/crypto v0.10.0 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 - golang.org/x/net v0.10.0 - golang.org/x/sys v0.8.0 + golang.org/x/net v0.11.0 + golang.org/x/sys v0.9.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 howett.net/plist v1.0.0 @@ -44,7 +45,6 @@ require ( github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 // indirect github.com/ameshkov/dnsstamps v1.0.3 // indirect github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0 // indirect - github.com/bluele/gcache v0.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/golang/mock v1.6.0 // indirect @@ -61,6 +61,6 @@ require ( github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/sync v0.2.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect golang.org/x/tools v0.9.3 // indirect ) diff --git a/go.sum b/go.sum index ce08ec3e..2f07b778 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/AdguardTeam/dnsproxy v0.50.2 h1:p1471SsMZ6SMo7T51Olw4aNluahvMwSLMorwx github.com/AdguardTeam/dnsproxy v0.50.2/go.mod h1:CQhZTkqC8X0ID6glrtyaxgqRRdiYfn1gJulC1cZ5Dn8= github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4= github.com/AdguardTeam/golibs v0.10.4/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw= -github.com/AdguardTeam/golibs v0.13.2 h1:BPASsyQKmb+b8VnvsNOHp7bKfcZl9Z+Z2UhPjOiupSc= -github.com/AdguardTeam/golibs v0.13.2/go.mod h1:7ylQLv2Lqsc3UW3jHoITynYk6Y1tYtgEMkR09ppfsN8= +github.com/AdguardTeam/golibs v0.13.3 h1:RT3QbzThtaLiFLkIUDS6/hlGEXrh0zYvdf4bd7UWpGo= +github.com/AdguardTeam/golibs v0.13.3/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI= github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU= github.com/AdguardTeam/urlfilter v0.16.1 h1:ZPi0rjqo8cQf2FVdzo6cqumNoHZx2KPXj2yZa1A5BBw= github.com/AdguardTeam/urlfilter v0.16.1/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI= @@ -113,17 +113,13 @@ github.com/quic-go/quic-go v0.35.1/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5 github.com/shirou/gopsutil/v3 v3.21.8 h1:nKct+uP0TV8DjjNiHanKf8SAuub+GNsbrOtM9Nl9biA= github.com/shirou/gopsutil/v3 v3.21.8/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/ti-mo/netfilter v0.2.0/go.mod h1:8GbBGsY/8fxtyIdfwy29JiluNcPK4K7wIT+x42ipqUU= github.com/ti-mo/netfilter v0.5.0 h1:MZmsUw5bFRecOb0AeyjOPxTHg4UxYzyEs0Ek/6Lxoy8= github.com/ti-mo/netfilter v0.5.0/go.mod h1:nt+8B9hx/QpqHr7Hazq+2qMCCA8u2OTkyc/7+U9ARz8= @@ -138,8 +134,8 @@ go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -156,8 +152,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= @@ -181,16 +177,16 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= diff --git a/internal/home/client.go b/internal/home/client.go index 1aee021e..92c88385 100644 --- a/internal/home/client.go +++ b/internal/home/client.go @@ -7,6 +7,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/AdGuardHome/internal/filtering/safesearch" + "github.com/AdguardTeam/AdGuardHome/internal/whois" "github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/golibs/stringutil" ) @@ -127,14 +128,13 @@ func (cs clientSource) MarshalText() (text []byte, err error) { // RuntimeClient is a client information about which has been obtained using the // source described in the Source field. type RuntimeClient struct { - WHOISInfo *RuntimeClientWHOISInfo - Host string - Source clientSource -} + // WHOIS is the filtered WHOIS data of a client. + WHOIS *whois.Info -// RuntimeClientWHOISInfo is the filtered WHOIS data for a runtime client. -type RuntimeClientWHOISInfo struct { - City string `json:"city,omitempty"` - Country string `json:"country,omitempty"` - Orgname string `json:"orgname,omitempty"` + // Host is the host name of a client. + Host string + + // Source is the source from which the information about the client has + // been obtained. + Source clientSource } diff --git a/internal/home/clients.go b/internal/home/clients.go index d2e4194b..6862cf25 100644 --- a/internal/home/clients.go +++ b/internal/home/clients.go @@ -14,6 +14,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/dnsforward" "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/AdGuardHome/internal/querylog" + "github.com/AdguardTeam/AdGuardHome/internal/whois" "github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/dnsproxy/upstream" "github.com/AdguardTeam/golibs/errors" @@ -307,18 +308,6 @@ func (clients *clientsContainer) clientSource(ip netip.Addr) (src clientSource) return rc.Source } -func toQueryLogWHOIS(wi *RuntimeClientWHOISInfo) (cw *querylog.ClientWHOIS) { - if wi == nil { - return &querylog.ClientWHOIS{} - } - - return &querylog.ClientWHOIS{ - City: wi.City, - Country: wi.Country, - Orgname: wi.Orgname, - } -} - // findMultiple is a wrapper around Find to make it a valid client finder for // the query log. c is never nil; if no information about the client is found, // it returns an artificial client record by only setting the blocking-related @@ -352,7 +341,7 @@ func (clients *clientsContainer) clientOrArtificial( defer func() { c.Disallowed, c.DisallowedRule = clients.dnsServer.IsBlockedClient(ip, id) if c.WHOIS == nil { - c.WHOIS = &querylog.ClientWHOIS{} + c.WHOIS = &whois.Info{} } }() @@ -369,7 +358,7 @@ func (clients *clientsContainer) clientOrArtificial( if ok { return &querylog.Client{ Name: rc.Host, - WHOIS: toQueryLogWHOIS(rc.WHOISInfo), + WHOIS: rc.WHOIS, }, false } @@ -701,7 +690,7 @@ func (clients *clientsContainer) Update(prev, c *Client) (err error) { } // setWHOISInfo sets the WHOIS information for a client. -func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *RuntimeClientWHOISInfo) { +func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *whois.Info) { clients.lock.Lock() defer clients.lock.Unlock() @@ -713,7 +702,7 @@ func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *RuntimeClientWH rc, ok := clients.ipToRC[ip] if ok { - rc.WHOISInfo = wi + rc.WHOIS = wi log.Debug("clients: set whois info for runtime client %s: %+v", rc.Host, wi) return @@ -725,7 +714,7 @@ func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *RuntimeClientWH Source: ClientSourceWHOIS, } - rc.WHOISInfo = wi + rc.WHOIS = wi clients.ipToRC[ip] = rc @@ -762,9 +751,9 @@ func (clients *clientsContainer) addHostLocked( rc.Source = src } else { rc = &RuntimeClient{ - Host: host, - Source: src, - WHOISInfo: &RuntimeClientWHOISInfo{}, + Host: host, + Source: src, + WHOIS: &whois.Info{}, } clients.ipToRC[ip] = rc diff --git a/internal/home/clients_test.go b/internal/home/clients_test.go index 8361528a..b203415f 100644 --- a/internal/home/clients_test.go +++ b/internal/home/clients_test.go @@ -9,7 +9,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/dhcpd" "github.com/AdguardTeam/AdGuardHome/internal/filtering" - + "github.com/AdguardTeam/AdGuardHome/internal/whois" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -199,7 +199,7 @@ func TestClients(t *testing.T) { func TestClientsWHOIS(t *testing.T) { clients := newClientsContainer() - whois := &RuntimeClientWHOISInfo{ + whois := &whois.Info{ Country: "AU", Orgname: "Example Org", } @@ -210,7 +210,7 @@ func TestClientsWHOIS(t *testing.T) { rc := clients.ipToRC[ip] require.NotNil(t, rc) - assert.Equal(t, rc.WHOISInfo, whois) + assert.Equal(t, rc.WHOIS, whois) }) t.Run("existing_auto-client", func(t *testing.T) { @@ -222,7 +222,7 @@ func TestClientsWHOIS(t *testing.T) { rc := clients.ipToRC[ip] require.NotNil(t, rc) - assert.Equal(t, rc.WHOISInfo, whois) + assert.Equal(t, rc.WHOIS, whois) }) t.Run("can't_set_manually-added", func(t *testing.T) { diff --git a/internal/home/clientshttp.go b/internal/home/clientshttp.go index 6425f941..9eb91341 100644 --- a/internal/home/clientshttp.go +++ b/internal/home/clientshttp.go @@ -9,6 +9,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghalg" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/filtering" + "github.com/AdguardTeam/AdGuardHome/internal/whois" ) // clientJSON is a common structure used by several handlers to deal with @@ -28,7 +29,8 @@ type clientJSON struct { // the allowlist. DisallowedRule *string `json:"disallowed_rule,omitempty"` - WHOISInfo *RuntimeClientWHOISInfo `json:"whois_info,omitempty"` + // WHOIS is the filtered WHOIS data of a client. + WHOIS *whois.Info `json:"whois_info,omitempty"` SafeSearchConf *filtering.SafeSearchConfig `json:"safe_search"` Name string `json:"name"` @@ -51,7 +53,7 @@ type clientJSON struct { } type runtimeClientJSON struct { - WHOISInfo *RuntimeClientWHOISInfo `json:"whois_info"` + WHOIS *whois.Info `json:"whois_info"` IP netip.Addr `json:"ip"` Name string `json:"name"` @@ -78,7 +80,7 @@ func (clients *clientsContainer) handleGetClients(w http.ResponseWriter, r *http for ip, rc := range clients.ipToRC { cj := runtimeClientJSON{ - WHOISInfo: rc.WHOISInfo, + WHOIS: rc.WHOIS, Name: rc.Host, Source: rc.Source, @@ -344,16 +346,16 @@ func (clients *clientsContainer) findRuntime(ip netip.Addr, idStr string) (cj *c IDs: []string{idStr}, Disallowed: &disallowed, DisallowedRule: &rule, - WHOISInfo: &RuntimeClientWHOISInfo{}, + WHOIS: &whois.Info{}, } return cj } cj = &clientJSON{ - Name: rc.Host, - IDs: []string{idStr}, - WHOISInfo: rc.WHOISInfo, + Name: rc.Host, + IDs: []string{idStr}, + WHOIS: rc.WHOIS, } disallowed, rule := clients.dnsServer.IsBlockedClient(ip, idStr) diff --git a/internal/home/dns.go b/internal/home/dns.go index 48b332f2..ebbcb16e 100644 --- a/internal/home/dns.go +++ b/internal/home/dns.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "path/filepath" + "time" "github.com/AdguardTeam/AdGuardHome/internal/aghalg" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" @@ -17,6 +18,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/AdGuardHome/internal/querylog" "github.com/AdguardTeam/AdGuardHome/internal/stats" + "github.com/AdguardTeam/AdGuardHome/internal/whois" "github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" @@ -25,7 +27,7 @@ import ( yaml "gopkg.in/yaml.v3" ) -// Default ports. +// Default listening ports. const ( defaultPortDNS = 53 defaultPortHTTP = 80 @@ -169,13 +171,72 @@ func initDNSServer( Context.rdns = NewRDNS(Context.dnsServer, &Context.clients, config.DNS.UsePrivateRDNS) } - if config.Clients.Sources.WHOIS { - Context.whois = initWHOIS(&Context.clients) - } + initWHOIS() return nil } +// initWHOIS initializes the WHOIS. +// +// TODO(s.chzhen): Consider making configurable. +func initWHOIS() { + const ( + // defaultQueueSize is the size of queue of IPs for WHOIS processing. + defaultQueueSize = 255 + + // defaultTimeout is the timeout for WHOIS requests. + defaultTimeout = 5 * time.Second + + // defaultCacheSize is the maximum size of the cache. If it's zero, + // cache size is unlimited. + defaultCacheSize = 10_000 + + // defaultMaxConnReadSize is an upper limit in bytes for reading from + // net.Conn. + defaultMaxConnReadSize = 64 * 1024 + + // defaultMaxRedirects is the maximum redirects count. + defaultMaxRedirects = 5 + + // defaultMaxInfoLen is the maximum length of whois.Info fields. + defaultMaxInfoLen = 250 + + // defaultIPTTL is the Time to Live duration for cached IP addresses. + defaultIPTTL = 1 * time.Hour + ) + + Context.whoisCh = make(chan netip.Addr, defaultQueueSize) + + var w whois.Interface + + if config.Clients.Sources.WHOIS { + w = whois.New(&whois.Config{ + DialContext: customDialContext, + ServerAddr: whois.DefaultServer, + Port: whois.DefaultPort, + Timeout: defaultTimeout, + CacheSize: defaultCacheSize, + MaxConnReadSize: defaultMaxConnReadSize, + MaxRedirects: defaultMaxRedirects, + MaxInfoLen: defaultMaxInfoLen, + CacheTTL: defaultIPTTL, + }) + } else { + w = whois.Empty{} + } + + go func() { + defer log.OnPanic("whois") + + for ip := range Context.whoisCh { + info, changed := w.Process(context.Background(), ip) + if info != nil && changed { + Context.clients.setWHOISInfo(ip, info) + } + } + }() +} + // parseSubnetSet parses a slice of subnets. If the slice is empty, it returns // a subnet set that matches all locally served networks, see // [netutil.IsLocallyServed]. @@ -218,9 +279,7 @@ func onDNSRequest(pctx *proxy.DNSContext) { Context.rdns.Begin(ip) } - if srcs.WHOIS && !netutil.IsSpecialPurposeAddr(ip) { - Context.whois.Begin(ip) - } + Context.whoisCh <- ip } func ipsToTCPAddrs(ips []netip.Addr, port int) (tcpAddrs []*net.TCPAddr) { @@ -463,9 +522,7 @@ func startDNSServer() error { Context.rdns.Begin(ip) } - if srcs.WHOIS && !netutil.IsSpecialPurposeAddr(ip) { - Context.whois.Begin(ip) - } + Context.whoisCh <- ip } return nil diff --git a/internal/home/home.go b/internal/home/home.go index 5f1dd6f2..b00b4721 100644 --- a/internal/home/home.go +++ b/internal/home/home.go @@ -57,7 +57,6 @@ type homeContext struct { queryLog querylog.QueryLog // query log module dnsServer *dnsforward.Server // DNS module rdns *RDNS // rDNS module - whois *WHOIS // WHOIS module dhcpServer dhcpd.Interface // DHCP module auth *Auth // HTTP authentication module filters *filtering.DNSFilter // DNS filtering module @@ -84,6 +83,9 @@ type homeContext struct { client *http.Client appSignalChannel chan os.Signal // Channel for receiving OS signals by the console app + // whoisCh is the channel for receiving IPs for WHOIS processing. + whoisCh chan netip.Addr + // tlsCipherIDs are the ID of the cipher suites that AdGuard Home must use. tlsCipherIDs []uint16 diff --git a/internal/home/whois.go b/internal/home/whois.go deleted file mode 100644 index 9ffee9e0..00000000 --- a/internal/home/whois.go +++ /dev/null @@ -1,259 +0,0 @@ -package home - -import ( - "context" - "encoding/binary" - "fmt" - "io" - "net" - "net/netip" - "strings" - "time" - - "github.com/AdguardTeam/AdGuardHome/internal/aghio" - "github.com/AdguardTeam/golibs/cache" - "github.com/AdguardTeam/golibs/errors" - "github.com/AdguardTeam/golibs/log" - "github.com/AdguardTeam/golibs/stringutil" -) - -const ( - defaultServer = "whois.arin.net" - defaultPort = "43" - maxValueLength = 250 - whoisTTL = 1 * 60 * 60 // 1 hour -) - -// WHOIS - module context -type WHOIS struct { - clients *clientsContainer - ipChan chan netip.Addr - - // dialContext specifies the dial function for creating unencrypted TCP - // connections. - dialContext func(ctx context.Context, network, addr string) (conn net.Conn, err error) - - // Contains IP addresses of clients - // An active IP address is resolved once again after it expires. - // If IP address couldn't be resolved, it stays here for some time to prevent further attempts to resolve the same IP. - ipAddrs cache.Cache - - // TODO(a.garipov): Rewrite to use time.Duration. Like, seriously, why? - timeoutMsec uint -} - -// initWHOIS creates the WHOIS module context. -func initWHOIS(clients *clientsContainer) *WHOIS { - w := WHOIS{ - timeoutMsec: 5000, - clients: clients, - ipAddrs: cache.New(cache.Config{ - EnableLRU: true, - MaxCount: 10000, - }), - dialContext: customDialContext, - ipChan: make(chan netip.Addr, 255), - } - - go w.workerLoop() - - return &w -} - -// If the value is too large - cut it and append "..." -func trimValue(s string) string { - if len(s) <= maxValueLength { - return s - } - return s[:maxValueLength-3] + "..." -} - -// isWHOISComment returns true if the string is empty or is a WHOIS comment. -func isWHOISComment(s string) (ok bool) { - return len(s) == 0 || s[0] == '#' || s[0] == '%' -} - -// strmap is an alias for convenience. -type strmap = map[string]string - -// whoisParse parses a subset of plain-text data from the WHOIS response into -// a string map. -func whoisParse(data string) (m strmap) { - m = strmap{} - - var orgname string - lines := strings.Split(data, "\n") - for _, l := range lines { - if isWHOISComment(l) { - continue - } - - kv := strings.SplitN(l, ":", 2) - if len(kv) != 2 { - continue - } - - k := strings.ToLower(strings.TrimSpace(kv[0])) - v := strings.TrimSpace(kv[1]) - if v == "" { - continue - } - - switch k { - case "orgname", "org-name": - k = "orgname" - v = trimValue(v) - orgname = v - case "city", "country": - v = trimValue(v) - case "descr", "netname": - k = "orgname" - v = stringutil.Coalesce(orgname, v) - orgname = v - case "whois": - k = "whois" - case "referralserver": - k = "whois" - v = strings.TrimPrefix(v, "whois://") - default: - continue - } - - m[k] = v - } - - return m -} - -// MaxConnReadSize is an upper limit in bytes for reading from net.Conn. -const MaxConnReadSize = 64 * 1024 - -// Send request to a server and receive the response -func (w *WHOIS) query(ctx context.Context, target, serverAddr string) (data string, err error) { - addr, _, _ := net.SplitHostPort(serverAddr) - if addr == "whois.arin.net" { - target = "n + " + target - } - - conn, err := w.dialContext(ctx, "tcp", serverAddr) - if err != nil { - return "", err - } - defer func() { err = errors.WithDeferred(err, conn.Close()) }() - - r, err := aghio.LimitReader(conn, MaxConnReadSize) - if err != nil { - return "", err - } - - _ = conn.SetReadDeadline(time.Now().Add(time.Duration(w.timeoutMsec) * time.Millisecond)) - _, err = conn.Write([]byte(target + "\r\n")) - if err != nil { - return "", err - } - - // This use of ReadAll is now safe, because we limited the conn Reader. - var whoisData []byte - whoisData, err = io.ReadAll(r) - if err != nil { - return "", err - } - - return string(whoisData), nil -} - -// Query WHOIS servers (handle redirects) -func (w *WHOIS) queryAll(ctx context.Context, target string) (string, error) { - server := net.JoinHostPort(defaultServer, defaultPort) - const maxRedirects = 5 - for i := 0; i != maxRedirects; i++ { - resp, err := w.query(ctx, target, server) - if err != nil { - return "", err - } - log.Debug("whois: received response (%d bytes) from %s IP:%s", len(resp), server, target) - - m := whoisParse(resp) - redir, ok := m["whois"] - if !ok { - return resp, nil - } - redir = strings.ToLower(redir) - - _, _, err = net.SplitHostPort(redir) - if err != nil { - server = net.JoinHostPort(redir, defaultPort) - } else { - server = redir - } - - log.Debug("whois: redirected to %s IP:%s", redir, target) - } - return "", fmt.Errorf("whois: redirect loop") -} - -// Request WHOIS information -func (w *WHOIS) process(ctx context.Context, ip netip.Addr) (wi *RuntimeClientWHOISInfo) { - resp, err := w.queryAll(ctx, ip.String()) - if err != nil { - log.Debug("whois: error: %s IP:%s", err, ip) - - return nil - } - - log.Debug("whois: IP:%s response: %d bytes", ip, len(resp)) - - m := whoisParse(resp) - - wi = &RuntimeClientWHOISInfo{ - City: m["city"], - Country: m["country"], - Orgname: m["orgname"], - } - - // Don't return an empty struct so that the frontend doesn't get - // confused. - if *wi == (RuntimeClientWHOISInfo{}) { - return nil - } - - return wi -} - -// Begin - begin requesting WHOIS info -func (w *WHOIS) Begin(ip netip.Addr) { - ipBytes := ip.AsSlice() - now := uint64(time.Now().Unix()) - expire := w.ipAddrs.Get(ipBytes) - if len(expire) != 0 { - exp := binary.BigEndian.Uint64(expire) - if exp > now { - return - } - } - - expire = make([]byte, 8) - binary.BigEndian.PutUint64(expire, now+whoisTTL) - _ = w.ipAddrs.Set(ipBytes, expire) - - log.Debug("whois: adding %s", ip) - - select { - case w.ipChan <- ip: - default: - log.Debug("whois: queue is full") - } -} - -// workerLoop processes the IP addresses it got from the channel and associates -// the retrieving WHOIS info with a client. -func (w *WHOIS) workerLoop() { - for ip := range w.ipChan { - info := w.process(context.Background(), ip) - if info == nil { - continue - } - - w.clients.setWHOISInfo(ip, info) - } -} diff --git a/internal/home/whois_test.go b/internal/home/whois_test.go deleted file mode 100644 index f05bd670..00000000 --- a/internal/home/whois_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package home - -import ( - "context" - "io" - "net" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// fakeConn is a mock implementation of net.Conn to simplify testing. -// -// TODO(e.burkov): Search for other places in code where it may be used. Move -// into aghtest then. -type fakeConn struct { - // Conn is embedded here simply to make *fakeConn a net.Conn without - // actually implementing all methods. - net.Conn - data []byte -} - -// Write implements net.Conn interface for *fakeConn. It always returns 0 and a -// nil error without mutating the slice. -func (c *fakeConn) Write(_ []byte) (n int, err error) { - return 0, nil -} - -// Read implements net.Conn interface for *fakeConn. It puts the content of -// c.data field into b up to the b's capacity. -func (c *fakeConn) Read(b []byte) (n int, err error) { - return copy(b, c.data), io.EOF -} - -// Close implements net.Conn interface for *fakeConn. It always returns nil. -func (c *fakeConn) Close() (err error) { - return nil -} - -// SetReadDeadline implements net.Conn interface for *fakeConn. It always -// returns nil. -func (c *fakeConn) SetReadDeadline(_ time.Time) (err error) { - return nil -} - -// fakeDial is a mock implementation of customDialContext to simplify testing. -func (c *fakeConn) fakeDial(ctx context.Context, network, addr string) (conn net.Conn, err error) { - return c, nil -} - -func TestWHOIS(t *testing.T) { - const ( - nl = "\n" - data = `OrgName: FakeOrg LLC` + nl + - `City: Nonreal` + nl + - `Country: Imagiland` + nl - ) - - fc := &fakeConn{ - data: []byte(data), - } - - w := WHOIS{ - timeoutMsec: 5000, - dialContext: fc.fakeDial, - } - resp, err := w.queryAll(context.Background(), "1.2.3.4") - assert.NoError(t, err) - - m := whoisParse(resp) - require.NotEmpty(t, m) - - assert.Equal(t, "FakeOrg LLC", m["orgname"]) - assert.Equal(t, "Imagiland", m["country"]) - assert.Equal(t, "Nonreal", m["city"]) -} - -func TestWHOISParse(t *testing.T) { - const ( - city = "Nonreal" - country = "Imagiland" - orgname = "FakeOrgLLC" - whois = "whois.example.net" - ) - - testCases := []struct { - want strmap - name string - in string - }{{ - want: strmap{}, - name: "empty", - in: ``, - }, { - want: strmap{}, - name: "comments", - in: "%\n#", - }, { - want: strmap{}, - name: "no_colon", - in: "city", - }, { - want: strmap{}, - name: "no_value", - in: "city:", - }, { - want: strmap{"city": city}, - name: "city", - in: `city: ` + city, - }, { - want: strmap{"country": country}, - name: "country", - in: `country: ` + country, - }, { - want: strmap{"orgname": orgname}, - name: "orgname", - in: `orgname: ` + orgname, - }, { - want: strmap{"orgname": orgname}, - name: "orgname_hyphen", - in: `org-name: ` + orgname, - }, { - want: strmap{"orgname": orgname}, - name: "orgname_descr", - in: `descr: ` + orgname, - }, { - want: strmap{"orgname": orgname}, - name: "orgname_netname", - in: `netname: ` + orgname, - }, { - want: strmap{"whois": whois}, - name: "whois", - in: `whois: ` + whois, - }, { - want: strmap{"whois": whois}, - name: "referralserver", - in: `referralserver: whois://` + whois, - }, { - want: strmap{}, - name: "other", - in: `other: value`, - }} - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - got := whoisParse(tc.in) - assert.Equal(t, tc.want, got) - }) - } -} diff --git a/internal/querylog/client.go b/internal/querylog/client.go index f25440cc..b12313d6 100644 --- a/internal/querylog/client.go +++ b/internal/querylog/client.go @@ -1,23 +1,15 @@ package querylog +import "github.com/AdguardTeam/AdGuardHome/internal/whois" + // Client is the information required by the query log to match against clients // during searches. type Client struct { - WHOIS *ClientWHOIS `json:"whois,omitempty"` - Name string `json:"name"` - DisallowedRule string `json:"disallowed_rule"` - Disallowed bool `json:"disallowed"` - IgnoreQueryLog bool `json:"-"` -} - -// ClientWHOIS is the filtered WHOIS data for the client. -// -// TODO(a.garipov): Merge with home.RuntimeClientWHOISInfo after the -// refactoring is done. -type ClientWHOIS struct { - City string `json:"city,omitempty"` - Country string `json:"country,omitempty"` - Orgname string `json:"orgname,omitempty"` + WHOIS *whois.Info `json:"whois,omitempty"` + Name string `json:"name"` + DisallowedRule string `json:"disallowed_rule"` + Disallowed bool `json:"disallowed"` + IgnoreQueryLog bool `json:"-"` } // clientCacheKey is the key by which a cached client information is found. diff --git a/internal/whois/whois.go b/internal/whois/whois.go new file mode 100644 index 00000000..e55bcaf7 --- /dev/null +++ b/internal/whois/whois.go @@ -0,0 +1,376 @@ +// Package whois provides WHOIS functionality. +package whois + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/netip" + "strconv" + "strings" + "time" + + "github.com/AdguardTeam/AdGuardHome/internal/aghio" + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/netutil" + "github.com/AdguardTeam/golibs/stringutil" + "github.com/bluele/gcache" +) + +const ( + // DefaultServer is the default WHOIS server. + DefaultServer = "whois.arin.net" + + // DefaultPort is the default port for WHOIS requests. + DefaultPort = 43 +) + +// Interface provides WHOIS functionality. +type Interface interface { + // Process makes WHOIS request and returns WHOIS information or nil. + // changed indicates that Info was updated since last request. + Process(ctx context.Context, ip netip.Addr) (info *Info, changed bool) +} + +// Empty is an empty [Interface] implementation which does nothing. +type Empty struct{} + +// type check +var _ Interface = (*Empty)(nil) + +// Process implements the [Interface] interface for Empty. +func (Empty) Process(_ context.Context, _ netip.Addr) (info *Info, changed bool) { + return nil, false +} + +// Config is the configuration structure for Default. +type Config struct { + // DialContext specifies the dial function for creating unencrypted TCP + // connections. + DialContext func(ctx context.Context, network, addr string) (conn net.Conn, err error) + + // ServerAddr is the address of the WHOIS server. + ServerAddr string + + // Timeout is the timeout for WHOIS requests. + Timeout time.Duration + + // CacheTTL is the Time to Live duration for cached IP addresses. + CacheTTL time.Duration + + // MaxConnReadSize is an upper limit in bytes for reading from net.Conn. + MaxConnReadSize int64 + + // MaxRedirects is the maximum redirects count. + MaxRedirects int + + // MaxInfoLen is the maximum length of Info fields returned by Process. + MaxInfoLen int + + // CacheSize is the maximum size of the cache. It must be greater than + // zero. + CacheSize int + + // Port is the port for WHOIS requests. + Port uint16 +} + +// Default is the default WHOIS information processor. +type Default struct { + // cache is the cache containing IP addresses of clients. An active IP + // address is resolved once again after it expires. If IP address couldn't + // be resolved, it stays here for some time to prevent further attempts to + // resolve the same IP. + cache gcache.Cache + + // dialContext connects to a remote server resolving hostname using our own + // DNS server and unecrypted TCP connection. + dialContext func(ctx context.Context, network, addr string) (conn net.Conn, err error) + + // serverAddr is the address of the WHOIS server. + serverAddr string + + // portStr is the port for WHOIS requests. + portStr string + + // timeout is the timeout for WHOIS requests. + timeout time.Duration + + // cacheTTL is the Time to Live duration for cached IP addresses. + cacheTTL time.Duration + + // maxConnReadSize is an upper limit in bytes for reading from net.Conn. + maxConnReadSize int64 + + // maxRedirects is the maximum redirects count. + maxRedirects int + + // maxInfoLen is the maximum length of Info fields returned by Process. + maxInfoLen int +} + +// New returns a new default WHOIS information processor. conf must not be +// nil. +func New(conf *Config) (w *Default) { + return &Default{ + serverAddr: conf.ServerAddr, + dialContext: conf.DialContext, + timeout: conf.Timeout, + cache: gcache.New(conf.CacheSize).LRU().Build(), + maxConnReadSize: conf.MaxConnReadSize, + maxRedirects: conf.MaxRedirects, + portStr: strconv.Itoa(int(conf.Port)), + maxInfoLen: conf.MaxInfoLen, + cacheTTL: conf.CacheTTL, + } +} + +// trimValue trims s and replaces the last 3 characters of the cut with "..." +// to fit into max. max must be greater than 3. +func trimValue(s string, max int) string { + if len(s) <= max { + return s + } + + return s[:max-3] + "..." +} + +// isWHOISComment returns true if the data is empty or is a WHOIS comment. +func isWHOISComment(data []byte) (ok bool) { + return len(data) == 0 || data[0] == '#' || data[0] == '%' +} + +// whoisParse parses a subset of plain-text data from the WHOIS response into a +// string map. It trims values of the returned map to maxLen. +func whoisParse(data []byte, maxLen int) (info map[string]string) { + info = map[string]string{} + + var orgname string + lines := bytes.Split(data, []byte("\n")) + for _, l := range lines { + if isWHOISComment(l) { + continue + } + + before, after, found := bytes.Cut(l, []byte(":")) + if !found { + continue + } + + key := strings.ToLower(string(before)) + val := strings.TrimSpace(string(after)) + if val == "" { + continue + } + + switch key { + case "orgname", "org-name": + key = "orgname" + val = trimValue(val, maxLen) + orgname = val + case "city", "country": + val = trimValue(val, maxLen) + case "descr", "netname": + key = "orgname" + val = stringutil.Coalesce(orgname, val) + orgname = val + case "whois": + key = "whois" + case "referralserver": + key = "whois" + val = strings.TrimPrefix(val, "whois://") + default: + continue + } + + info[key] = val + } + + return info +} + +// query sends request to a server and returns the response or error. +func (w *Default) query(ctx context.Context, target, serverAddr string) (data []byte, err error) { + addr, _, _ := net.SplitHostPort(serverAddr) + if addr == DefaultServer { + // Display type flags for query. + // + // See https://www.arin.net/resources/registry/whois/rws/api/#nicname-whois-queries. + target = "n + " + target + } + + conn, err := w.dialContext(ctx, "tcp", serverAddr) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + defer func() { err = errors.WithDeferred(err, conn.Close()) }() + + r, err := aghio.LimitReader(conn, w.maxConnReadSize) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + _ = conn.SetReadDeadline(time.Now().Add(w.timeout)) + _, err = io.WriteString(conn, target+"\r\n") + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + // This use of ReadAll is now safe, because we limited the conn Reader. + data, err = io.ReadAll(r) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + return data, nil +} + +// queryAll queries WHOIS server and handles redirects. +func (w *Default) queryAll(ctx context.Context, target string) (info map[string]string, err error) { + server := net.JoinHostPort(w.serverAddr, w.portStr) + var data []byte + + for i := 0; i < w.maxRedirects; i++ { + data, err = w.query(ctx, target, server) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + log.Debug("whois: received response (%d bytes) from %q about %q", len(data), server, target) + + info = whoisParse(data, w.maxInfoLen) + redir, ok := info["whois"] + if !ok { + return info, nil + } + + redir = strings.ToLower(redir) + + _, _, err = net.SplitHostPort(redir) + if err != nil { + server = net.JoinHostPort(redir, w.portStr) + } else { + server = redir + } + + log.Debug("whois: redirected to %q about %q", redir, target) + } + + return nil, fmt.Errorf("whois: redirect loop") +} + +// type check +var _ Interface = (*Default)(nil) + +// Process makes WHOIS request and returns WHOIS information or nil. changed +// indicates that Info was updated since last request. +func (w *Default) Process(ctx context.Context, ip netip.Addr) (wi *Info, changed bool) { + if netutil.IsSpecialPurposeAddr(ip) { + return nil, false + } + + wi, expired := w.findInCache(ip) + if wi != nil && !expired { + // Don't return an empty struct so that the frontend doesn't get + // confused. + if (*wi == Info{}) { + return nil, false + } + + return wi, false + } + + var info Info + + defer func() { + item := toCacheItem(info, w.cacheTTL) + err := w.cache.Set(ip, item) + if err != nil { + log.Debug("whois: cache: adding item %q: %s", ip, err) + } + }() + + kv, err := w.queryAll(ctx, ip.String()) + if err != nil { + log.Debug("whois: quering about %q: %s", ip, err) + + return nil, true + } + + info = Info{ + City: kv["city"], + Country: kv["country"], + Orgname: kv["orgname"], + } + + // Don't return an empty struct so that the frontend doesn't get confused. + if (info == Info{}) { + return nil, true + } + + return &info, wi == nil || info != *wi +} + +// findInCache finds Info in the cache. expired indicates that Info is valid. +func (w *Default) findInCache(ip netip.Addr) (wi *Info, expired bool) { + val, err := w.cache.Get(ip) + if err != nil { + if !errors.Is(err, gcache.KeyNotFoundError) { + log.Debug("whois: cache: retrieving info about %q: %s", ip, err) + } + + return nil, false + } + + item, ok := val.(*cacheItem) + if !ok { + log.Debug("whois: cache: %q bad type %T", ip, val) + + return nil, false + } + + return fromCacheItem(item) +} + +// Info is the filtered WHOIS data for a runtime client. +type Info struct { + City string `json:"city,omitempty"` + Country string `json:"country,omitempty"` + Orgname string `json:"orgname,omitempty"` +} + +// cacheItem represents an item that we will store in the cache. +type cacheItem struct { + // expiry is the time when cacheItem will expire. + expiry time.Time + + // info is the WHOIS data for a runtime client. + info *Info +} + +// toCacheItem creates a cached item from a WHOIS info and Time to Live +// duration. +func toCacheItem(info Info, ttl time.Duration) (item *cacheItem) { + return &cacheItem{ + expiry: time.Now().Add(ttl), + info: &info, + } +} + +// fromCacheItem creates a WHOIS info from the cached item. expired indicates +// that WHOIS info is valid. item must not be nil. +func fromCacheItem(item *cacheItem) (info *Info, expired bool) { + if time.Now().After(item.expiry) { + return item.info, true + } + + return item.info, false +} diff --git a/internal/whois/whois_test.go b/internal/whois/whois_test.go new file mode 100644 index 00000000..2fe32255 --- /dev/null +++ b/internal/whois/whois_test.go @@ -0,0 +1,155 @@ +package whois_test + +import ( + "context" + "io" + "net" + "net/netip" + "testing" + "time" + + "github.com/AdguardTeam/AdGuardHome/internal/whois" + "github.com/AdguardTeam/golibs/testutil/fakenet" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDefault_Process(t *testing.T) { + const ( + nl = "\n" + city = "Nonreal" + country = "Imagiland" + orgname = "FakeOrgLLC" + referralserver = "whois.example.net" + ) + + ip := netip.MustParseAddr("1.2.3.4") + + testCases := []struct { + want *whois.Info + name string + data string + }{{ + want: nil, + name: "empty", + data: "", + }, { + want: nil, + name: "comments", + data: "%\n#", + }, { + want: nil, + name: "no_colon", + data: "city", + }, { + want: nil, + name: "no_value", + data: "city:", + }, { + want: &whois.Info{ + City: city, + }, + name: "city", + data: "city: " + city, + }, { + want: &whois.Info{ + Country: country, + }, + name: "country", + data: "country: " + country, + }, { + want: &whois.Info{ + Orgname: orgname, + }, + name: "orgname", + data: "orgname: " + orgname, + }, { + want: &whois.Info{ + Orgname: orgname, + }, + name: "orgname_hyphen", + data: "org-name: " + orgname, + }, { + want: &whois.Info{ + Orgname: orgname, + }, + name: "orgname_descr", + data: "descr: " + orgname, + }, { + want: &whois.Info{ + Orgname: orgname, + }, + name: "orgname_netname", + data: "netname: " + orgname, + }, { + want: &whois.Info{ + City: city, + Country: country, + Orgname: orgname, + }, + name: "full", + data: "OrgName: " + orgname + nl + "City: " + city + nl + "Country: " + country, + }, { + want: nil, + name: "whois", + data: "whois: " + referralserver, + }, { + want: nil, + name: "referralserver", + data: "referralserver: whois://" + referralserver, + }, { + want: nil, + name: "other", + data: "other: value", + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + hit := 0 + + fakeConn := &fakenet.Conn{ + OnRead: func(b []byte) (n int, err error) { + hit++ + + return copy(b, tc.data), io.EOF + }, + OnWrite: func(b []byte) (n int, err error) { + return len(b), nil + }, + OnClose: func() (err error) { + return nil + }, + OnSetReadDeadline: func(t time.Time) (err error) { + return nil + }, + } + + w := whois.New(&whois.Config{ + Timeout: 5 * time.Second, + DialContext: func(_ context.Context, _, addr string) (_ net.Conn, _ error) { + hit = 0 + + return fakeConn, nil + }, + MaxConnReadSize: 1024, + MaxRedirects: 3, + MaxInfoLen: 250, + CacheSize: 100, + CacheTTL: time.Hour, + }) + + got, changed := w.Process(context.Background(), ip) + require.True(t, changed) + + assert.Equal(t, tc.want, got) + assert.Equal(t, 1, hit) + + // From cache. + got, changed = w.Process(context.Background(), ip) + require.False(t, changed) + + assert.Equal(t, tc.want, got) + assert.Equal(t, 1, hit) + }) + } +}