From 48ee2f8a4249e4d03e8eb8032567566d1bef499d Mon Sep 17 00:00:00 2001 From: Ainar Garipov Date: Wed, 26 Jul 2023 13:18:44 +0300 Subject: [PATCH] all: sync with master; upd chlog --- .github/ISSUE_TEMPLATE/bug.yml | 60 ++- .github/ISSUE_TEMPLATE/feature.yml | 6 +- CHANGELOG.md | 41 +- Makefile | 7 + README.md | 3 +- client/src/__locales/cs.json | 2 +- client/src/__locales/da.json | 2 +- client/src/__locales/de.json | 2 +- client/src/__locales/en.json | 2 +- client/src/__locales/es.json | 2 +- client/src/__locales/fi.json | 24 +- client/src/__locales/fr.json | 2 +- client/src/__locales/it.json | 2 +- client/src/__locales/ja.json | 2 +- client/src/__locales/ko.json | 2 +- client/src/__locales/nl.json | 8 +- client/src/__locales/pl.json | 4 +- client/src/__locales/pt-br.json | 2 +- client/src/__locales/pt-pt.json | 2 +- client/src/__locales/ru.json | 6 +- client/src/__locales/si-lk.json | 8 +- client/src/__locales/sk.json | 10 +- client/src/__locales/sl.json | 2 +- client/src/__locales/sr-cs.json | 39 +- client/src/__locales/tr.json | 2 +- client/src/__locales/zh-cn.json | 2 +- client/src/__locales/zh-hk.json | 11 +- client/src/__locales/zh-tw.json | 2 +- .../Clients/ClientsTable/ClientsTable.js | 2 +- client/src/helpers/filters/filters.js | 16 +- client/src/helpers/trackers/trackers.json | 13 +- docker/Dockerfile | 10 +- go.mod | 19 +- go.sum | 36 +- internal/aghio/limitedreader_test.go | 11 +- internal/aghnet/hostscontainer.go | 6 +- .../aghnet/hostscontainer_internal_test.go | 144 +++++ internal/aghnet/hostscontainer_test.go | 162 +----- internal/aghnet/net.go | 5 + internal/aghnet/net_darwin_test.go | 4 +- internal/aghnet/net_internal_test.go | 334 ++++++++++++ internal/aghnet/net_linux.go | 2 +- internal/aghnet/net_test.go | 324 +---------- internal/aghrenameio/renameio.go | 52 ++ internal/aghrenameio/renameio_test.go | 101 ++++ internal/aghrenameio/renameio_unix.go | 48 ++ internal/aghrenameio/renameio_windows.go | 74 +++ internal/aghtest/aghtest.go | 9 + internal/aghtest/interface.go | 134 ++--- internal/aghtest/interface_test.go | 8 + internal/aghtest/resolver.go | 57 -- internal/client/addrproc.go | 294 ++++++++++ internal/client/addrproc_test.go | 259 +++++++++ internal/client/client.go | 5 + internal/client/client_test.go | 25 + internal/dhcpd/db.go | 2 +- internal/dnsforward/config.go | 13 +- internal/dnsforward/dialcontext.go | 57 ++ internal/dnsforward/dnsforward.go | 175 +++--- internal/dnsforward/dnsforward_test.go | 167 +++--- internal/dnsforward/filter.go | 8 +- internal/dnsforward/http.go | 9 +- internal/dnsforward/{dns.go => process.go} | 75 ++- .../{dns_test.go => process_internal_test.go} | 123 ++++- internal/dnsforward/upstreams.go | 10 +- internal/filtering/filter.go | 232 ++++---- internal/filtering/filtering.go | 2 +- internal/filtering/rulelist/parser.go | 55 +- internal/filtering/rulelist/parser_test.go | 57 +- internal/filtering/rulelist/rulelist.go | 8 +- internal/filtering/rulelist/rulelist_test.go | 9 +- .../safesearch/safesearch_internal_test.go | 25 +- .../filtering/safesearch/safesearch_test.go | 13 +- internal/filtering/servicelist.go | 17 +- internal/home/clients.go | 41 +- ...ients_test.go => clients_internal_test.go} | 16 +- internal/home/config.go | 10 +- internal/home/control.go | 10 +- internal/home/controlinstall.go | 2 +- internal/home/controlupdate.go | 26 +- internal/home/dns.go | 183 ++----- internal/home/home.go | 246 +++------ internal/home/httpclient.go | 47 ++ internal/home/pprof.go | 39 -- internal/home/service.go | 24 +- internal/home/upgrade.go | 2 +- internal/home/web.go | 36 +- internal/rdns/rdns.go | 12 +- internal/rdns/rdns_test.go | 26 +- internal/tools/go.mod | 8 +- internal/tools/go.sum | 18 +- internal/whois/whois.go | 15 +- internal/whois/whois_test.go | 14 +- scripts/README.md | 22 +- scripts/make/go-lint.sh | 18 + scripts/translations/download.go | 177 ++++++ scripts/translations/main.go | 506 +++++++----------- scripts/translations/upload.go | 120 +++++ scripts/vetted-filters/main.go | 2 +- 99 files changed, 3202 insertions(+), 1886 deletions(-) create mode 100644 internal/aghnet/hostscontainer_internal_test.go create mode 100644 internal/aghnet/net_internal_test.go create mode 100644 internal/aghrenameio/renameio.go create mode 100644 internal/aghrenameio/renameio_test.go create mode 100644 internal/aghrenameio/renameio_unix.go create mode 100644 internal/aghrenameio/renameio_windows.go delete mode 100644 internal/aghtest/resolver.go create mode 100644 internal/client/addrproc.go create mode 100644 internal/client/addrproc_test.go create mode 100644 internal/client/client.go create mode 100644 internal/client/client_test.go create mode 100644 internal/dnsforward/dialcontext.go rename internal/dnsforward/{dns.go => process.go} (93%) rename internal/dnsforward/{dns_test.go => process_internal_test.go} (87%) rename internal/home/{clients_test.go => clients_internal_test.go} (94%) create mode 100644 internal/home/httpclient.go delete mode 100644 internal/home/pprof.go create mode 100644 scripts/translations/download.go create mode 100644 scripts/translations/upload.go diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 98db03e0..37d61e56 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -32,31 +32,33 @@ - 'attributes': 'description': 'On which Platform does the issue occur?' 'label': 'Platform (OS and CPU architecture)' + # NOTE: Keep the 386 at the bottom for each OS, because a lot of people + # Seem to confuse them with AMD64, which is what they actually need. 'options': - - 'Darwin (aka macOS)/AMD64 (aka x86_64)' - - 'Darwin (aka macOS)/ARM64' - - 'FreeBSD/386' - - 'FreeBSD/AMD64 (aka x86_64)' - - 'FreeBSD/ARM64' - - 'FreeBSD/ARMv5' - - 'FreeBSD/ARMv6' - - 'FreeBSD/ARMv7' - - 'Linux/386' - - 'Linux/AMD64 (aka x86_64)' - - 'Linux/ARM64' - - 'Linux/ARMv5' - - 'Linux/ARMv6' - - 'Linux/ARMv7' - - 'Linux/MIPS LE' - - 'Linux/MIPS' - - 'Linux/MIPS64 LE' - - 'Linux/MIPS64' - - 'Linux/PPC64 LE' - - 'OpenBSD/AMD64 (aka x86_64)' - - 'OpenBSD/ARM64' - - 'Windows/386' - - 'Windows/AMD64 (aka x86_64)' - - 'Windows/ARM64' + - 'Darwin (aka macOS), AMD64 (aka x86_64)' + - 'Darwin (aka macOS), ARM64' + - 'FreeBSD, AMD64 (aka x86_64)' + - 'FreeBSD, ARM64' + - 'FreeBSD, ARMv5' + - 'FreeBSD, ARMv6' + - 'FreeBSD, ARMv7' + - 'FreeBSD, 32-bit Intel (aka 386)' + - 'Linux, AMD64 (aka x86_64)' + - 'Linux, ARM64' + - 'Linux, ARMv5' + - 'Linux, ARMv6' + - 'Linux, ARMv7' + - 'Linux, MIPS LE' + - 'Linux, MIPS' + - 'Linux, MIPS64 LE' + - 'Linux, MIPS64' + - 'Linux, PPC64 LE' + - 'Linux, 32-bit Intel (aka 386)' + - 'OpenBSD, AMD64 (aka x86_64)' + - 'OpenBSD, ARM64' + - 'Windows, AMD64 (aka x86_64)' + - 'Windows, ARM64' + - 'Windows, 32-bit Intel (aka 386)' - 'Custom (please mention in the description)' 'id': 'os' 'type': 'dropdown' @@ -142,8 +144,10 @@ 'type': 'textarea' 'validations': 'required': false -'description': > - Open a bug report. Please do not open bug reports for questions or help - with configuring clients. If you want to ask for help, use the Discussions - section. +# NOTE: GitHub limits the description length to 200 characters. Also, Markdown +# doesn't work here. +'description': | + For help, use the Discussions section instead. Write the title in English + to make it easier for other people to search for duplicates. (Any language + is fine in the body.) 'name': 'Bug' diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index 154a137d..f86ff470 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -48,7 +48,11 @@ 'type': 'textarea' 'validations': 'required': false -'description': 'Suggest a feature or an enhancement for AdGuard Home' +# NOTE: GitHub limits the description length to 200 characters. Also, Markdown +# doesn't work here. +'description': | + Write the title in English to make it easier for other people to search for + duplicates. (Any language is fine in the body.) 'labels': - 'feature request' 'name': 'Feature request or enhancement' diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c3a8803..e04f023d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,11 +14,11 @@ and this project adheres to @@ -29,6 +29,34 @@ NOTE: Add new changes ABOVE THIS COMMENT. +## [v0.107.35] - 2023-07-26 + +See also the [v0.107.35 GitHub milestone][ms-v0.107.35]. + +### Changed + +- Improved reliability filtering-rule list updates on Unix systems. + +### Fixed + +- Occasional client information lookup failures that could lead to the DNS + server getting stuck ([#6006]). +- `bufio.Scanner: token too long` and other errors when trying to add + filtering-rule lists with lines over 1024 bytes long or containing cosmetic + rules ([#6003]). + +### Removed + +- Default exposure of the non-standard ports 784 and 8853 for DNS-over-QUIC in + the `Dockerfile`. + +[#6003]: https://github.com/AdguardTeam/AdGuardHome/issues/6003 +[#6006]: https://github.com/AdguardTeam/AdGuardHome/issues/6006 + +[ms-v0.107.35]: https://github.com/AdguardTeam/AdGuardHome/milestone/70?closed=1 + + + ## [v0.107.34] - 2023-07-12 See also the [v0.107.34 GitHub milestone][ms-v0.107.34]. @@ -2242,11 +2270,12 @@ See also the [v0.104.2 GitHub milestone][ms-v0.104.2]. -[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...HEAD +[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.35...HEAD +[v0.107.35]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...v0.107.35 [v0.107.34]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...v0.107.34 [v0.107.33]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.32...v0.107.33 [v0.107.32]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.31...v0.107.32 diff --git a/Makefile b/Makefile index 5d76419d..770faa3e 100644 --- a/Makefile +++ b/Makefile @@ -127,3 +127,10 @@ openapi-lint: ; cd ./openapi/ && $(YARN) test openapi-show: ; cd ./openapi/ && $(YARN) start txt-lint: ; $(ENV) "$(SHELL)" ./scripts/make/txt-lint.sh + +# TODO(a.garipov): Consider adding to scripts/ and the common project +# structure. +go-upd-tools: + cd ./internal/tools/ &&\ + "$(GO.MACRO)" get -u &&\ + "$(GO.MACRO)" mod tidy diff --git a/README.md b/README.md index 623ef2bf..52aedc32 100644 --- a/README.md +++ b/README.md @@ -416,7 +416,8 @@ There are three options how you can install an unstable version: ### Report issues If you run into any problem or have a suggestion, head to [this page][iss] and -click on the “New issue” button. +click on the “New issue” button. Please follow the instructions in the issue +form carefully and don't forget to start by searching for duplicates. [iss]: https://github.com/AdguardTeam/AdGuardHome/issues diff --git a/client/src/__locales/cs.json b/client/src/__locales/cs.json index 7289175b..94f89d8a 100644 --- a/client/src/__locales/cs.json +++ b/client/src/__locales/cs.json @@ -444,7 +444,7 @@ "client_confirm_delete": "Opravdu chcete odstranit klienta \"{{key}}\"?", "list_confirm_delete": "Opravdu chcete smazat tento seznam?", "auto_clients_title": "Spuštění klienti", - "auto_clients_desc": "Zařízení, která nejsou na seznamu stálých klientů, a mohou nadále používat AdGuard Home", + "auto_clients_desc": "Informace o IP adresách zařízení, která používají nebo mohou používat AdGuard Home. Tyto informace se získávají z několika zdrojů, včetně souborů hosts, reverzního DNS atd.", "access_title": "Nastavení přístupu", "access_desc": "Zde můžete konfigurovat pravidla přístupu pro server DNS AdGuard Home", "access_allowed_title": "Povolení klienti", diff --git a/client/src/__locales/da.json b/client/src/__locales/da.json index 51dc4394..d8168b8e 100644 --- a/client/src/__locales/da.json +++ b/client/src/__locales/da.json @@ -444,7 +444,7 @@ "client_confirm_delete": "Sikker på, at du vil slette klient \"{{key}}\"?", "list_confirm_delete": "Sikker på, at du vil slette denne liste?", "auto_clients_title": "Klienter (runtime)", - "auto_clients_desc": "Enheder, som ikke er på listen over Permanente klienter, kan stadig bruge AdGuard Home", + "auto_clients_desc": "Oplysninger om IP-adresser på enheder, som (måske) bruger AdGuard Home. Disse oplysninger indsamles fra flere kilder, herunder hosts-filer, reverse DNS mv.", "access_title": "Adgangsindstillinger", "access_desc": "Her kan adgangsregler for AdGuard Home DNS-serveren opsættes", "access_allowed_title": "Tilladte klienter", diff --git a/client/src/__locales/de.json b/client/src/__locales/de.json index dc6010a8..7dc23d85 100644 --- a/client/src/__locales/de.json +++ b/client/src/__locales/de.json @@ -444,7 +444,7 @@ "client_confirm_delete": "Möchten Sie den Client „{{key}}“ wirklich löschen?", "list_confirm_delete": "Möchten Sie diese Liste wirklich löschen?", "auto_clients_title": "Laufzeit-Clients", - "auto_clients_desc": "Geräte, die nicht auf der Liste der persistenten Clients stehen und trotzdem AdGuard Home verwenden dürfen", + "auto_clients_desc": "Informationen über IP-Adressen der Geräten, die AdGuard Home nutzen oder nutzen könnten. Diese Informationen werden aus verschiedenen Quellen gesammelt, darunter Hosts-Dateien, Reverse-DNS usw.", "access_title": "Zugriffsrechte", "access_desc": "Hier können Sie die Zugriffsregeln für den DNS-Server von AdGuard Home konfigurieren", "access_allowed_title": "Zugelassene Clients", diff --git a/client/src/__locales/en.json b/client/src/__locales/en.json index 644f466d..6b73220a 100644 --- a/client/src/__locales/en.json +++ b/client/src/__locales/en.json @@ -444,7 +444,7 @@ "client_confirm_delete": "Are you sure you want to delete client \"{{key}}\"?", "list_confirm_delete": "Are you sure you want to delete this list?", "auto_clients_title": "Runtime clients", - "auto_clients_desc": "Devices not on the list of Persistent clients that may still use AdGuard Home", + "auto_clients_desc": "Information about IP addresses of devices that are using or may use AdGuard Home. This information is gathered from several sources, including hosts files, reverse DNS, etc.", "access_title": "Access settings", "access_desc": "Here you can configure access rules for the AdGuard Home DNS server", "access_allowed_title": "Allowed clients", diff --git a/client/src/__locales/es.json b/client/src/__locales/es.json index addaec9b..47502910 100644 --- a/client/src/__locales/es.json +++ b/client/src/__locales/es.json @@ -444,7 +444,7 @@ "client_confirm_delete": "¿Estás seguro de que deseas eliminar el cliente \"{{key}}\"?", "list_confirm_delete": "¿Estás seguro de que deseas eliminar esta lista?", "auto_clients_title": "Clientes activos", - "auto_clients_desc": "Dispositivos que no están en la lista de clientes persistentes que aún pueden utilizar AdGuard Home", + "auto_clients_desc": "Información sobre las direcciones IP de los dispositivos que usan o pueden usar AdGuard Home. Esta información se recopila de varias fuentes, incluidos ficheros de host, DNS inverso, etc.", "access_title": "Configuración de acceso", "access_desc": "Aquí puedes configurar las reglas de acceso para el servidor DNS de AdGuard Home", "access_allowed_title": "Clientes permitidos", diff --git a/client/src/__locales/fi.json b/client/src/__locales/fi.json index 69272312..f56db735 100644 --- a/client/src/__locales/fi.json +++ b/client/src/__locales/fi.json @@ -2,21 +2,21 @@ "client_settings": "Päätelaiteasetukset", "example_upstream_reserved": "ylävirta <0>tietyille verkkotunnuksille;", "example_upstream_comment": "kommentti.", - "upstream_parallel": "Käytä rinnakkaisia pyyntöjä ja nopeuta selvitystä käyttämällä kaikkia ylävirran palvelimia samanaikaisesti.", + "upstream_parallel": "Käytä rinnakkaisia pyyntöjä ja nopeuta selvitystä käyttämällä kaikkia ylävirtapalvelimia samanaikaisesti.", "parallel_requests": "Rinnakkaiset pyynnöt", "load_balancing": "Kuormantasaus", - "load_balancing_desc": "Lähetä pyyntö yhdelle ylävirran palvelimelle kerrallaan. AdGuard Home pyrkii valitsemaan nopeimman palvelimen painotetun satunnaisalgoritminsa avulla.", + "load_balancing_desc": "Lähetä pyyntö yhdelle ylävirtapalvelimelle kerrallaan. AdGuard Home pyrkii valitsemaan nopeimman palvelimen painotetun satunnaisalgoritminsa avulla.", "bootstrap_dns": "Bootstrap DNS-palvelimet", "bootstrap_dns_desc": "Bootstrap DNS-palvelimia käytetään ylävirroiksi määritettyjen DoH/DoT-resolvereiden IP-osoitteiden selvitykseen.", - "local_ptr_title": "Yksityiset käänteiset DNS-palvelimet", - "local_ptr_desc": "DNS-palvelimet, joita AdGuard Home käyttää paikallisille PTR-pyynnöille. Näitä palvelimia käytetään yksityistä IP-osoitetta käyttävien PTR-pyyntöjen osoitteiden, kuten \"192.168.12.34\", selvitykseen käänteisen DNS:n avulla. Jos ei käytössä, AdGuard Home käyttää käyttöjärjestelmän oletusarvoisia DNS-resolvereita, poislukien AdGuard Homen omat osoitteet.", - "local_ptr_default_resolver": "Oletusarvoisesti AdGuard Home käyttää seuraavia käänteisiä DNS-resolvereita: {{ip}}.", - "local_ptr_no_default_resolver": "AdGuard Home ei voinut määrittää tälle järjestelmälle sopivaa yksityistä käänteistä DNS-resolveria.", + "local_ptr_title": "Yksityiset käänteis-DNS-palvelimet", + "local_ptr_desc": "DNS-palvelimet, joita AdGuard Home käyttää paikallisille PTR-pyynnöille. Näitä palvelimia käytetään yksityistä IP-osoitetta käyttävien PTR-pyyntöjen osoitteiden, kuten \"192.168.12.34\", selvitykseen käänteis-DNS:n avulla. Jos ei käytössä, AdGuard Home käyttää käyttöjärjestelmän oletusarvoisia DNS-resolvereita, poislukien AdGuard Homen omat osoitteet.", + "local_ptr_default_resolver": "Oletusarvoisesti AdGuard Home käyttää seuraavia käänteis-DNS-resolvereita: {{ip}}.", + "local_ptr_no_default_resolver": "AdGuard Home ei voinut määrittää tälle järjestelmälle sopivaa yksityistä käänteis-DNS-resolveria.", "local_ptr_placeholder": "Syötä yksi palvelimen osoite per rivi", "resolve_clients_title": "Käytä päätelaitteiden IP-osoitteille käänteistä selvitystä", - "resolve_clients_desc": "Selvitä päätelaitteiden IP-osoitteiden isäntänimet käänteisesti lähettämällä PTR-pyynnöt sopiville resolvereille (yksityiset DNS-palvelimet paikallisille päätelaitteille, lähtevät palvelimet päätelaitteille, joilla on julkiset IP-osoitteet).", - "use_private_ptr_resolvers_title": "Käytä yksityisiä käänteisiä DNS-resolvereita", - "use_private_ptr_resolvers_desc": "Suorita käänteiset DNS-selvitykset paikallisesti tarjotuille osoitteille käyttäen näitä ylävirran palvelimia. Jos ei käytössä, vastaa AdGuard Home kaikkiin sen tyyppisiin PTR-pyyntöihin NXDOMAIN-arvolla, pois lukien DHCP, /etc/hosts, yms. -tiedoista tunnistettut päätelaitteet.", + "resolve_clients_desc": "Selvitä päätelaitteiden IP-osoitteiden isäntänimet käänteisesti lähettämällä PTR-pyynnöt sopiville resolvereille (yksityiset DNS-palvelimet paikallisille päätelaitteille, ylävirtapalvelimet päätelaitteille, joilla on julkiset IP-osoitteet).", + "use_private_ptr_resolvers_title": "Käytä yksityisiä käänteis-DNS-resolvereita", + "use_private_ptr_resolvers_desc": "Suorita käänteis-DNS-selvitykset paikallisesti tarjotuille osoitteille käyttäen näitä ylävirtapalvelimia. Jos ei käytössä, vastaa AdGuard Home kaikkiin sen tyyppisiin PTR-pyyntöihin NXDOMAIN-arvolla, pois lukien DHCP, /etc/hosts, yms. -tiedoista tunnistettut päätelaitteet.", "check_dhcp_servers": "Etsi DHCP-palvelimia", "save_config": "Tallenna asetukset", "enabled_dhcp": "DHCP-palvelin otettiin käyttöön", @@ -220,7 +220,7 @@ "example_upstream_tcp_port": "tavallinen DNS (TCP, portti);", "example_upstream_tcp_hostname": "tavallinen DNS (TCP, isäntänimi);", "all_lists_up_to_date_toast": "Kaikki listat ovat ajan tasalla", - "updated_upstream_dns_toast": "Ylävirtojen palvelimet tallennettiin", + "updated_upstream_dns_toast": "Ylävirtapalvelimet tallennettiin", "dns_test_ok_toast": "Määritetyt DNS-palvelimet toimivat oikein", "dns_test_not_ok_toast": "Palvelin \"{{key}}\": Ei voitu käyttää, tarkista oikeinkirjoitus", "dns_test_warning_toast": "Datavuon \"{{key}}\" ei vastaa testipyyntöihin eikä välttämättä toimi kunnolla", @@ -444,7 +444,7 @@ "client_confirm_delete": "Haluatko varmasti poistaa päätelaitteen \"{{key}}\"?", "list_confirm_delete": "Haluatko varmasti poistaa tämän listan?", "auto_clients_title": "Määrittämättömät päätelaitteet", - "auto_clients_desc": "Päätelaitteet, joita ei ole määritetty pysyviksi ja jotka voivat silti käyttää AdGuard Homea.", + "auto_clients_desc": "Päätelaitteet, joita ei ole määritetty pysyviksi ja jotka voivat silti käyttää AdGuard Homea. Näitä tietoja kertään useista lähteistä, mm. hosts-tiedostoista ja kääteis-DNS:llä.", "access_title": "Käytön asetukset", "access_desc": "Tässä voidaan määrittää AdGuard Homen DNS-palvelimen käyttöoikeussääntöjä.", "access_allowed_title": "Sallitut päätelaitteet", @@ -623,7 +623,7 @@ "enter_cache_size": "Syötä välimuistin koko (tavuina)", "enter_cache_ttl_min_override": "Syötä vähimmäis-TTL (sekunteina)", "enter_cache_ttl_max_override": "Syötä enimmäis-TTL (sekunteina)", - "cache_ttl_min_override_desc": "Pidennä ylävirran palvelimelta vastaanotettuja, lyhyitä elinaika-arvoja (sekunteina) tallennettaessa DNS-vastauksia välimuistiin.", + "cache_ttl_min_override_desc": "Pidennä ylävirtapalvelimelta vastaanotettuja, lyhyitä elinaika-arvoja (sekunteina) tallennettaessa DNS-vastauksia välimuistiin.", "cache_ttl_max_override_desc": "Määritä DNS-välimuistin kohteiden enimmäiselinaika (sekunteina).", "ttl_cache_validation": "Välimuistin vähimmäiselinajan on oltava pienempi tai sama kuin enimmäiselinajan", "cache_optimistic": "Optimistinen välimuisti", diff --git a/client/src/__locales/fr.json b/client/src/__locales/fr.json index 867beb17..d69f9762 100644 --- a/client/src/__locales/fr.json +++ b/client/src/__locales/fr.json @@ -444,7 +444,7 @@ "client_confirm_delete": "Voulez-vous vraiment supprimer le client « {{key}} » ?", "list_confirm_delete": "Voulez-vous vraiment supprimer cette liste ?", "auto_clients_title": "Clients d'exécution", - "auto_clients_desc": "Appareils ne figurant pas sur la liste des clients persistants qui peuvent encore utiliser AdGuard Home.", + "auto_clients_desc": "Informations sur les adresses IP des appareils qui utilisent ou pourraient utiliser AdGuard Home. Ces informations sont recueillies à partir de plusieurs sources, notamment les fichiers hosts, le DNS inverse, etc.", "access_title": "Paramètres d'accès", "access_desc": "Ici vous pouvez configurer les règles d'accès au serveur DNS AdGuard Home", "access_allowed_title": "Clients autorisés", diff --git a/client/src/__locales/it.json b/client/src/__locales/it.json index 548edc03..c4ddf3aa 100644 --- a/client/src/__locales/it.json +++ b/client/src/__locales/it.json @@ -444,7 +444,7 @@ "client_confirm_delete": "Sei sicuro di voler eliminare il client \"{{key}}\"?", "list_confirm_delete": "Sei sicuro di voler eliminare questo elenco?", "auto_clients_title": "Client in tempo reale", - "auto_clients_desc": "Dispositivi non presenti nell'elenco dei client Persistenti che possono ancora utilizzare AdGuard Home", + "auto_clients_desc": "Informazioni sugli indirizzi IP dei dispositivi che utilizzano o potrebbero utilizzare AdGuard Home. Queste informazioni vengono raccolte da diverse fonti, inclusi file host, DNS inverso, ecc.", "access_title": "Impostazioni di accesso", "access_desc": "Qui puoi configurare le regole d'accesso per il server DNS di AdGuard Home", "access_allowed_title": "Client permessi", diff --git a/client/src/__locales/ja.json b/client/src/__locales/ja.json index f90970c8..95199d5a 100644 --- a/client/src/__locales/ja.json +++ b/client/src/__locales/ja.json @@ -444,7 +444,7 @@ "client_confirm_delete": "クライアント \"{{key}}\" を削除してもよろしいですか？", "list_confirm_delete": "このリストを削除してもよろしいですか？", "auto_clients_title": "ランタイムクライアント", - "auto_clients_desc": "永続的クライアントのリストに未登録で、AdGuard Homeを使用する場合があるデバイスのリスト。", + "auto_clients_desc": "AdGuard Home を使用している、または使用する可能性のあるデバイスの IP アドレスに関する情報です。この情報は、hosts ファイル、リバース DNS など、複数の情報源から収集されます。", "access_title": "アクセス設定", "access_desc": "こちらでは、AdGuard Home DNSサーバーのアクセスルールを設定できます。", "access_allowed_title": "許可されたクライアント", diff --git a/client/src/__locales/ko.json b/client/src/__locales/ko.json index 75afb44f..6ec7a352 100644 --- a/client/src/__locales/ko.json +++ b/client/src/__locales/ko.json @@ -444,7 +444,7 @@ "client_confirm_delete": "정말 클라이언트 '{{key}}'을(를) 삭제하시겠습니까?", "list_confirm_delete": "정말로 이 목록을 제거하시겠습니까?", "auto_clients_title": "런타임 클라이언트", - "auto_clients_desc": "AdGuard Home을 계속 사용할 수 있는 영구 클라이언트 목록에 없는 디바이스입니다", + "auto_clients_desc": "AdGuard Home을 사용 중이거나 사용할 수 있는 기기의 IP 주소에 대한 정보가 표시됩니다. 이 정보는 호스트 파일, 역방향 DNS 등 여러 소스에서 수집됩니다.", "access_title": "접근 설정", "access_desc": "여기에서 AdGuard Home DNS 서버에 대한 액세스 규칙을 설정할 수 있습니다", "access_allowed_title": "허용된 클라이언트", diff --git a/client/src/__locales/nl.json b/client/src/__locales/nl.json index 6ec569be..6bde6d51 100644 --- a/client/src/__locales/nl.json +++ b/client/src/__locales/nl.json @@ -186,7 +186,7 @@ "cancel_btn": "Annuleren", "enter_name_hint": "Voeg naam toe", "enter_url_or_path_hint": "Voer een URL in of het pad van de lijst", - "check_updates_btn": "Controleer op updates", + "check_updates_btn": "Controleren op updates", "new_blocklist": "Nieuwe blokkeerlijst", "new_allowlist": "Nieuwe toelatingslijst", "edit_blocklist": "Blokkeerlijst beheren", @@ -444,7 +444,7 @@ "client_confirm_delete": "Ben je zeker dat je deze gebruiker \"{{key}}\" wilt verwijderen?", "list_confirm_delete": "Ben je zeker om deze lijst te verwijderen?", "auto_clients_title": "Runtime-clients", - "auto_clients_desc": "Apparaten die niet op de lijst van permanente clients staan die mogelijk nog steeds AdGuard Home gebruiken", + "auto_clients_desc": "Informatie over IP-adressen van apparaten die AdGuard Home gebruiken of kunnen gebruiken. Deze informatie wordt verzameld uit verschillende bronnen, waaronder hosts-bestanden, reverse DNS, enz.", "access_title": "Toegangs instellingen", "access_desc": "Hier kan je toegangsregels voor de AdGuard Home DNS-server instellen", "access_allowed_title": "Toegestane gebruikers", @@ -456,7 +456,7 @@ "access_settings_saved": "Toegangsinstellingen succesvol opgeslagen", "updates_checked": "Een nieuwe versie van AdGuard Home is beschikbaar\n", "updates_version_equal": "AdGuard Home is actueel", - "check_updates_now": "Controleer op updates", + "check_updates_now": "Nu controleren op updates", "version_request_error": "Updatecontrole mislukt. Controleer je internetverbinding.", "dns_privacy": "DNS Privacy", "setup_dns_privacy_1": "<0>DNS-via-TLS: Gebruik <1>{{address}} string.", @@ -573,7 +573,7 @@ "tags_title": "Labels", "tags_desc": "Je kunt labels selecteren die overeenkomen met de client. Labels kunnen worden opgenomen in de filterregels om ze \n nauwkeuriger toe te passen. <0>Meer informatie.", "form_select_tags": "Client tags selecteren", - "check_title": "Controleer de filtering", + "check_title": "De filtering controleren", "check_desc": "Controleren of een hostnaam wordt gefilterd.", "check": "Controleren", "form_enter_host": "Voer een hostnaam in", diff --git a/client/src/__locales/pl.json b/client/src/__locales/pl.json index 5fb189f7..f9e83813 100644 --- a/client/src/__locales/pl.json +++ b/client/src/__locales/pl.json @@ -444,7 +444,7 @@ "client_confirm_delete": "Czy na pewno chcesz usunąć klienta \"{{key}}\"?", "list_confirm_delete": "Czy na pewno chcesz usunąć tę listę?", "auto_clients_title": "Uruchomieni klienci", - "auto_clients_desc": "Urządzenia, których nie ma na liście stałych klientów, które mogą nadal korzystać z AdGuard Home", + "auto_clients_desc": "Informacje o adresach IP urządzeń korzystających lub mogących korzystać z AdGuard Home. Te informacje są gromadzone z wielu źródeł takich jak pliki hosta, odwrotna translacja DNS, itp.", "access_title": "Ustawienia dostępu", "access_desc": "Tutaj możesz skonfigurować reguły dostępu dla serwera DNS AdGuard Home", "access_allowed_title": "Dozwoleni klienci", @@ -470,7 +470,7 @@ "setup_dns_privacy_ios_2": "Aplikacja <0>AdGuard dla iOS obsługuje <1>DNS-over-HTTPS i <1>DNS-over-TLS.", "setup_dns_privacy_other_title": "Inne implementacje", "setup_dns_privacy_other_1": "Sam AdGuard Home może być bezpiecznym klientem DNS na dowolnej platformie.", - "setup_dns_privacy_other_2": "<0>dnsproxy obsługuje wszystkie znane bezpieczne protokoły DNS.\n\n", + "setup_dns_privacy_other_2": "<0>dnsproxy obsługuje wszystkie znane bezpieczne protokoły DNS.", "setup_dns_privacy_other_3": "<0>dnscrypt-proxy obsługuje <1>DNS-over-HTTPS.", "setup_dns_privacy_other_4": "<0>Mozilla Firefox obsługuje <1>DNS-over-HTTPS.", "setup_dns_privacy_other_5": "Znajdziesz więcej implementacji <0>tutaj i <1>tutaj.", diff --git a/client/src/__locales/pt-br.json b/client/src/__locales/pt-br.json index e1b7c45b..21094be5 100644 --- a/client/src/__locales/pt-br.json +++ b/client/src/__locales/pt-br.json @@ -444,7 +444,7 @@ "client_confirm_delete": "Você tem certeza de que deseja excluir o cliente \"{{key}}\"?", "list_confirm_delete": "Você tem certeza de que deseja excluir essa lista?", "auto_clients_title": "Clientes ativos", - "auto_clients_desc": "Dispositivo não está na lista de dispositivos persistentes que podem ser utilizados no AdGuard Home", + "auto_clients_desc": "Informações sobre endereços IP de dispositivos que usam ou podem usar o AdGuard Home. Essas informações são coletadas de várias fontes, incluindo arquivos de hosts, DNS reverso, etc.", "access_title": "Configurações de acessos", "access_desc": "Aqui você pode configurar as regras de acesso para o servidores de DNS do AdGuard Home", "access_allowed_title": "Clientes permitidos", diff --git a/client/src/__locales/pt-pt.json b/client/src/__locales/pt-pt.json index 48eb2845..c6e8953a 100644 --- a/client/src/__locales/pt-pt.json +++ b/client/src/__locales/pt-pt.json @@ -444,7 +444,7 @@ "client_confirm_delete": "Tem a certeza de que deseja excluir o cliente \"{{key}}\"?", "list_confirm_delete": "Você tem certeza de que deseja excluir essa lista?", "auto_clients_title": "Clientes ativos", - "auto_clients_desc": "Dispositivo não está na lista de dispositivos persistentes que podem ser utilizados no AdGuard Home", + "auto_clients_desc": "Informações sobre endereços IP de dispositivos que estão a utilizar ou podem utilizar o AdGuard Home. Estas informações são recolhidas a partir de várias fontes, incluindo ficheiros hosts, DNS reverso etc.", "access_title": "Definições de acesso", "access_desc": "Aqui pode configurar as regras de acesso para o servidores de DNS do AdGuard Home", "access_allowed_title": "Clientes permitidos", diff --git a/client/src/__locales/ru.json b/client/src/__locales/ru.json index 771d1100..053cf448 100644 --- a/client/src/__locales/ru.json +++ b/client/src/__locales/ru.json @@ -135,7 +135,7 @@ "number_of_dns_query_to_safe_search": "Количество запросов DNS для поисковых систем, для которых был применён Безопасный поиск", "average_processing_time": "Среднее время обработки запроса", "average_processing_time_hint": "Среднее время для обработки запроса DNS в миллисекундах", - "block_domain_use_filters_and_hosts": "Блокировать домены с использованием фильтров и файлов хостов", + "block_domain_use_filters_and_hosts": "Блокировать домены с использованием фильтров и файлов hosts", "filters_block_toggle_hint": "Вы можете настроить правила блокировки в «Фильтрах».", "use_adguard_browsing_sec": "Включить Безопасную навигацию AdGuard", "use_adguard_browsing_sec_hint": "AdGuard Home проверит, включён ли домен в веб-службу безопасности браузера. Он будет использовать API, чтобы выполнить проверку: на сервер отправляется только короткий префикс имени домена SHA256.", @@ -296,7 +296,7 @@ "rate_limit_desc": "Ограничение на количество запросов в секунду для каждого клиента (0 — неограниченно).", "blocking_ipv4_desc": "IP-адрес, возвращаемый при блокировке A-запроса", "blocking_ipv6_desc": "IP-адрес, возвращаемый при блокировке AAAA-запроса", - "blocking_mode_default": "Стандартный: Отвечает с нулевым IP-адресом, (0.0.0.0 для A; :: для AAAA) когда заблокировано правилом в стиле Adblock; отвечает с IP-адресом, указанным в правиле, когда заблокировано правилом в стиле /etc/hosts-style", + "blocking_mode_default": "Стандартный: Отвечает с нулевым IP-адресом, (0.0.0.0 для A; :: для AAAA) когда заблокировано правилом в стиле Adblock; отвечает с IP-адресом, указанным в правиле, когда заблокировано правилом в стиле файлов hosts", "blocking_mode_refused": "REFUSED: Отвечает с кодом REFUSED", "blocking_mode_nxdomain": "NXDOMAIN: Отвечает с кодом NXDOMAIN\n", "blocking_mode_null_ip": "Нулевой IP: Отвечает с нулевым IP-адресом (0.0.0.0 для A; :: для AAAA)", @@ -444,7 +444,7 @@ "client_confirm_delete": "Вы уверены, что хотите удалить клиента «{{key}}»?", "list_confirm_delete": "Вы уверены, что хотите удалить этот список?", "auto_clients_title": "Клиенты (runtime)", - "auto_clients_desc": "Несохранённые клиенты, которые могут пользоваться AdGuard Home", + "auto_clients_desc": "Информация об IP-адресах устройств, которые используют или могут использовать AdGuard Home. Эта информация собирается из нескольких источников, включая файлы hosts, обратный DNS и так далее.", "access_title": "Настройки доступа", "access_desc": "Здесь вы можете настроить правила доступа к DNS-серверу AdGuard Home", "access_allowed_title": "Разрешённые клиенты", diff --git a/client/src/__locales/si-lk.json b/client/src/__locales/si-lk.json index 609e2228..9d14463a 100644 --- a/client/src/__locales/si-lk.json +++ b/client/src/__locales/si-lk.json @@ -435,6 +435,7 @@ "updates_checked": "ඇඩ්ගාර්ඩ් හෝම් හි නව අනුවාදයක් තිබේ", "updates_version_equal": "ඇඩ්ගාර්ඩ් හෝම් යාවත්කාලීනයි", "check_updates_now": "දැන් යාවත්කාල පරීක්‍ෂා කරන්න", + "version_request_error": "යාවත්කාලීන පරීක්‍ෂාවට අසමත් විය. ඔබගේ අන්තර්ජාල සම්බන්ධතාවය පරීක්‍ෂා කරන්න.", "dns_privacy": "ව.නා.ප. රහස්‍යතා", "setup_dns_privacy_1": "<0>TLS-මගින්-ව.නා.ප. සඳහා <1>{{address}}.", "setup_dns_privacy_2": "<0>HTTPS-මගින්-ව.නා.ප. සඳහා <1>{{address}}.", @@ -453,7 +454,9 @@ "setup_dns_notice": "ඔබට <1>HTTPS-මගින්-ව.නා.ප. හෝ <1>DNS-මගින්-ව.නා.ප. භාවිතයට ඇඩ්ගාර්ඩ් හෝම් සැකසුම් තුළ <0>සංකේතනය වින්‍යාසගත කළ යුතුය.", "rewrite_added": "\"{{key}}\" සඳහා ව.නා.ප. නැවත ලිවීම සාර්ථකව එකතු කෙරිණි", "rewrite_deleted": "\"{{key}}\" සඳහා ව.නා.ප. නැවත ලිවීම ඉවත් කෙරිණි", - "rewrite_add": "ව.නා.ප. නැවත ලිවීමක් එකතු කරන්න", + "rewrite_updated": "ව.නා.ප. නැවත ලිවීම සාර්ථකව යාවත්කාලීන කෙරිණි", + "rewrite_add": "ව.නා.ප. නැවත ලිවීමක් යොදන්න", + "rewrite_edit": "ව.නා.ප. නැවත ලිවීම සංස්කරණය", "rewrite_not_found": "ව.නා.ප. නැවත ලිවීම් හමු නොවිණි", "rewrite_confirm_delete": "\"{{key}}\" සඳහා ව.නා.ප. නැවත ලිවීම ඉවත් කිරීමට අවශ්‍ය බව ඔබට විශ්වාසද?", "rewrite_desc": "නිශ්චිත වසම් නාමයක් සඳහා අභිරුචි ව.නා.ප. ප්‍රතිචාර පහසුවෙන් වින්‍යාසගත කිරීමට ඉඩ දෙයි.", @@ -611,9 +614,12 @@ "safe_browsing": "ආරක්‍ෂිත පිරික්සුම", "served_from_cache": "{{value}} (නිහිතයෙන් ගැනිණි)", "form_error_password_length": "මුරපදය අවම වශයෙන් අකුරු {{value}} ක් දිගු විය යුතුමයි", + "anonymizer_notification": "<0>සටහන: අ.ජා.කෙ. නිර්නාමිකකරණය සබලයි. ඔබට එය <1>පොදු සැකසුම් හරහා අබල කිරීමට හැකිය .", + "confirm_dns_cache_clear": "ඔබට ව.නා.ප. නිහිතය හිස් කිරීමට වුවමනාද?", "cache_cleared": "ව.නා.ප. නිහිතය හිස් කෙරිණි", "clear_cache": "නිහිතය මකන්න", "make_static": "ස්ථිතික කරන්න", + "theme_auto_desc": "ස්වයං (උපාංගයේ වර්ණ පරිපාටිය මත පදනම්ව)", "theme_dark_desc": "අඳුරු තේමාව", "theme_light_desc": "දීප්ත තේමාව", "disable_for_seconds": "තත්පර {{count}} ක්", diff --git a/client/src/__locales/sk.json b/client/src/__locales/sk.json index 559007cd..740a8e8f 100644 --- a/client/src/__locales/sk.json +++ b/client/src/__locales/sk.json @@ -387,7 +387,7 @@ "encryption_key": "Súkromný kľúč", "encryption_key_input": "Skopírujte a prilepte sem svoj súkromný kľúč vo formáte PEM pre Váš certifikát.", "encryption_enable": "Zapnite šifrovanie (HTTPS, DNS-cez-HTTPS a DNS-cez-TLS)", - "encryption_enable_desc": "Ak je šifrovanie zapnuté, AdGuard Home administrátorské rozhranie bude pracovať cez HTTPS a DNS server bude počúvať požiadavky cez DNS-cez-HTTPS a DNS-cez-TLS.", + "encryption_enable_desc": "Ak je šifrovanie zapnuté, AdGuard Home administrátorské rozhranie bude pracovať cez HTTPS a DNS server bude počúvať dopyty cez DNS-cez-HTTPS a DNS-cez-TLS.", "encryption_chain_valid": "Certifikačný reťazec je platný", "encryption_chain_invalid": "Certifikačný reťazec je neplatný", "encryption_key_valid": "Toto je platný {{type}} súkromný kľúč", @@ -497,7 +497,7 @@ "blocked_services": "Blokované služby", "blocked_services_desc": "Umožňuje rýchlo blokovať populárne stránky a služby.", "blocked_services_saved": "Blokované služby boli úspešne uložené", - "blocked_services_global": "Použite globálne blokované služby", + "blocked_services_global": "Použiť globálne blokované služby", "blocked_service": "Blokované služby", "block_all": "Blokovať všetko", "unblock_all": "Odblokovať všetko", @@ -554,7 +554,7 @@ "whois": "WHOIS", "filtering_rules_learn_more": "<0>Dozvedieť sa viac o tvorbe vlastných zoznamov hostiteľov.", "blocked_by_response": "Blokované pomocou CNAME alebo IP v odpovedi", - "blocked_by_cname_or_ip": "Zablokované na základe CNAME alebo IP", + "blocked_by_cname_or_ip": "Blokované pomocou CNAME alebo IP", "try_again": "Skúste znova", "domain_desc": "Zadajte meno domény alebo zástupný znak, ktorý chcete prepísať.", "example_rewrite_domain": "prepísať odpovede iba pre toto meno domény.", @@ -571,7 +571,7 @@ "autofix_warning_list": "Bude vykonávať tieto úlohy: <0>Deaktivovať systém DNSStubListener <0>Nastaviť adresu servera DNS na 127.0.0.1 <0>Nahradiť cieľový symbolický odkaz /etc/resolv.conf na /run/systemd/resolve/resolv.conf <0>Zastaviť službu DNSStubListener (znova načítať službu systemd-resolved)", "autofix_warning_result": "Výsledkom bude, že všetky DNS dopyty z Vášho systému budú štandardne spracované službou AdGuard Home.", "tags_title": "Tagy", - "tags_desc": "Môžete vybrať značky, ktoré zodpovedajú klientovi. Zahrňte značky do pravidiel filtrovania, aby ste ich použili presnejšie. <0>Viac informácií.", + "tags_desc": "Môžete vybrať značky, ktoré zodpovedajú klientovi. Zahrňte značky do pravidiel filtrácie, aby ste ich použili presnejšie. <0>Viac informácií.", "form_select_tags": "Zvoľte tagy klienta", "check_title": "Skontrolujte filtráciu", "check_desc": "Skontrolujte, či je názov hostiteľa filtrovaný.", @@ -608,7 +608,7 @@ "show_whitelisted_responses": "Obsiahnuté v bielej listine", "show_processed_responses": "Spracované", "blocked_safebrowsing": "Zablokované modulom Bezpečné prehliadanie", - "blocked_adult_websites": "Zablokovaná stránka pre dospelých", + "blocked_adult_websites": "Zablokované Rodičovskou kontrolou", "blocked_threats": "Zablokované hrozby", "allowed": "Povolené", "filtered": "Filtrované", diff --git a/client/src/__locales/sl.json b/client/src/__locales/sl.json index ba821f3e..d1362923 100644 --- a/client/src/__locales/sl.json +++ b/client/src/__locales/sl.json @@ -444,7 +444,7 @@ "client_confirm_delete": "Ali ste prepričani, da želite izbrisati odjemalca \"{{key}}\"?", "list_confirm_delete": "Ali ste prepričani, da želite izbrisati ta seznam?", "auto_clients_title": "Odjemalci izvajanja", - "auto_clients_desc": "Naprave, ki niso na seznamu trajnih odjemalcev, ki morda še vedno uporabljajo AdGuard Home", + "auto_clients_desc": "Informacije o naslovih IP naprav, ki uporabljajo ali bi lahko uporabljale AdGuard Home. Te informacije so zbrane iz več virov, vključno z datotekami gostiteljev, povratnim DNS-jem itd.", "access_title": "Nastavitve dostopa", "access_desc": "Tukaj lahko nastavite pravila dostopa strežnika DNS AdGuard Home", "access_allowed_title": "Dovoljeni odjemalci", diff --git a/client/src/__locales/sr-cs.json b/client/src/__locales/sr-cs.json index 68eee868..f9e737ff 100644 --- a/client/src/__locales/sr-cs.json +++ b/client/src/__locales/sr-cs.json @@ -167,6 +167,7 @@ "enabled_parental_toast": "Uključena roditeljska kontrola", "disabled_safe_search_toast": "Isključena sigurna pretraga", "enabled_save_search_toast": "Uključeno sigurno pretraživanje", + "updated_save_search_toast": "Ažurirane postavke bezbedne pretrage", "enabled_table_header": "Uključeno", "name_table_header": "Ime", "list_url_table_header": "URL do liste", @@ -256,12 +257,12 @@ "query_log_cleared": "Dnevnik unosa je uspešno očišćen", "query_log_updated": "Dnevnik zapisa je uspešno ažuriran", "query_log_clear": "Očisti dnevnike unosa", - "query_log_retention": "Zadržavanje dnevnika unosa", + "query_log_retention": "Rotacija evidencija upita", "query_log_enable": "Uključi dnevnik", "query_log_configuration": "Konfiguracija dnevnika", "query_log_disabled": "Dnevnik unosa je isključen ali se može konfigurisati u <0>postavkama", "query_log_strict_search": "Koristi duple navodnike za striktnu pretragu", - "query_log_retention_confirm": "Jeste li sigurni da želite da promenite zadržavanje dnevnika unosa? Ako smanjite vrednost intervala, neki podaci će biti izgubljeni", + "query_log_retention_confirm": "Želite li zaista da promenite rotaciju evidencije upita? Ako smanjite vrednost intervala, neki podaci će biti izgubljeni", "anonymize_client_ip": "Anonimizuj IP klijenta", "anonymize_client_ip_desc": "Ne čuvaj punu IP adresu klijenta u dnevnicima i statistikama", "dns_config": "Konfiguracija DNS servera", @@ -290,6 +291,8 @@ "rate_limit": "Ograničenje brzine", "edns_enable": "Uključi EDNS Client Subnet", "edns_cs_desc": "Dodajte opciju podmreži EDNS klijenta (ECS) uzvodnim zahtevima i evidentirajte vrednosti koje klijenti šalju u evidenciji upita.", + "edns_use_custom_ip": "Koristi prilagođeni IP za EDNS", + "edns_use_custom_ip_desc": "Dozvoli korišćenje prilagođenog IP-a za EDNS", "rate_limit_desc": "Broj zahteva u sekundi dozvoljen po klijentu. Postavljanje na 0 znači da nema ograničenja.", "blocking_ipv4_desc": "IP adresa koja će biti vraćena za blokirane zahteve", "blocking_ipv6_desc": "IP adresa koja će biti vraćena za blokirane AAAA zahteve", @@ -441,7 +444,7 @@ "client_confirm_delete": "Jeste li sigurni da želite da izbrišete klijenta \"{{key}}\"?", "list_confirm_delete": "Jeste li sigurni da želite da izbrišete ovu listu?", "auto_clients_title": "Klijenti (runtime)", - "auto_clients_desc": "Uređaji koji nisu na listi upornih klijenata koji i dalje mogu da koriste AdGuard Home", + "auto_clients_desc": "Podaci o klijentima koji koriste AdGuard Home, ali nisu sačuvani u konfiguraciji", "access_title": "Postavke pristupa", "access_desc": "Ovde možete konfigurisati pravila pristupa za AdGuard Home DNS server", "access_allowed_title": "Dozvoljeni klijenti", @@ -525,6 +528,10 @@ "statistics_retention_confirm": "Jeste li sigurni da želite da promenite zadržavanje statistike? Ako smanjite vrednost intervala, neki podaci će biti izgubljeni", "statistics_cleared": "Statistika je uspešno očišćena", "statistics_enable": "Uključi statistiku", + "ignore_domains": "Zanemari domene (razdvojene novom linijom)", + "ignore_domains_title": "Zanemareni domeni", + "ignore_domains_desc_stats": "Upiti za ove domene nisu upisani u statistiku", + "ignore_domains_desc_query": "Upiti za ove domene nisu upisani u evidenciju upita", "interval_hours": "{{count}} čas", "interval_hours_plural": "{{count}} časova", "filters_configuration": "Konfiguracija filtera", @@ -645,5 +652,29 @@ "confirm_dns_cache_clear": "Želite li zaista da obrišite DNS keš?", "cache_cleared": "DNS keš je uspešno očišćen", "clear_cache": "Obriši keš memoriju", - "protection_section_label": "Zaštita" + "make_static": "Učini statičnim", + "theme_auto_desc": "Automatski (na osnovu šeme boja uređaja)", + "theme_dark_desc": "Tamna tema", + "theme_light_desc": "Svetla tema", + "disable_for_seconds": "Za {{count}} sekund", + "disable_for_seconds_plural": "Za {{count}} sekundi", + "disable_for_minutes": "Za {{count}} minut", + "disable_for_minutes_plural": "Za {{count}} minuta", + "disable_for_hours": "Za {{count}} sat", + "disable_for_hours_plural": "Za {{count}} sati", + "disable_until_tomorrow": "Do sutra", + "disable_notify_for_seconds": "Isključi zaštitu na {{count}} sekund", + "disable_notify_for_seconds_plural": "Isključi zaštitu na {{count}} sekundi", + "disable_notify_for_minutes": "Isključi zaštitu na {{count}} minut", + "disable_notify_for_minutes_plural": "Isključi zaštitu na {{count}} minuta", + "disable_notify_for_hours": "Isključi zaštitu na {{count}} sat", + "disable_notify_for_hours_plural": "Isključi zaštitu na {{count}} sati", + "disable_notify_until_tomorrow": "Isključi zaštitu do sutra", + "enable_protection_timer": "Zaštita će biti uključena u {{time}}", + "custom_retention_input": "Unesite zadržavanje u časovima", + "custom_rotation_input": "Unesite rotaciju u časovima", + "protection_section_label": "Zaštita", + "log_and_stats_section_label": "Evidencija upita i statistika", + "ignore_query_log": "Zanemari ovog klijenta u evidenciji upita", + "ignore_statistics": "Zanemari ovog klijenta u statističkim podacima" } diff --git a/client/src/__locales/tr.json b/client/src/__locales/tr.json index f87b2f23..3771f894 100644 --- a/client/src/__locales/tr.json +++ b/client/src/__locales/tr.json @@ -444,7 +444,7 @@ "client_confirm_delete": "\"{{key}}\" istemcisini silmek istediğinizden emin misiniz?", "list_confirm_delete": "Bu listeyi silmek istediğinizden emin misiniz?", "auto_clients_title": "Çalışma zamanı istemcileri", - "auto_clients_desc": "Henüz AdGuard Home'u kullanabilecek Kalıcı istemciler listesinde olmayan cihazlar", + "auto_clients_desc": "AdGuard Home'u kullanan veya kullanabilecek cihazların IP adresleri hakkında bilgiler. Bu bilgiler, hosts dosyaları, ters DNS, vb. dahil olmak üzere çeşitli kaynaklardan toplanır.", "access_title": "Erişim ayarları", "access_desc": "AdGuard Home DNS sunucusu için erişim kurallarını buradan yapılandırabilirsiniz", "access_allowed_title": "İzin verilen istemciler", diff --git a/client/src/__locales/zh-cn.json b/client/src/__locales/zh-cn.json index 585fdd9c..baa98d37 100644 --- a/client/src/__locales/zh-cn.json +++ b/client/src/__locales/zh-cn.json @@ -444,7 +444,7 @@ "client_confirm_delete": "您确定要删除客户端 \"{{key}}\"？", "list_confirm_delete": "您确定要删除此列表吗？", "auto_clients_title": "客户端（运行时间）", - "auto_clients_desc": "不在可继续使用 AdGuard Home 的持久客户端列表中的设备。", + "auto_clients_desc": "有关正在使用或可能使用 AdGuard Home 的设备的 IP 地址的信息。此信息是从多个来源收集的，包括 hosts 文件、反向 DNS 等。", "access_title": "访问设置", "access_desc": "您可以在此处配置 AdGuard Home 的 DNS 服务器的访问规则", "access_allowed_title": "允许的客户端", diff --git a/client/src/__locales/zh-hk.json b/client/src/__locales/zh-hk.json index 91d3d9f3..48d45a5c 100644 --- a/client/src/__locales/zh-hk.json +++ b/client/src/__locales/zh-hk.json @@ -164,7 +164,7 @@ "disabled_parental_toast": "已停用家長監護", "enabled_parental_toast": "已啟用家長監護", "disabled_safe_search_toast": "已停用安全搜尋", - "enabled_save_search_toast": "已啟用安全搜尋", + "updated_save_search_toast": "已更新安全搜尋設定", "enabled_table_header": "啟用", "name_table_header": "名稱", "list_url_table_header": "清單 URL 網址", @@ -211,6 +211,10 @@ "example_upstream_doq": "加密 <0>DNS-over-QUIC", "example_upstream_sdns": "您可以使透過 <0>DNS Stamps 來解析 <1>DNSCrypt 或 <2>DNS-over-HTTPS", "example_upstream_tcp": "一般 DNS（透過 TCP）", + "example_upstream_regular_port": "一般 DNS（透過 UDP，連接埠）", + "example_upstream_udp": "一般 DNS（透過 UDP，主機名稱）", + "example_upstream_tcp_port": "一般 DNS（透過 TCP，連接埠）", + "example_upstream_tcp_hostname": "一般 DNS（透過 TCP，主機名稱）", "all_lists_up_to_date_toast": "所有清單已更新至最新", "dns_test_ok_toast": "設定中的 DNS 上游運作正常", "dns_test_not_ok_toast": "DNS 設定中的 \"{{key}}\" 出現錯誤，請確認是否正確輸入", @@ -468,6 +472,7 @@ "rewrite_added": "「{{key}}」的 DNS 覆寫新增成功", "rewrite_deleted": "「{{key}}」的 DNS 覆寫刪除成功", "rewrite_add": "新增 DNS 覆寫", + "rewrite_edit": "編輯 DNS 覆寫", "rewrite_not_found": "找不到 DNS 覆寫", "rewrite_confirm_delete": "您確定要刪除 \"{{key}}\" 的 DNS 覆寫？", "rewrite_desc": "提供簡單的方式對特定網域自訂 DNS 回應。", @@ -501,6 +506,7 @@ "interval_days": "{{count}} 天", "interval_days_plural": "{{count}} 天", "domain": "網域", + "ecs": "EDNS 子網", "punycode": "Punycode", "answer": "回應", "filter_added_successfully": "已成功新增清單", @@ -514,6 +520,8 @@ "statistics_retention_confirm": "您確定要更改統計資料保存時間嗎？如果您縮短期限部分資料可能將會遺失", "statistics_cleared": "已清除統計資料", "statistics_enable": "啟用統計數據", + "ignore_domains": "已忽略網域（每行一個）", + "ignore_domains_title": "已忽略網域", "interval_hours": "{{count}} 小時", "interval_hours_plural": "{{count}} 小時", "filters_configuration": "過濾器設定", @@ -626,6 +634,7 @@ "safe_browsing": "安全瀏覽", "served_from_cache": "{{value}} (由快取回應)", "form_error_password_length": "密碼必須至少 {{value}} 個字元長度", + "make_static": "新增為靜態", "theme_dark_desc": "深色主題", "theme_light_desc": "淺色主題", "disable_for_seconds": "{{count}} 秒", diff --git a/client/src/__locales/zh-tw.json b/client/src/__locales/zh-tw.json index a22be579..e31ed29c 100644 --- a/client/src/__locales/zh-tw.json +++ b/client/src/__locales/zh-tw.json @@ -444,7 +444,7 @@ "client_confirm_delete": "您確定您想要刪除用戶端 \"{{key}}\" 嗎？", "list_confirm_delete": "您確定您想要刪除該清單嗎？", "auto_clients_title": "執行時期用戶端", - "auto_clients_desc": "未於可能仍然使用 AdGuard Home 的持續性用戶端之清單上的裝置", + "auto_clients_desc": "AdGuard Home 使用或可能使用的裝置的 IP 地址資訊。這些資訊來自多個來源，包括主機檔案、反向 DNS 等。", "access_title": "存取設定", "access_desc": "於此您可配置用於 AdGuard Home DNS 伺服器之存取規則", "access_allowed_title": "已允許的用戶端", diff --git a/client/src/components/Settings/Clients/ClientsTable/ClientsTable.js b/client/src/components/Settings/Clients/ClientsTable/ClientsTable.js index 41ed4e3b..dab8994c 100644 --- a/client/src/components/Settings/Clients/ClientsTable/ClientsTable.js +++ b/client/src/components/Settings/Clients/ClientsTable/ClientsTable.js @@ -57,7 +57,7 @@ const ClientsTable = ({ }; const handleSubmit = (values) => { - const config = values; + const config = { ...values }; if (values) { if (values.blocked_services) { diff --git a/client/src/helpers/filters/filters.js b/client/src/helpers/filters/filters.js index a100d0bb..8bc5a230 100644 --- a/client/src/helpers/filters/filters.js +++ b/client/src/helpers/filters/filters.js @@ -64,12 +64,6 @@ export default { "homepage": "https://github.com/MasterKia/PersianBlocker", "source": "https://adguardteam.github.io/HostlistsRegistry/assets/filter_19.txt" }, - "ITA_filtri_dns": { - "name": "ITA: Filtri-DNS", - "categoryId": "regional", - "homepage": "https://filtri-dns.ga/", - "source": "https://adguardteam.github.io/HostlistsRegistry/assets/filter_18.txt" - }, "KOR_list_kr": { "name": "KOR: List-KR DNS", "categoryId": "regional", @@ -166,14 +160,20 @@ export default { "homepage": "https://github.com/DandelionSprout/adfilt", "source": "https://adguardteam.github.io/HostlistsRegistry/assets/filter_12.txt" }, + "dandelion_sprouts_anti_push_notifications": { + "name": "Dandelion Sprout's Anti Push Notifications", + "categoryId": "other", + "homepage": "https://github.com/DandelionSprout/adfilt", + "source": "https://adguardteam.github.io/HostlistsRegistry/assets/filter_39.txt" + }, "dandelion_sprouts_game_console_adblock_list": { "name": "Dandelion Sprout's Game Console Adblock List", "categoryId": "other", "homepage": "https://github.com/DandelionSprout/adfilt", "source": "https://adguardteam.github.io/HostlistsRegistry/assets/filter_6.txt" }, - "hagezi_personal": { - "name": "HaGeZi Personal Black \u0026 White", + "hagezi_multinormal": { + "name": "HaGeZi Multi NORMAL", "categoryId": "general", "homepage": "https://github.com/hagezi/dns-blocklists", "source": "https://adguardteam.github.io/HostlistsRegistry/assets/filter_34.txt" diff --git a/client/src/helpers/trackers/trackers.json b/client/src/helpers/trackers/trackers.json index b7bb4664..2b26213e 100644 --- a/client/src/helpers/trackers/trackers.json +++ b/client/src/helpers/trackers/trackers.json @@ -1,5 +1,5 @@ { - "timeUpdated": "2023-07-01T00:11:37.465Z", + "timeUpdated": "2023-07-15T00:10:47.501Z", "categories": { "0": "audio_video_player", "1": "comments", @@ -14088,10 +14088,11 @@ "companyId": "qihoo_360_technology" }, "qq.com": { - "name": "qq.com", - "categoryId": 8, - "url": "http://www.qq.com/", - "companyId": "qq.com" + "name": "QQ International", + "categoryId": 2, + "url": "https://www.qq.com/", + "companyId": "tencent", + "source": "AdGuard" }, "qrius": { "name": "Qrius", @@ -20508,6 +20509,7 @@ "amazon.com.au": "amazon", "amazon-corp.com": "amazon", "a2z.com": "amazon", + "firetvcaptiveportal.com": "amazon", "amazon-adsystem.com": "amazon_adsystem", "serving-sys.com": "amazon_adsystem", "sizmek.com": "amazon_adsystem", @@ -22997,6 +22999,7 @@ "mrskincash.com": "mrskincash", "e-msedge.net": "msedge", "l-msedge.net": "msedge", + "s-msedge.net": "msedge", "msn.com": "msn", "s-msn.com": "msn", "musculahq.appspot.com": "muscula", diff --git a/docker/Dockerfile b/docker/Dockerfile index 2689a86c..38198aa6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -41,18 +41,12 @@ RUN setcap 'cap_net_bind_service=+eip' /opt/adguardhome/AdGuardHome # 68 : UDP : DHCP (client) # 80 : TCP : HTTP (main) # 443 : TCP, UDP : HTTPS, DNS-over-HTTPS (incl. HTTP/3), DNSCrypt (main) -# 784 : UDP : DNS-over-QUIC (deprecated; use 853) # 853 : TCP, UDP : DNS-over-TLS, DNS-over-QUIC # 3000 : TCP, UDP : HTTP(S) (alt, incl. HTTP/3) # 5443 : TCP, UDP : DNSCrypt (alt) # 6060 : TCP : HTTP (pprof) -# 8853 : UDP : DNS-over-QUIC (deprecated; use 853) -# -# TODO(a.garipov): Remove the old, non-standard 784 and 8853 ports for -# DNS-over-QUIC in a future release. -EXPOSE 53/tcp 53/udp 67/udp 68/udp 80/tcp 443/tcp 443/udp 784/udp\ - 853/tcp 853/udp 3000/tcp 3000/udp 5443/tcp 5443/udp 6060/tcp\ - 8853/udp +EXPOSE 53/tcp 53/udp 67/udp 68/udp 80/tcp 443/tcp 443/udp 853/tcp\ + 853/udp 3000/tcp 3000/udp 5443/tcp 5443/udp 6060/tcp WORKDIR /opt/adguardhome/work diff --git a/go.mod b/go.mod index f10b0887..0e1be23c 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,8 @@ module github.com/AdguardTeam/AdGuardHome go 1.19 require ( - // TODO(a.garipov): Update to a tagged version when it's released. - github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768 - github.com/AdguardTeam/golibs v0.13.3 + github.com/AdguardTeam/dnsproxy v0.52.0 + github.com/AdguardTeam/golibs v0.13.6 github.com/AdguardTeam/urlfilter v0.16.1 github.com/NYTimes/gziphandler v1.1.1 github.com/ameshkov/dnscrypt/v2 v2.2.7 @@ -16,7 +15,7 @@ require ( github.com/go-ping/ping v1.1.0 github.com/google/go-cmp v0.5.9 github.com/google/gopacket v1.1.19 - github.com/google/renameio v1.0.1 + github.com/google/renameio/v2 v2.0.0 github.com/google/uuid v1.3.0 github.com/insomniacslk/dhcp v0.0.0-20230612134759-b20c9ba983df github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 @@ -28,14 +27,14 @@ require ( // own code for that. Perhaps, use gopacket. github.com/mdlayher/raw v0.1.0 github.com/miekg/dns v1.1.55 - github.com/quic-go/quic-go v0.35.1 + github.com/quic-go/quic-go v0.36.1 github.com/stretchr/testify v1.8.4 github.com/ti-mo/netfilter v0.5.0 go.etcd.io/bbolt v1.3.7 - golang.org/x/crypto v0.10.0 - golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df - golang.org/x/net v0.11.0 - golang.org/x/sys v0.9.0 + golang.org/x/crypto v0.11.0 + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 + golang.org/x/net v0.12.0 + golang.org/x/sys v0.10.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 howett.net/plist v1.0.0 @@ -62,6 +61,6 @@ require ( github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63 // indirect golang.org/x/mod v0.11.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/text v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/tools v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index e708812c..adee4015 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,9 @@ -github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768 h1:5Ia6wA+tqAlTyzuaOVGSlHmb0osLWXeJUs3NxCuC4gA= -github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768/go.mod h1:CQhZTkqC8X0ID6glrtyaxgqRRdiYfn1gJulC1cZ5Dn8= +github.com/AdguardTeam/dnsproxy v0.52.0 h1:uZxCXflHSAwtJ7uTYXP6qgWcxaBsH0pJvldpwTqIDJk= +github.com/AdguardTeam/dnsproxy v0.52.0/go.mod h1:Jo2zeRe97Rxt3yikXc+fn0LdLtqCj0Xlyh1PNBj6bpM= github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4= github.com/AdguardTeam/golibs v0.10.4/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw= -github.com/AdguardTeam/golibs v0.13.3 h1:RT3QbzThtaLiFLkIUDS6/hlGEXrh0zYvdf4bd7UWpGo= -github.com/AdguardTeam/golibs v0.13.3/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI= +github.com/AdguardTeam/golibs v0.13.6 h1:z/0Q25pRLdaQxtoxvfSaooz5mdv8wj0R8KREj54q8yQ= +github.com/AdguardTeam/golibs v0.13.6/go.mod h1:hOtcb8dPfKcFjWTPA904hTA4dl1aWvzeebdJpE72IPk= github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU= github.com/AdguardTeam/urlfilter v0.16.1 h1:ZPi0rjqo8cQf2FVdzo6cqumNoHZx2KPXj2yZa1A5BBw= github.com/AdguardTeam/urlfilter v0.16.1/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI= @@ -52,8 +52,8 @@ github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= -github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU= -github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= +github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= +github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -108,8 +108,8 @@ github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc8 github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E= github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= -github.com/quic-go/quic-go v0.35.1 h1:b0kzj6b/cQAf05cT0CkQubHM31wiA+xH3IBkxP62poo= -github.com/quic-go/quic-go v0.35.1/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5yGcwOO1g= +github.com/quic-go/quic-go v0.36.1 h1:WsG73nVtnDy1TiACxFxhQ3TqaW+DipmqzLEtNlAwZyY= +github.com/quic-go/quic-go v0.36.1/go.mod h1:zPetvwDlILVxt15n3hr3Gf/I3mDf7LpLKPhR4Ez0AZQ= github.com/shirou/gopsutil/v3 v3.21.8 h1:nKct+uP0TV8DjjNiHanKf8SAuub+GNsbrOtM9Nl9biA= github.com/shirou/gopsutil/v3 v3.21.8/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -134,10 +134,10 @@ go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -152,8 +152,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= @@ -177,16 +177,16 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= diff --git a/internal/aghio/limitedreader_test.go b/internal/aghio/limitedreader_test.go index b3cef08d..a85fd051 100644 --- a/internal/aghio/limitedreader_test.go +++ b/internal/aghio/limitedreader_test.go @@ -1,10 +1,11 @@ -package aghio +package aghio_test import ( "io" "strings" "testing" + "github.com/AdguardTeam/AdGuardHome/internal/aghio" "github.com/AdguardTeam/golibs/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -31,7 +32,7 @@ func TestLimitReader(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, err := LimitReader(nil, tc.n) + _, err := aghio.LimitReader(nil, tc.n) testutil.AssertErrorMsg(t, tc.wantErrMsg, err) }) } @@ -57,7 +58,7 @@ func TestLimitedReader_Read(t *testing.T) { limit: 3, want: 0, }, { - err: &LimitReachedError{ + err: &aghio.LimitReachedError{ Limit: 0, }, name: "limit_reached", @@ -74,7 +75,7 @@ func TestLimitedReader_Read(t *testing.T) { for _, tc := range testCases { readCloser := io.NopCloser(strings.NewReader(tc.rStr)) - lreader, err := LimitReader(readCloser, tc.limit) + lreader, err := aghio.LimitReader(readCloser, tc.limit) require.NoError(t, err) require.NotNil(t, lreader) @@ -89,7 +90,7 @@ func TestLimitedReader_Read(t *testing.T) { } func TestLimitedReader_LimitReachedError(t *testing.T) { - testutil.AssertErrorMsg(t, "attempted to read more than 0 bytes", &LimitReachedError{ + testutil.AssertErrorMsg(t, "attempted to read more than 0 bytes", &aghio.LimitReachedError{ Limit: 0, }) } diff --git a/internal/aghnet/hostscontainer.go b/internal/aghnet/hostscontainer.go index 2fecbc6f..f2e57c46 100644 --- a/internal/aghnet/hostscontainer.go +++ b/internal/aghnet/hostscontainer.go @@ -141,9 +141,9 @@ type HostsRecord struct { Canonical string } -// equal returns true if all fields of rec are equal to field in other or they +// Equal returns true if all fields of rec are equal to field in other or they // both are nil. -func (rec *HostsRecord) equal(other *HostsRecord) (ok bool) { +func (rec *HostsRecord) Equal(other *HostsRecord) (ok bool) { if rec == nil { return other == nil } else if other == nil { @@ -495,7 +495,7 @@ func (hc *HostsContainer) refresh() (err error) { } // hc.last is nil on the first refresh, so let that one through. - if hc.last != nil && maps.EqualFunc(hp.table, hc.last, (*HostsRecord).equal) { + if hc.last != nil && maps.EqualFunc(hp.table, hc.last, (*HostsRecord).Equal) { log.Debug("%s: no changes detected", hostsContainerPrefix) return nil diff --git a/internal/aghnet/hostscontainer_internal_test.go b/internal/aghnet/hostscontainer_internal_test.go new file mode 100644 index 00000000..e3855f39 --- /dev/null +++ b/internal/aghnet/hostscontainer_internal_test.go @@ -0,0 +1,144 @@ +package aghnet + +import ( + "io/fs" + "net/netip" + "path" + "testing" + "testing/fstest" + + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/netutil" + "github.com/AdguardTeam/golibs/testutil/fakefs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const nl = "\n" + +func TestHostsContainer_PathsToPatterns(t *testing.T) { + gsfs := fstest.MapFS{ + "dir_0/file_1": &fstest.MapFile{Data: []byte{1}}, + "dir_0/file_2": &fstest.MapFile{Data: []byte{2}}, + "dir_0/dir_1/file_3": &fstest.MapFile{Data: []byte{3}}, + } + + testCases := []struct { + name string + paths []string + want []string + }{{ + name: "no_paths", + paths: nil, + want: nil, + }, { + name: "single_file", + paths: []string{"dir_0/file_1"}, + want: []string{"dir_0/file_1"}, + }, { + name: "several_files", + paths: []string{"dir_0/file_1", "dir_0/file_2"}, + want: []string{"dir_0/file_1", "dir_0/file_2"}, + }, { + name: "whole_dir", + paths: []string{"dir_0"}, + want: []string{"dir_0/*"}, + }, { + name: "file_and_dir", + paths: []string{"dir_0/file_1", "dir_0/dir_1"}, + want: []string{"dir_0/file_1", "dir_0/dir_1/*"}, + }, { + name: "non-existing", + paths: []string{path.Join("dir_0", "file_3")}, + want: nil, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + patterns, err := pathsToPatterns(gsfs, tc.paths) + require.NoError(t, err) + + assert.Equal(t, tc.want, patterns) + }) + } + + t.Run("bad_file", func(t *testing.T) { + const errStat errors.Error = "bad file" + + badFS := &fakefs.StatFS{ + OnOpen: func(_ string) (f fs.File, err error) { panic("not implemented") }, + OnStat: func(name string) (fi fs.FileInfo, err error) { + return nil, errStat + }, + } + + _, err := pathsToPatterns(badFS, []string{""}) + assert.ErrorIs(t, err, errStat) + }) +} + +func TestUniqueRules_ParseLine(t *testing.T) { + ip := netutil.IPv4Localhost() + ipStr := ip.String() + + testCases := []struct { + name string + line string + wantIP netip.Addr + wantHosts []string + }{{ + name: "simple", + line: ipStr + ` hostname`, + wantIP: ip, + wantHosts: []string{"hostname"}, + }, { + name: "aliases", + line: ipStr + ` hostname alias`, + wantIP: ip, + wantHosts: []string{"hostname", "alias"}, + }, { + name: "invalid_line", + line: ipStr, + wantIP: netip.Addr{}, + wantHosts: nil, + }, { + name: "invalid_line_hostname", + line: ipStr + ` # hostname`, + wantIP: ip, + wantHosts: nil, + }, { + name: "commented_aliases", + line: ipStr + ` hostname # alias`, + wantIP: ip, + wantHosts: []string{"hostname"}, + }, { + name: "whole_comment", + line: `# ` + ipStr + ` hostname`, + wantIP: netip.Addr{}, + wantHosts: nil, + }, { + name: "partial_comment", + line: ipStr + ` host#name`, + wantIP: ip, + wantHosts: []string{"host"}, + }, { + name: "empty", + line: ``, + wantIP: netip.Addr{}, + wantHosts: nil, + }, { + name: "bad_hosts", + line: ipStr + ` bad..host bad._tld empty.tld. ok.host`, + wantIP: ip, + wantHosts: []string{"ok.host"}, + }} + + for _, tc := range testCases { + hp := hostsParser{} + t.Run(tc.name, func(t *testing.T) { + got, hosts := hp.parseLine(tc.line) + assert.Equal(t, tc.wantIP, got) + assert.Equal(t, tc.wantHosts, hosts) + }) + } +} diff --git a/internal/aghnet/hostscontainer_test.go b/internal/aghnet/hostscontainer_test.go index d145a7b4..00c2aeed 100644 --- a/internal/aghnet/hostscontainer_test.go +++ b/internal/aghnet/hostscontainer_test.go @@ -1,9 +1,7 @@ -package aghnet +package aghnet_test import ( - "io/fs" "net" - "net/netip" "path" "strings" "sync/atomic" @@ -12,6 +10,7 @@ import ( "time" "github.com/AdguardTeam/AdGuardHome/internal/aghchan" + "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/AdGuardHome/internal/aghtest" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/netutil" @@ -24,10 +23,7 @@ import ( "github.com/stretchr/testify/require" ) -const ( - nl = "\n" - sp = " " -) +const nl = "\n" func TestNewHostsContainer(t *testing.T) { const dirname = "dir" @@ -48,11 +44,11 @@ func TestNewHostsContainer(t *testing.T) { name: "one_file", paths: []string{p}, }, { - wantErr: ErrNoHostsPaths, + wantErr: aghnet.ErrNoHostsPaths, name: "no_files", paths: []string{}, }, { - wantErr: ErrNoHostsPaths, + wantErr: aghnet.ErrNoHostsPaths, name: "non-existent_file", paths: []string{path.Join(dirname, filename+"2")}, }, { @@ -77,7 +73,7 @@ func TestNewHostsContainer(t *testing.T) { return eventsCh } - hc, err := NewHostsContainer(0, testFS, &aghtest.FSWatcher{ + hc, err := aghnet.NewHostsContainer(0, testFS, &aghtest.FSWatcher{ OnEvents: onEvents, OnAdd: onAdd, OnClose: func() (err error) { return nil }, @@ -103,7 +99,7 @@ func TestNewHostsContainer(t *testing.T) { t.Run("nil_fs", func(t *testing.T) { require.Panics(t, func() { - _, _ = NewHostsContainer(0, nil, &aghtest.FSWatcher{ + _, _ = aghnet.NewHostsContainer(0, nil, &aghtest.FSWatcher{ // Those shouldn't panic. OnEvents: func() (e <-chan struct{}) { return nil }, OnAdd: func(name string) (err error) { return nil }, @@ -114,7 +110,7 @@ func TestNewHostsContainer(t *testing.T) { t.Run("nil_watcher", func(t *testing.T) { require.Panics(t, func() { - _, _ = NewHostsContainer(0, testFS, nil, p) + _, _ = aghnet.NewHostsContainer(0, testFS, nil, p) }) }) @@ -127,7 +123,7 @@ func TestNewHostsContainer(t *testing.T) { OnClose: func() (err error) { return nil }, } - hc, err := NewHostsContainer(0, testFS, errWatcher, p) + hc, err := aghnet.NewHostsContainer(0, testFS, errWatcher, p) require.ErrorIs(t, err, errOnAdd) assert.Nil(t, hc) @@ -158,11 +154,11 @@ func TestHostsContainer_refresh(t *testing.T) { OnClose: func() (err error) { return nil }, } - hc, err := NewHostsContainer(0, testFS, w, "dir") + hc, err := aghnet.NewHostsContainer(0, testFS, w, "dir") require.NoError(t, err) testutil.CleanupAndRequireSuccess(t, hc.Close) - checkRefresh := func(t *testing.T, want *HostsRecord) { + checkRefresh := func(t *testing.T, want *aghnet.HostsRecord) { t.Helper() upd, ok := aghchan.MustReceive(hc.Upd(), 1*time.Second) @@ -175,11 +171,11 @@ func TestHostsContainer_refresh(t *testing.T) { require.True(t, ok) require.NotNil(t, rec) - assert.Truef(t, rec.equal(want), "%+v != %+v", rec, want) + assert.Truef(t, rec.Equal(want), "%+v != %+v", rec, want) } t.Run("initial_refresh", func(t *testing.T) { - checkRefresh(t, &HostsRecord{ + checkRefresh(t, &aghnet.HostsRecord{ Aliases: stringutil.NewSet(), Canonical: "hostname", }) @@ -189,7 +185,7 @@ func TestHostsContainer_refresh(t *testing.T) { testFS["dir/file2"] = &fstest.MapFile{Data: []byte(ipStr + ` alias` + nl)} eventsCh <- event{} - checkRefresh(t, &HostsRecord{ + checkRefresh(t, &aghnet.HostsRecord{ Aliases: stringutil.NewSet("alias"), Canonical: "hostname", }) @@ -228,66 +224,6 @@ func TestHostsContainer_refresh(t *testing.T) { }) } -func TestHostsContainer_PathsToPatterns(t *testing.T) { - gsfs := fstest.MapFS{ - "dir_0/file_1": &fstest.MapFile{Data: []byte{1}}, - "dir_0/file_2": &fstest.MapFile{Data: []byte{2}}, - "dir_0/dir_1/file_3": &fstest.MapFile{Data: []byte{3}}, - } - - testCases := []struct { - name string - paths []string - want []string - }{{ - name: "no_paths", - paths: nil, - want: nil, - }, { - name: "single_file", - paths: []string{"dir_0/file_1"}, - want: []string{"dir_0/file_1"}, - }, { - name: "several_files", - paths: []string{"dir_0/file_1", "dir_0/file_2"}, - want: []string{"dir_0/file_1", "dir_0/file_2"}, - }, { - name: "whole_dir", - paths: []string{"dir_0"}, - want: []string{"dir_0/*"}, - }, { - name: "file_and_dir", - paths: []string{"dir_0/file_1", "dir_0/dir_1"}, - want: []string{"dir_0/file_1", "dir_0/dir_1/*"}, - }, { - name: "non-existing", - paths: []string{path.Join("dir_0", "file_3")}, - want: nil, - }} - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - patterns, err := pathsToPatterns(gsfs, tc.paths) - require.NoError(t, err) - - assert.Equal(t, tc.want, patterns) - }) - } - - t.Run("bad_file", func(t *testing.T) { - const errStat errors.Error = "bad file" - - badFS := &aghtest.StatFS{ - OnStat: func(name string) (fs.FileInfo, error) { - return nil, errStat - }, - } - - _, err := pathsToPatterns(badFS, []string{""}) - assert.ErrorIs(t, err, errStat) - }) -} - func TestHostsContainer_Translate(t *testing.T) { stubWatcher := aghtest.FSWatcher{ OnEvents: func() (e <-chan struct{}) { return nil }, @@ -297,7 +233,7 @@ func TestHostsContainer_Translate(t *testing.T) { require.NoError(t, fstest.TestFS(testdata, "etc_hosts")) - hc, err := NewHostsContainer(0, testdata, &stubWatcher, "etc_hosts") + hc, err := aghnet.NewHostsContainer(0, testdata, &stubWatcher, "etc_hosts") require.NoError(t, err) testutil.CleanupAndRequireSuccess(t, hc.Close) @@ -527,7 +463,7 @@ func TestHostsContainer(t *testing.T) { OnClose: func() (err error) { return nil }, } - hc, err := NewHostsContainer(listID, testdata, &stubWatcher, "etc_hosts") + hc, err := aghnet.NewHostsContainer(listID, testdata, &stubWatcher, "etc_hosts") require.NoError(t, err) testutil.CleanupAndRequireSuccess(t, hc.Close) @@ -558,69 +494,3 @@ func TestHostsContainer(t *testing.T) { }) } } - -func TestUniqueRules_ParseLine(t *testing.T) { - ip := netutil.IPv4Localhost() - ipStr := ip.String() - - testCases := []struct { - name string - line string - wantIP netip.Addr - wantHosts []string - }{{ - name: "simple", - line: ipStr + ` hostname`, - wantIP: ip, - wantHosts: []string{"hostname"}, - }, { - name: "aliases", - line: ipStr + ` hostname alias`, - wantIP: ip, - wantHosts: []string{"hostname", "alias"}, - }, { - name: "invalid_line", - line: ipStr, - wantIP: netip.Addr{}, - wantHosts: nil, - }, { - name: "invalid_line_hostname", - line: ipStr + ` # hostname`, - wantIP: ip, - wantHosts: nil, - }, { - name: "commented_aliases", - line: ipStr + ` hostname # alias`, - wantIP: ip, - wantHosts: []string{"hostname"}, - }, { - name: "whole_comment", - line: `# ` + ipStr + ` hostname`, - wantIP: netip.Addr{}, - wantHosts: nil, - }, { - name: "partial_comment", - line: ipStr + ` host#name`, - wantIP: ip, - wantHosts: []string{"host"}, - }, { - name: "empty", - line: ``, - wantIP: netip.Addr{}, - wantHosts: nil, - }, { - name: "bad_hosts", - line: ipStr + ` bad..host bad._tld empty.tld. ok.host`, - wantIP: ip, - wantHosts: []string{"ok.host"}, - }} - - for _, tc := range testCases { - hp := hostsParser{} - t.Run(tc.name, func(t *testing.T) { - got, hosts := hp.parseLine(tc.line) - assert.Equal(t, tc.wantIP, got) - assert.Equal(t, tc.wantHosts, hosts) - }) - } -} diff --git a/internal/aghnet/net.go b/internal/aghnet/net.go index b8c8c05e..919b03d6 100644 --- a/internal/aghnet/net.go +++ b/internal/aghnet/net.go @@ -3,6 +3,7 @@ package aghnet import ( "bytes" + "context" "encoding/json" "fmt" "io" @@ -15,6 +16,10 @@ import ( "github.com/AdguardTeam/golibs/log" ) +// DialContextFunc is the semantic alias for dialing functions, such as +// [http.Transport.DialContext]. +type DialContextFunc = func(ctx context.Context, network, addr string) (conn net.Conn, err error) + // Variables and functions to substitute in tests. var ( // aghosRunCommand is the function to run shell commands. diff --git a/internal/aghnet/net_darwin_test.go b/internal/aghnet/net_darwin_test.go index 905600d5..06e7eeaf 100644 --- a/internal/aghnet/net_darwin_test.go +++ b/internal/aghnet/net_darwin_test.go @@ -5,9 +5,9 @@ import ( "testing" "testing/fstest" - "github.com/AdguardTeam/AdGuardHome/internal/aghtest" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/testutil" + "github.com/AdguardTeam/golibs/testutil/fakefs" "github.com/stretchr/testify/assert" ) @@ -118,7 +118,7 @@ func TestIfaceSetStaticIP(t *testing.T) { Data: []byte(`nameserver 1.1.1.1`), }, } - panicFsys := &aghtest.FS{ + panicFsys := &fakefs.FS{ OnOpen: func(name string) (fs.File, error) { panic("not implemented") }, } diff --git a/internal/aghnet/net_internal_test.go b/internal/aghnet/net_internal_test.go new file mode 100644 index 00000000..9c4cff8c --- /dev/null +++ b/internal/aghnet/net_internal_test.go @@ -0,0 +1,334 @@ +package aghnet + +import ( + "bytes" + "encoding/json" + "fmt" + "io/fs" + "net" + "net/netip" + "os" + "strings" + "testing" + + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/netutil" + "github.com/AdguardTeam/golibs/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testdata is the filesystem containing data for testing the package. +var testdata fs.FS = os.DirFS("./testdata") + +// substRootDirFS replaces the aghos.RootDirFS function used throughout the +// package with fsys for tests ran under t. +func substRootDirFS(t testing.TB, fsys fs.FS) { + t.Helper() + + prev := rootDirFS + t.Cleanup(func() { rootDirFS = prev }) + rootDirFS = fsys +} + +// RunCmdFunc is the signature of aghos.RunCommand function. +type RunCmdFunc func(cmd string, args ...string) (code int, out []byte, err error) + +// substShell replaces the the aghos.RunCommand function used throughout the +// package with rc for tests ran under t. +func substShell(t testing.TB, rc RunCmdFunc) { + t.Helper() + + prev := aghosRunCommand + t.Cleanup(func() { aghosRunCommand = prev }) + aghosRunCommand = rc +} + +// mapShell is a substitution of aghos.RunCommand that maps the command to it's +// execution result. It's only needed to simplify testing. +// +// TODO(e.burkov): Perhaps put all the shell interactions behind an interface. +type mapShell map[string]struct { + err error + out string + code int +} + +// theOnlyCmd returns mapShell that only handles a single command and arguments +// combination from cmd. +func theOnlyCmd(cmd string, code int, out string, err error) (s mapShell) { + return mapShell{cmd: {code: code, out: out, err: err}} +} + +// RunCmd is a RunCmdFunc handled by s. +func (s mapShell) RunCmd(cmd string, args ...string) (code int, out []byte, err error) { + key := strings.Join(append([]string{cmd}, args...), " ") + ret, ok := s[key] + if !ok { + return 0, nil, fmt.Errorf("unexpected shell command %q", key) + } + + return ret.code, []byte(ret.out), ret.err +} + +// ifaceAddrsFunc is the signature of net.InterfaceAddrs function. +type ifaceAddrsFunc func() (ifaces []net.Addr, err error) + +// substNetInterfaceAddrs replaces the the net.InterfaceAddrs function used +// throughout the package with f for tests ran under t. +func substNetInterfaceAddrs(t *testing.T, f ifaceAddrsFunc) { + t.Helper() + + prev := netInterfaceAddrs + t.Cleanup(func() { netInterfaceAddrs = prev }) + netInterfaceAddrs = f +} + +func TestGatewayIP(t *testing.T) { + const ifaceName = "ifaceName" + const cmd = "ip route show dev " + ifaceName + + testCases := []struct { + shell mapShell + want netip.Addr + name string + }{{ + shell: theOnlyCmd(cmd, 0, `default via 1.2.3.4 onlink`, nil), + want: netip.MustParseAddr("1.2.3.4"), + name: "success_v4", + }, { + shell: theOnlyCmd(cmd, 0, `default via ::ffff onlink`, nil), + want: netip.MustParseAddr("::ffff"), + name: "success_v6", + }, { + shell: theOnlyCmd(cmd, 0, `non-default via 1.2.3.4 onlink`, nil), + want: netip.Addr{}, + name: "bad_output", + }, { + shell: theOnlyCmd(cmd, 0, "", errors.Error("can't run command")), + want: netip.Addr{}, + name: "err_runcmd", + }, { + shell: theOnlyCmd(cmd, 1, "", nil), + want: netip.Addr{}, + name: "bad_code", + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + substShell(t, tc.shell.RunCmd) + + assert.Equal(t, tc.want, GatewayIP(ifaceName)) + }) + } +} + +func TestInterfaceByIP(t *testing.T) { + ifaces, err := GetValidNetInterfacesForWeb() + require.NoError(t, err) + require.NotEmpty(t, ifaces) + + for _, iface := range ifaces { + t.Run(iface.Name, func(t *testing.T) { + require.NotEmpty(t, iface.Addresses) + + for _, ip := range iface.Addresses { + ifaceName := InterfaceByIP(ip) + require.Equal(t, iface.Name, ifaceName) + } + }) + } +} + +func TestBroadcastFromIPNet(t *testing.T) { + known4 := netip.MustParseAddr("192.168.0.1") + fullBroadcast4 := netip.MustParseAddr("255.255.255.255") + + known6 := netip.MustParseAddr("102:304:506:708:90a:b0c:d0e:f10") + + testCases := []struct { + pref netip.Prefix + want netip.Addr + name string + }{{ + pref: netip.PrefixFrom(known4, 0), + want: fullBroadcast4, + name: "full", + }, { + pref: netip.PrefixFrom(known4, 20), + want: netip.MustParseAddr("192.168.15.255"), + name: "full", + }, { + pref: netip.PrefixFrom(known6, netutil.IPv6BitLen), + want: known6, + name: "ipv6_no_mask", + }, { + pref: netip.PrefixFrom(known4, netutil.IPv4BitLen), + want: known4, + name: "ipv4_no_mask", + }, { + pref: netip.PrefixFrom(netip.IPv4Unspecified(), 0), + want: fullBroadcast4, + name: "unspecified", + }, { + pref: netip.Prefix{}, + want: netip.Addr{}, + name: "invalid", + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, BroadcastFromPref(tc.pref)) + }) + } +} + +func TestCheckPort(t *testing.T) { + laddr := netip.AddrPortFrom(netutil.IPv4Localhost(), 0) + + t.Run("tcp_bound", func(t *testing.T) { + l, err := net.Listen("tcp", laddr.String()) + require.NoError(t, err) + testutil.CleanupAndRequireSuccess(t, l.Close) + + ipp := testutil.RequireTypeAssert[*net.TCPAddr](t, l.Addr()).AddrPort() + require.Equal(t, laddr.Addr(), ipp.Addr()) + require.NotZero(t, ipp.Port()) + + err = CheckPort("tcp", ipp) + target := &net.OpError{} + require.ErrorAs(t, err, &target) + + assert.Equal(t, "listen", target.Op) + }) + + t.Run("udp_bound", func(t *testing.T) { + conn, err := net.ListenPacket("udp", laddr.String()) + require.NoError(t, err) + testutil.CleanupAndRequireSuccess(t, conn.Close) + + ipp := testutil.RequireTypeAssert[*net.UDPAddr](t, conn.LocalAddr()).AddrPort() + require.Equal(t, laddr.Addr(), ipp.Addr()) + require.NotZero(t, ipp.Port()) + + err = CheckPort("udp", ipp) + target := &net.OpError{} + require.ErrorAs(t, err, &target) + + assert.Equal(t, "listen", target.Op) + }) + + t.Run("bad_network", func(t *testing.T) { + err := CheckPort("bad_network", netip.AddrPortFrom(netip.Addr{}, 0)) + assert.NoError(t, err) + }) + + t.Run("can_bind", func(t *testing.T) { + err := CheckPort("udp", netip.AddrPortFrom(netip.IPv4Unspecified(), 0)) + assert.NoError(t, err) + }) +} + +func TestCollectAllIfacesAddrs(t *testing.T) { + testCases := []struct { + name string + wantErrMsg string + addrs []net.Addr + wantAddrs []string + }{{ + name: "success", + wantErrMsg: ``, + addrs: []net.Addr{&net.IPNet{ + IP: net.IP{1, 2, 3, 4}, + Mask: net.CIDRMask(24, netutil.IPv4BitLen), + }, &net.IPNet{ + IP: net.IP{4, 3, 2, 1}, + Mask: net.CIDRMask(16, netutil.IPv4BitLen), + }}, + wantAddrs: []string{"1.2.3.4", "4.3.2.1"}, + }, { + name: "not_cidr", + wantErrMsg: `parsing cidr: invalid CIDR address: 1.2.3.4`, + addrs: []net.Addr{&net.IPAddr{ + IP: net.IP{1, 2, 3, 4}, + }}, + wantAddrs: nil, + }, { + name: "empty", + wantErrMsg: ``, + addrs: []net.Addr{}, + wantAddrs: nil, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + substNetInterfaceAddrs(t, func() ([]net.Addr, error) { return tc.addrs, nil }) + + addrs, err := CollectAllIfacesAddrs() + testutil.AssertErrorMsg(t, tc.wantErrMsg, err) + + assert.Equal(t, tc.wantAddrs, addrs) + }) + } + + t.Run("internal_error", func(t *testing.T) { + const errAddrs errors.Error = "can't get addresses" + const wantErrMsg string = `getting interfaces addresses: ` + string(errAddrs) + + substNetInterfaceAddrs(t, func() ([]net.Addr, error) { return nil, errAddrs }) + + _, err := CollectAllIfacesAddrs() + testutil.AssertErrorMsg(t, wantErrMsg, err) + }) +} + +func TestIsAddrInUse(t *testing.T) { + t.Run("addr_in_use", func(t *testing.T) { + l, err := net.Listen("tcp", "0.0.0.0:0") + require.NoError(t, err) + testutil.CleanupAndRequireSuccess(t, l.Close) + + _, err = net.Listen(l.Addr().Network(), l.Addr().String()) + assert.True(t, IsAddrInUse(err)) + }) + + t.Run("another", func(t *testing.T) { + const anotherErr errors.Error = "not addr in use" + + assert.False(t, IsAddrInUse(anotherErr)) + }) +} + +func TestNetInterface_MarshalJSON(t *testing.T) { + const want = `{` + + `"hardware_address":"aa:bb:cc:dd:ee:ff",` + + `"flags":"up|multicast",` + + `"ip_addresses":["1.2.3.4","aaaa::1"],` + + `"name":"iface0",` + + `"mtu":1500` + + `}` + "\n" + + ip4, ok := netip.AddrFromSlice([]byte{1, 2, 3, 4}) + require.True(t, ok) + + ip6, ok := netip.AddrFromSlice([]byte{0xAA, 0xAA, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) + require.True(t, ok) + + net4 := netip.PrefixFrom(ip4, 24) + net6 := netip.PrefixFrom(ip6, 8) + + iface := &NetInterface{ + Addresses: []netip.Addr{ip4, ip6}, + Subnets: []netip.Prefix{net4, net6}, + Name: "iface0", + HardwareAddr: net.HardwareAddr{0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF}, + Flags: net.FlagUp | net.FlagMulticast, + MTU: 1500, + } + + b := &bytes.Buffer{} + err := json.NewEncoder(b).Encode(iface) + require.NoError(t, err) + + assert.Equal(t, want, b.String()) +} diff --git a/internal/aghnet/net_linux.go b/internal/aghnet/net_linux.go index fc83d56a..0c0784c6 100644 --- a/internal/aghnet/net_linux.go +++ b/internal/aghnet/net_linux.go @@ -14,7 +14,7 @@ import ( "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/stringutil" - "github.com/google/renameio/maybe" + "github.com/google/renameio/v2/maybe" "golang.org/x/sys/unix" ) diff --git a/internal/aghnet/net_test.go b/internal/aghnet/net_test.go index 6e9e612e..8615eed9 100644 --- a/internal/aghnet/net_test.go +++ b/internal/aghnet/net_test.go @@ -1,21 +1,11 @@ -package aghnet +package aghnet_test import ( - "bytes" - "encoding/json" - "fmt" "io/fs" - "net" - "net/netip" "os" - "strings" "testing" - "github.com/AdguardTeam/golibs/errors" - "github.com/AdguardTeam/golibs/netutil" "github.com/AdguardTeam/golibs/testutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestMain(m *testing.M) { @@ -24,315 +14,3 @@ func TestMain(m *testing.M) { // testdata is the filesystem containing data for testing the package. var testdata fs.FS = os.DirFS("./testdata") - -// substRootDirFS replaces the aghos.RootDirFS function used throughout the -// package with fsys for tests ran under t. -func substRootDirFS(t testing.TB, fsys fs.FS) { - t.Helper() - - prev := rootDirFS - t.Cleanup(func() { rootDirFS = prev }) - rootDirFS = fsys -} - -// RunCmdFunc is the signature of aghos.RunCommand function. -type RunCmdFunc func(cmd string, args ...string) (code int, out []byte, err error) - -// substShell replaces the the aghos.RunCommand function used throughout the -// package with rc for tests ran under t. -func substShell(t testing.TB, rc RunCmdFunc) { - t.Helper() - - prev := aghosRunCommand - t.Cleanup(func() { aghosRunCommand = prev }) - aghosRunCommand = rc -} - -// mapShell is a substitution of aghos.RunCommand that maps the command to it's -// execution result. It's only needed to simplify testing. -// -// TODO(e.burkov): Perhaps put all the shell interactions behind an interface. -type mapShell map[string]struct { - err error - out string - code int -} - -// theOnlyCmd returns mapShell that only handles a single command and arguments -// combination from cmd. -func theOnlyCmd(cmd string, code int, out string, err error) (s mapShell) { - return mapShell{cmd: {code: code, out: out, err: err}} -} - -// RunCmd is a RunCmdFunc handled by s. -func (s mapShell) RunCmd(cmd string, args ...string) (code int, out []byte, err error) { - key := strings.Join(append([]string{cmd}, args...), " ") - ret, ok := s[key] - if !ok { - return 0, nil, fmt.Errorf("unexpected shell command %q", key) - } - - return ret.code, []byte(ret.out), ret.err -} - -// ifaceAddrsFunc is the signature of net.InterfaceAddrs function. -type ifaceAddrsFunc func() (ifaces []net.Addr, err error) - -// substNetInterfaceAddrs replaces the the net.InterfaceAddrs function used -// throughout the package with f for tests ran under t. -func substNetInterfaceAddrs(t *testing.T, f ifaceAddrsFunc) { - t.Helper() - - prev := netInterfaceAddrs - t.Cleanup(func() { netInterfaceAddrs = prev }) - netInterfaceAddrs = f -} - -func TestGatewayIP(t *testing.T) { - const ifaceName = "ifaceName" - const cmd = "ip route show dev " + ifaceName - - testCases := []struct { - shell mapShell - want netip.Addr - name string - }{{ - shell: theOnlyCmd(cmd, 0, `default via 1.2.3.4 onlink`, nil), - want: netip.MustParseAddr("1.2.3.4"), - name: "success_v4", - }, { - shell: theOnlyCmd(cmd, 0, `default via ::ffff onlink`, nil), - want: netip.MustParseAddr("::ffff"), - name: "success_v6", - }, { - shell: theOnlyCmd(cmd, 0, `non-default via 1.2.3.4 onlink`, nil), - want: netip.Addr{}, - name: "bad_output", - }, { - shell: theOnlyCmd(cmd, 0, "", errors.Error("can't run command")), - want: netip.Addr{}, - name: "err_runcmd", - }, { - shell: theOnlyCmd(cmd, 1, "", nil), - want: netip.Addr{}, - name: "bad_code", - }} - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - substShell(t, tc.shell.RunCmd) - - assert.Equal(t, tc.want, GatewayIP(ifaceName)) - }) - } -} - -func TestInterfaceByIP(t *testing.T) { - ifaces, err := GetValidNetInterfacesForWeb() - require.NoError(t, err) - require.NotEmpty(t, ifaces) - - for _, iface := range ifaces { - t.Run(iface.Name, func(t *testing.T) { - require.NotEmpty(t, iface.Addresses) - - for _, ip := range iface.Addresses { - ifaceName := InterfaceByIP(ip) - require.Equal(t, iface.Name, ifaceName) - } - }) - } -} - -func TestBroadcastFromIPNet(t *testing.T) { - known4 := netip.MustParseAddr("192.168.0.1") - fullBroadcast4 := netip.MustParseAddr("255.255.255.255") - - known6 := netip.MustParseAddr("102:304:506:708:90a:b0c:d0e:f10") - - testCases := []struct { - pref netip.Prefix - want netip.Addr - name string - }{{ - pref: netip.PrefixFrom(known4, 0), - want: fullBroadcast4, - name: "full", - }, { - pref: netip.PrefixFrom(known4, 20), - want: netip.MustParseAddr("192.168.15.255"), - name: "full", - }, { - pref: netip.PrefixFrom(known6, netutil.IPv6BitLen), - want: known6, - name: "ipv6_no_mask", - }, { - pref: netip.PrefixFrom(known4, netutil.IPv4BitLen), - want: known4, - name: "ipv4_no_mask", - }, { - pref: netip.PrefixFrom(netip.IPv4Unspecified(), 0), - want: fullBroadcast4, - name: "unspecified", - }, { - pref: netip.Prefix{}, - want: netip.Addr{}, - name: "invalid", - }} - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.want, BroadcastFromPref(tc.pref)) - }) - } -} - -func TestCheckPort(t *testing.T) { - laddr := netip.AddrPortFrom(netutil.IPv4Localhost(), 0) - - t.Run("tcp_bound", func(t *testing.T) { - l, err := net.Listen("tcp", laddr.String()) - require.NoError(t, err) - testutil.CleanupAndRequireSuccess(t, l.Close) - - ipp := testutil.RequireTypeAssert[*net.TCPAddr](t, l.Addr()).AddrPort() - require.Equal(t, laddr.Addr(), ipp.Addr()) - require.NotZero(t, ipp.Port()) - - err = CheckPort("tcp", ipp) - target := &net.OpError{} - require.ErrorAs(t, err, &target) - - assert.Equal(t, "listen", target.Op) - }) - - t.Run("udp_bound", func(t *testing.T) { - conn, err := net.ListenPacket("udp", laddr.String()) - require.NoError(t, err) - testutil.CleanupAndRequireSuccess(t, conn.Close) - - ipp := testutil.RequireTypeAssert[*net.UDPAddr](t, conn.LocalAddr()).AddrPort() - require.Equal(t, laddr.Addr(), ipp.Addr()) - require.NotZero(t, ipp.Port()) - - err = CheckPort("udp", ipp) - target := &net.OpError{} - require.ErrorAs(t, err, &target) - - assert.Equal(t, "listen", target.Op) - }) - - t.Run("bad_network", func(t *testing.T) { - err := CheckPort("bad_network", netip.AddrPortFrom(netip.Addr{}, 0)) - assert.NoError(t, err) - }) - - t.Run("can_bind", func(t *testing.T) { - err := CheckPort("udp", netip.AddrPortFrom(netip.IPv4Unspecified(), 0)) - assert.NoError(t, err) - }) -} - -func TestCollectAllIfacesAddrs(t *testing.T) { - testCases := []struct { - name string - wantErrMsg string - addrs []net.Addr - wantAddrs []string - }{{ - name: "success", - wantErrMsg: ``, - addrs: []net.Addr{&net.IPNet{ - IP: net.IP{1, 2, 3, 4}, - Mask: net.CIDRMask(24, netutil.IPv4BitLen), - }, &net.IPNet{ - IP: net.IP{4, 3, 2, 1}, - Mask: net.CIDRMask(16, netutil.IPv4BitLen), - }}, - wantAddrs: []string{"1.2.3.4", "4.3.2.1"}, - }, { - name: "not_cidr", - wantErrMsg: `parsing cidr: invalid CIDR address: 1.2.3.4`, - addrs: []net.Addr{&net.IPAddr{ - IP: net.IP{1, 2, 3, 4}, - }}, - wantAddrs: nil, - }, { - name: "empty", - wantErrMsg: ``, - addrs: []net.Addr{}, - wantAddrs: nil, - }} - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - substNetInterfaceAddrs(t, func() ([]net.Addr, error) { return tc.addrs, nil }) - - addrs, err := CollectAllIfacesAddrs() - testutil.AssertErrorMsg(t, tc.wantErrMsg, err) - - assert.Equal(t, tc.wantAddrs, addrs) - }) - } - - t.Run("internal_error", func(t *testing.T) { - const errAddrs errors.Error = "can't get addresses" - const wantErrMsg string = `getting interfaces addresses: ` + string(errAddrs) - - substNetInterfaceAddrs(t, func() ([]net.Addr, error) { return nil, errAddrs }) - - _, err := CollectAllIfacesAddrs() - testutil.AssertErrorMsg(t, wantErrMsg, err) - }) -} - -func TestIsAddrInUse(t *testing.T) { - t.Run("addr_in_use", func(t *testing.T) { - l, err := net.Listen("tcp", "0.0.0.0:0") - require.NoError(t, err) - testutil.CleanupAndRequireSuccess(t, l.Close) - - _, err = net.Listen(l.Addr().Network(), l.Addr().String()) - assert.True(t, IsAddrInUse(err)) - }) - - t.Run("another", func(t *testing.T) { - const anotherErr errors.Error = "not addr in use" - - assert.False(t, IsAddrInUse(anotherErr)) - }) -} - -func TestNetInterface_MarshalJSON(t *testing.T) { - const want = `{` + - `"hardware_address":"aa:bb:cc:dd:ee:ff",` + - `"flags":"up|multicast",` + - `"ip_addresses":["1.2.3.4","aaaa::1"],` + - `"name":"iface0",` + - `"mtu":1500` + - `}` + "\n" - - ip4, ok := netip.AddrFromSlice([]byte{1, 2, 3, 4}) - require.True(t, ok) - - ip6, ok := netip.AddrFromSlice([]byte{0xAA, 0xAA, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) - require.True(t, ok) - - net4 := netip.PrefixFrom(ip4, 24) - net6 := netip.PrefixFrom(ip6, 8) - - iface := &NetInterface{ - Addresses: []netip.Addr{ip4, ip6}, - Subnets: []netip.Prefix{net4, net6}, - Name: "iface0", - HardwareAddr: net.HardwareAddr{0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF}, - Flags: net.FlagUp | net.FlagMulticast, - MTU: 1500, - } - - b := &bytes.Buffer{} - err := json.NewEncoder(b).Encode(iface) - require.NoError(t, err) - - assert.Equal(t, want, b.String()) -} diff --git a/internal/aghrenameio/renameio.go b/internal/aghrenameio/renameio.go new file mode 100644 index 00000000..f982ba78 --- /dev/null +++ b/internal/aghrenameio/renameio.go @@ -0,0 +1,52 @@ +// Package aghrenameio is a wrapper around package github.com/google/renameio/v2 +// that provides a similar stream-based API for both Unix and Windows systems. +// While the Windows API is not technically atomic, it still provides a +// consistent stream-based interface, and atomic renames of files do not seem to +// be possible in all cases anyway. +// +// See https://github.com/google/renameio/issues/1. +// +// TODO(a.garipov): Consider moving to golibs/renameioutil once tried and +// tested. +package aghrenameio + +import ( + "io/fs" + + "github.com/AdguardTeam/golibs/errors" +) + +// PendingFile is the interface for pending temporary files. +type PendingFile interface { + // Cleanup closes the file, and removes it without performing the renaming. + // To close and rename the file, use CloseReplace. + Cleanup() (err error) + + // CloseReplace closes the temporary file and replaces the destination file + // with it, possibly atomically. + // + // This method is not safe for concurrent use by multiple goroutines. + CloseReplace() (err error) + + // Write writes len(b) bytes from b to the File. It returns the number of + // bytes written and an error, if any. Write returns a non-nil error when n + // != len(b). + Write(b []byte) (n int, err error) +} + +// NewPendingFile is a wrapper around [renameio.NewPendingFile] on Unix systems +// and [os.CreateTemp] on Windows. +func NewPendingFile(filePath string, mode fs.FileMode) (f PendingFile, err error) { + return newPendingFile(filePath, mode) +} + +// WithDeferredCleanup is a helper that performs the necessary cleanups and +// finalizations of the temporary files based on the returned error. +func WithDeferredCleanup(returned error, file PendingFile) (err error) { + // Make sure that any error returned from here is marked as a deferred one. + if returned != nil { + return errors.WithDeferred(returned, file.Cleanup()) + } + + return errors.WithDeferred(nil, file.CloseReplace()) +} diff --git a/internal/aghrenameio/renameio_test.go b/internal/aghrenameio/renameio_test.go new file mode 100644 index 00000000..2aa75b34 --- /dev/null +++ b/internal/aghrenameio/renameio_test.go @@ -0,0 +1,101 @@ +package aghrenameio_test + +import ( + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/AdguardTeam/AdGuardHome/internal/aghrenameio" + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testPerm is the common permission mode for tests. +const testPerm fs.FileMode = 0o644 + +// Common file data for tests. +var ( + initialData = []byte("initial data\n") + newData = []byte("new data\n") +) + +func TestPendingFile(t *testing.T) { + t.Parallel() + + targetPath := newInitialFile(t) + f, err := aghrenameio.NewPendingFile(targetPath, testPerm) + require.NoError(t, err) + + _, err = f.Write(newData) + require.NoError(t, err) + + err = f.CloseReplace() + require.NoError(t, err) + + gotData, err := os.ReadFile(targetPath) + require.NoError(t, err) + + assert.Equal(t, newData, gotData) +} + +// newInitialFile is a test helper that returns the path to the file containing +// [initialData]. +func newInitialFile(t *testing.T) (targetPath string) { + t.Helper() + + dir := t.TempDir() + targetPath = filepath.Join(dir, "target") + + err := os.WriteFile(targetPath, initialData, 0o644) + require.NoError(t, err) + + return targetPath +} + +func TestWithDeferredCleanup(t *testing.T) { + t.Parallel() + + const testError errors.Error = "test error" + + testCases := []struct { + error error + name string + wantErrMsg string + wantData []byte + }{{ + name: "success", + error: nil, + wantErrMsg: "", + wantData: newData, + }, { + name: "error", + error: testError, + wantErrMsg: testError.Error(), + wantData: initialData, + }} + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + targetPath := newInitialFile(t) + f, err := aghrenameio.NewPendingFile(targetPath, testPerm) + require.NoError(t, err) + + _, err = f.Write(newData) + require.NoError(t, err) + + err = aghrenameio.WithDeferredCleanup(tc.error, f) + testutil.AssertErrorMsg(t, tc.wantErrMsg, err) + + gotData, err := os.ReadFile(targetPath) + require.NoError(t, err) + + assert.Equal(t, tc.wantData, gotData) + }) + } +} diff --git a/internal/aghrenameio/renameio_unix.go b/internal/aghrenameio/renameio_unix.go new file mode 100644 index 00000000..8f16f905 --- /dev/null +++ b/internal/aghrenameio/renameio_unix.go @@ -0,0 +1,48 @@ +//go:build unix + +package aghrenameio + +import ( + "io/fs" + + "github.com/google/renameio/v2" +) + +// pendingFile is a wrapper around [*renameio.PendingFile] making it an +// [io.WriteCloser]. +type pendingFile struct { + file *renameio.PendingFile +} + +// type check +var _ PendingFile = pendingFile{} + +// Cleanup implements the [PendingFile] interface for pendingFile. +func (f pendingFile) Cleanup() (err error) { + return f.file.Cleanup() +} + +// CloseReplace implements the [PendingFile] interface for pendingFile. +func (f pendingFile) CloseReplace() (err error) { + return f.file.CloseAtomicallyReplace() +} + +// Write implements the [PendingFile] interface for pendingFile. +func (f pendingFile) Write(b []byte) (n int, err error) { + return f.file.Write(b) +} + +// NewPendingFile is a wrapper around [renameio.NewPendingFile]. +// +// f.Close must be called to finish the renaming. +func newPendingFile(filePath string, mode fs.FileMode) (f PendingFile, err error) { + file, err := renameio.NewPendingFile(filePath, renameio.WithPermissions(mode)) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + return pendingFile{ + file: file, + }, nil +} diff --git a/internal/aghrenameio/renameio_windows.go b/internal/aghrenameio/renameio_windows.go new file mode 100644 index 00000000..d4f7cddd --- /dev/null +++ b/internal/aghrenameio/renameio_windows.go @@ -0,0 +1,74 @@ +//go:build windows + +package aghrenameio + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/AdguardTeam/golibs/errors" +) + +// pendingFile is a wrapper around [*os.File] calling [os.Rename] in its Close +// method. +type pendingFile struct { + file *os.File + targetPath string +} + +// type check +var _ PendingFile = (*pendingFile)(nil) + +// Cleanup implements the [PendingFile] interface for *pendingFile. +func (f *pendingFile) Cleanup() (err error) { + closeErr := f.file.Close() + err = os.Remove(f.file.Name()) + + // Put closeErr into the deferred error because that's where it is usually + // expected. + return errors.WithDeferred(err, closeErr) +} + +// CloseReplace implements the [PendingFile] interface for *pendingFile. +func (f *pendingFile) CloseReplace() (err error) { + err = f.file.Close() + if err != nil { + return fmt.Errorf("closing: %w", err) + } + + err = os.Rename(f.file.Name(), f.targetPath) + if err != nil { + return fmt.Errorf("renaming: %w", err) + } + + return nil +} + +// Write implements the [PendingFile] interface for *pendingFile. +func (f *pendingFile) Write(b []byte) (n int, err error) { + return f.file.Write(b) +} + +// NewPendingFile is a wrapper around [os.CreateTemp]. +// +// f.Close must be called to finish the renaming. +func newPendingFile(filePath string, mode fs.FileMode) (f PendingFile, err error) { + // Use the same directory as the file itself, because moves across + // filesystems can be especially problematic. + file, err := os.CreateTemp(filepath.Dir(filePath), "") + if err != nil { + return nil, fmt.Errorf("opening pending file: %w", err) + } + + err = file.Chmod(mode) + if err != nil { + return nil, fmt.Errorf("preparing pending file: %w", err) + } + + return &pendingFile{ + file: file, + targetPath: filePath, + }, nil +} diff --git a/internal/aghtest/aghtest.go b/internal/aghtest/aghtest.go index 850446e0..dfe0551d 100644 --- a/internal/aghtest/aghtest.go +++ b/internal/aghtest/aghtest.go @@ -2,7 +2,9 @@ package aghtest import ( + "crypto/sha256" "io" + "net" "testing" "github.com/AdguardTeam/golibs/log" @@ -34,3 +36,10 @@ func ReplaceLogLevel(t testing.TB, l log.Level) { t.Cleanup(func() { log.SetLevel(prev) }) log.SetLevel(l) } + +// HostToIPs is a helper that generates one IPv4 and one IPv6 address from host. +func HostToIPs(host string) (ipv4, ipv6 net.IP) { + hash := sha256.Sum256([]byte(host)) + + return net.IP(hash[:4]), net.IP(hash[4:20]) +} diff --git a/internal/aghtest/interface.go b/internal/aghtest/interface.go index cce49776..10789d8e 100644 --- a/internal/aghtest/interface.go +++ b/internal/aghtest/interface.go @@ -2,11 +2,15 @@ package aghtest import ( "context" - "io" - "io/fs" + "net" + "net/netip" + "time" "github.com/AdguardTeam/AdGuardHome/internal/aghos" + "github.com/AdguardTeam/AdGuardHome/internal/client" "github.com/AdguardTeam/AdGuardHome/internal/next/agh" + "github.com/AdguardTeam/AdGuardHome/internal/rdns" + "github.com/AdguardTeam/AdGuardHome/internal/whois" "github.com/AdguardTeam/dnsproxy/upstream" "github.com/miekg/dns" ) @@ -15,67 +19,6 @@ import ( // // Keep entities in this file in alphabetic order. -// Standard Library - -// Package fs - -// FS is a fake [fs.FS] implementation for tests. -type FS struct { - OnOpen func(name string) (fs.File, error) -} - -// type check -var _ fs.FS = (*FS)(nil) - -// Open implements the [fs.FS] interface for *FS. -func (fsys *FS) Open(name string) (fs.File, error) { - return fsys.OnOpen(name) -} - -// type check -var _ fs.GlobFS = (*GlobFS)(nil) - -// GlobFS is a fake [fs.GlobFS] implementation for tests. -type GlobFS struct { - // FS is embedded here to avoid implementing all it's methods. - FS - OnGlob func(pattern string) ([]string, error) -} - -// Glob implements the [fs.GlobFS] interface for *GlobFS. -func (fsys *GlobFS) Glob(pattern string) ([]string, error) { - return fsys.OnGlob(pattern) -} - -// type check -var _ fs.StatFS = (*StatFS)(nil) - -// StatFS is a fake [fs.StatFS] implementation for tests. -type StatFS struct { - // FS is embedded here to avoid implementing all it's methods. - FS - OnStat func(name string) (fs.FileInfo, error) -} - -// Stat implements the [fs.StatFS] interface for *StatFS. -func (fsys *StatFS) Stat(name string) (fs.FileInfo, error) { - return fsys.OnStat(name) -} - -// Package io - -// Writer is a fake [io.Writer] implementation for tests. -type Writer struct { - OnWrite func(b []byte) (n int, err error) -} - -var _ io.Writer = (*Writer)(nil) - -// Write implements the [io.Writer] interface for *Writer. -func (w *Writer) Write(b []byte) (n int, err error) { - return w.OnWrite(b) -} - // Module adguard-home // Package aghos @@ -135,6 +78,71 @@ func (s *ServiceWithConfig[ConfigType]) Config() (c ConfigType) { return s.OnConfig() } +// Package client + +// AddressProcessor is a fake [client.AddressProcessor] implementation for +// tests. +type AddressProcessor struct { + OnProcess func(ip netip.Addr) + OnClose func() (err error) +} + +// type check +var _ client.AddressProcessor = (*AddressProcessor)(nil) + +// Process implements the [client.AddressProcessor] interface for +// *AddressProcessor. +func (p *AddressProcessor) Process(ip netip.Addr) { + p.OnProcess(ip) +} + +// Close implements the [client.AddressProcessor] interface for +// *AddressProcessor. +func (p *AddressProcessor) Close() (err error) { + return p.OnClose() +} + +// AddressUpdater is a fake [client.AddressUpdater] implementation for tests. +type AddressUpdater struct { + OnUpdateAddress func(ip netip.Addr, host string, info *whois.Info) +} + +// type check +var _ client.AddressUpdater = (*AddressUpdater)(nil) + +// UpdateAddress implements the [client.AddressUpdater] interface for +// *AddressUpdater. +func (p *AddressUpdater) UpdateAddress(ip netip.Addr, host string, info *whois.Info) { + p.OnUpdateAddress(ip, host, info) +} + +// Package filtering + +// Resolver is a fake [filtering.Resolver] implementation for tests. +type Resolver struct { + OnLookupIP func(ctx context.Context, network, host string) (ips []net.IP, err error) +} + +// LookupIP implements the [filtering.Resolver] interface for *Resolver. +func (r *Resolver) LookupIP(ctx context.Context, network, host string) (ips []net.IP, err error) { + return r.OnLookupIP(ctx, network, host) +} + +// Package rdns + +// Exchanger is a fake [rdns.Exchanger] implementation for tests. +type Exchanger struct { + OnExchange func(ip netip.Addr) (host string, ttl time.Duration, err error) +} + +// type check +var _ rdns.Exchanger = (*Exchanger)(nil) + +// Exchange implements [rdns.Exchanger] interface for *Exchanger. +func (e *Exchanger) Exchange(ip netip.Addr) (host string, ttl time.Duration, err error) { + return e.OnExchange(ip) +} + // Module dnsproxy // Package upstream diff --git a/internal/aghtest/interface_test.go b/internal/aghtest/interface_test.go index 9141d132..a17c5e67 100644 --- a/internal/aghtest/interface_test.go +++ b/internal/aghtest/interface_test.go @@ -1,3 +1,11 @@ package aghtest_test +import ( + "github.com/AdguardTeam/AdGuardHome/internal/aghtest" + "github.com/AdguardTeam/AdGuardHome/internal/filtering" +) + // Put interface checks that cause import cycles here. + +// type check +var _ filtering.Resolver = (*aghtest.Resolver)(nil) diff --git a/internal/aghtest/resolver.go b/internal/aghtest/resolver.go deleted file mode 100644 index 3c2df964..00000000 --- a/internal/aghtest/resolver.go +++ /dev/null @@ -1,57 +0,0 @@ -package aghtest - -import ( - "context" - "crypto/sha256" - "net" - "sync" -) - -// TestResolver is a Resolver for tests. -type TestResolver struct { - counter int - counterLock sync.Mutex -} - -// HostToIPs generates IPv4 and IPv6 from host. -func (r *TestResolver) HostToIPs(host string) (ipv4, ipv6 net.IP) { - hash := sha256.Sum256([]byte(host)) - - return net.IP(hash[:4]), net.IP(hash[4:20]) -} - -// LookupIP implements Resolver interface for *testResolver. It returns the -// slice of net.IP with IPv4 and IPv6 instances. -func (r *TestResolver) LookupIP(_ context.Context, _, host string) (ips []net.IP, err error) { - ipv4, ipv6 := r.HostToIPs(host) - addrs := []net.IP{ipv4, ipv6} - - r.counterLock.Lock() - defer r.counterLock.Unlock() - r.counter++ - - return addrs, nil -} - -// LookupHost implements Resolver interface for *testResolver. It returns the -// slice of IPv4 and IPv6 instances converted to strings. -func (r *TestResolver) LookupHost(host string) (addrs []string, err error) { - ipv4, ipv6 := r.HostToIPs(host) - - r.counterLock.Lock() - defer r.counterLock.Unlock() - r.counter++ - - return []string{ - ipv4.String(), - ipv6.String(), - }, nil -} - -// Counter returns the number of requests handled. -func (r *TestResolver) Counter() int { - r.counterLock.Lock() - defer r.counterLock.Unlock() - - return r.counter -} diff --git a/internal/client/addrproc.go b/internal/client/addrproc.go new file mode 100644 index 00000000..04ee50d5 --- /dev/null +++ b/internal/client/addrproc.go @@ -0,0 +1,294 @@ +package client + +import ( + "context" + "net/netip" + "sync" + "time" + + "github.com/AdguardTeam/AdGuardHome/internal/aghnet" + "github.com/AdguardTeam/AdGuardHome/internal/rdns" + "github.com/AdguardTeam/AdGuardHome/internal/whois" + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/netutil" +) + +// ErrClosed is returned from [AddressProcessor.Close] if it's closed more than +// once. +const ErrClosed errors.Error = "use of closed address processor" + +// AddressProcessor is the interface for types that can process clients. +type AddressProcessor interface { + Process(ip netip.Addr) + Close() (err error) +} + +// EmptyAddrProc is an [AddressProcessor] that does nothing. +type EmptyAddrProc struct{} + +// type check +var _ AddressProcessor = EmptyAddrProc{} + +// Process implements the [AddressProcessor] interface for EmptyAddrProc. +func (EmptyAddrProc) Process(_ netip.Addr) {} + +// Close implements the [AddressProcessor] interface for EmptyAddrProc. +func (EmptyAddrProc) Close() (_ error) { return nil } + +// DefaultAddrProcConfig is the configuration structure for address processors. +type DefaultAddrProcConfig struct { + // DialContext is used to create TCP connections to WHOIS servers. + // DialContext must not be nil if [DefaultAddrProcConfig.UseWHOIS] is true. + DialContext aghnet.DialContextFunc + + // Exchanger is used to perform rDNS queries. Exchanger must not be nil if + // [DefaultAddrProcConfig.UseRDNS] is true. + Exchanger rdns.Exchanger + + // PrivateSubnets are used to determine if an incoming IP address is + // private. It must not be nil. + PrivateSubnets netutil.SubnetSet + + // AddressUpdater is used to update the information about a client's IP + // address. It must not be nil. + AddressUpdater AddressUpdater + + // InitialAddresses are the addresses that are queued for processing + // immediately by [NewDefaultAddrProc]. + InitialAddresses []netip.Addr + + // UseRDNS, if true, enables resolving of client IP addresses using reverse + // DNS. + UseRDNS bool + + // UsePrivateRDNS, if true, enables resolving of private client IP addresses + // using reverse DNS. See [DefaultAddrProcConfig.PrivateSubnets]. + UsePrivateRDNS bool + + // UseWHOIS, if true, enables resolving of client IP addresses using WHOIS. + UseWHOIS bool +} + +// AddressUpdater is the interface for storages of DNS clients that can update +// information about them. +// +// TODO(a.garipov): Consider using the actual client storage once it is moved +// into this package. +type AddressUpdater interface { + // UpdateAddress updates information about an IP address, setting host (if + // not empty) and WHOIS information (if not nil). + UpdateAddress(ip netip.Addr, host string, info *whois.Info) +} + +// DefaultAddrProc processes incoming client addresses with rDNS and WHOIS, if +// configured, and updates that information in a client storage. +type DefaultAddrProc struct { + // clientIPsMu serializes closure of clientIPs and access to isClosed. + clientIPsMu *sync.Mutex + + // clientIPs is the channel queueing client processing tasks. + clientIPs chan netip.Addr + + // rdns is used to perform rDNS lookups of clients' IP addresses. + rdns rdns.Interface + + // whois is used to perform WHOIS lookups of clients' IP addresses. + whois whois.Interface + + // addrUpdater is used to update the information about a client's IP + // address. + addrUpdater AddressUpdater + + // privateSubnets are used to determine if an incoming IP address is + // private. + privateSubnets netutil.SubnetSet + + // isClosed is set to true once the address processor is closed. + isClosed bool + + // usePrivateRDNS, if true, enables resolving of private client IP addresses + // using reverse DNS. + usePrivateRDNS bool +} + +const ( + // defaultQueueSize is the size of queue of IPs for rDNS and WHOIS + // processing. + defaultQueueSize = 255 + + // defaultCacheSize is the maximum size of the cache for rDNS and WHOIS + // processing. It must be greater than zero. + defaultCacheSize = 10_000 + + // defaultIPTTL is the Time to Live duration for IP addresses cached by + // rDNS and WHOIS. + defaultIPTTL = 1 * time.Hour +) + +// NewDefaultAddrProc returns a new running client address processor. c must +// not be nil. +func NewDefaultAddrProc(c *DefaultAddrProcConfig) (p *DefaultAddrProc) { + p = &DefaultAddrProc{ + clientIPsMu: &sync.Mutex{}, + clientIPs: make(chan netip.Addr, defaultQueueSize), + rdns: &rdns.Empty{}, + addrUpdater: c.AddressUpdater, + whois: &whois.Empty{}, + privateSubnets: c.PrivateSubnets, + usePrivateRDNS: c.UsePrivateRDNS, + } + + if c.UseRDNS { + p.rdns = rdns.New(&rdns.Config{ + Exchanger: c.Exchanger, + CacheSize: defaultCacheSize, + CacheTTL: defaultIPTTL, + }) + } + + if c.UseWHOIS { + p.whois = newWHOIS(c.DialContext) + } + + go p.process() + + for _, ip := range c.InitialAddresses { + p.Process(ip) + } + + return p +} + +// newWHOIS returns a whois.Interface instance using the given function for +// dialing. +func newWHOIS(dialFunc aghnet.DialContextFunc) (w whois.Interface) { + // TODO(s.chzhen): Consider making configurable. + const ( + // defaultTimeout is the timeout for WHOIS requests. + defaultTimeout = 5 * time.Second + + // defaultMaxConnReadSize is an upper limit in bytes for reading from a + // net.Conn. + defaultMaxConnReadSize = 64 * 1024 + + // defaultMaxRedirects is the maximum redirects count. + defaultMaxRedirects = 5 + + // defaultMaxInfoLen is the maximum length of whois.Info fields. + defaultMaxInfoLen = 250 + ) + + return whois.New(&whois.Config{ + DialContext: dialFunc, + ServerAddr: whois.DefaultServer, + Port: whois.DefaultPort, + Timeout: defaultTimeout, + CacheSize: defaultCacheSize, + MaxConnReadSize: defaultMaxConnReadSize, + MaxRedirects: defaultMaxRedirects, + MaxInfoLen: defaultMaxInfoLen, + CacheTTL: defaultIPTTL, + }) +} + +// type check +var _ AddressProcessor = (*DefaultAddrProc)(nil) + +// Process implements the [AddressProcessor] interface for *DefaultAddrProc. +func (p *DefaultAddrProc) Process(ip netip.Addr) { + p.clientIPsMu.Lock() + defer p.clientIPsMu.Unlock() + + if p.isClosed { + return + } + + select { + case p.clientIPs <- ip: + // Go on. + default: + log.Debug("clients: ip channel is full; len: %d", len(p.clientIPs)) + } +} + +// process processes the incoming client IP-address information. It is intended +// to be used as a goroutine. Once clientIPs is closed, process exits. +func (p *DefaultAddrProc) process() { + defer log.OnPanic("addrProcessor.process") + + log.Info("clients: processing addresses") + + for ip := range p.clientIPs { + host := p.processRDNS(ip) + info := p.processWHOIS(ip) + + p.addrUpdater.UpdateAddress(ip, host, info) + } + + log.Info("clients: finished processing addresses") +} + +// processRDNS resolves the clients' IP addresses using reverse DNS. host is +// empty if there were errors or if the information hasn't changed. +func (p *DefaultAddrProc) processRDNS(ip netip.Addr) (host string) { + start := time.Now() + log.Debug("clients: processing %s with rdns", ip) + defer func() { + log.Debug("clients: finished processing %s with rdns in %s", ip, time.Since(start)) + }() + + ok := p.shouldResolve(ip) + if !ok { + return + } + + host, changed := p.rdns.Process(ip) + if !changed { + host = "" + } + + return host +} + +// shouldResolve returns false if ip is a loopback address, or ip is private and +// resolving of private addresses is disabled. +func (p *DefaultAddrProc) shouldResolve(ip netip.Addr) (ok bool) { + return !ip.IsLoopback() && + (p.usePrivateRDNS || !p.privateSubnets.Contains(ip.AsSlice())) +} + +// processWHOIS looks up the information about clients' IP addresses in the +// WHOIS databases. info is nil if there were errors or if the information +// hasn't changed. +func (p *DefaultAddrProc) processWHOIS(ip netip.Addr) (info *whois.Info) { + start := time.Now() + log.Debug("clients: processing %s with whois", ip) + defer func() { + log.Debug("clients: finished processing %s with whois in %s", ip, time.Since(start)) + }() + + // TODO(s.chzhen): Move the timeout logic from WHOIS configuration to the + // context. + info, changed := p.whois.Process(context.Background(), ip) + if !changed { + info = nil + } + + return info +} + +// Close implements the [AddressProcessor] interface for *DefaultAddrProc. +func (p *DefaultAddrProc) Close() (err error) { + p.clientIPsMu.Lock() + defer p.clientIPsMu.Unlock() + + if p.isClosed { + return ErrClosed + } + + close(p.clientIPs) + p.isClosed = true + + return nil +} diff --git a/internal/client/addrproc_test.go b/internal/client/addrproc_test.go new file mode 100644 index 00000000..c6b847cd --- /dev/null +++ b/internal/client/addrproc_test.go @@ -0,0 +1,259 @@ +package client_test + +import ( + "context" + "io" + "net" + "net/netip" + "testing" + "time" + + "github.com/AdguardTeam/AdGuardHome/internal/aghtest" + "github.com/AdguardTeam/AdGuardHome/internal/client" + "github.com/AdguardTeam/AdGuardHome/internal/whois" + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/netutil" + "github.com/AdguardTeam/golibs/testutil" + "github.com/AdguardTeam/golibs/testutil/fakenet" + "github.com/stretchr/testify/assert" +) + +func TestEmptyAddrProc(t *testing.T) { + t.Parallel() + + p := client.EmptyAddrProc{} + + assert.NotPanics(t, func() { + p.Process(testIP) + }) + + assert.NotPanics(t, func() { + err := p.Close() + assert.NoError(t, err) + }) +} + +func TestDefaultAddrProc_Process_rDNS(t *testing.T) { + t.Parallel() + + privateIP := netip.MustParseAddr("192.168.0.1") + + testCases := []struct { + rdnsErr error + ip netip.Addr + name string + host string + usePrivate bool + wantUpd bool + }{{ + rdnsErr: nil, + ip: testIP, + name: "success", + host: testHost, + usePrivate: false, + wantUpd: true, + }, { + rdnsErr: nil, + ip: testIP, + name: "no_host", + host: "", + usePrivate: false, + wantUpd: false, + }, { + rdnsErr: nil, + ip: netip.MustParseAddr("127.0.0.1"), + name: "localhost", + host: "", + usePrivate: false, + wantUpd: false, + }, { + rdnsErr: nil, + ip: privateIP, + name: "private_ignored", + host: "", + usePrivate: false, + wantUpd: false, + }, { + rdnsErr: nil, + ip: privateIP, + name: "private_processed", + host: "private.example", + usePrivate: true, + wantUpd: true, + }, { + rdnsErr: errors.Error("rdns error"), + ip: testIP, + name: "rdns_error", + host: "", + usePrivate: false, + wantUpd: false, + }} + + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + updIPCh := make(chan netip.Addr, 1) + updHostCh := make(chan string, 1) + updInfoCh := make(chan *whois.Info, 1) + + p := client.NewDefaultAddrProc(&client.DefaultAddrProcConfig{ + DialContext: func(_ context.Context, _, _ string) (conn net.Conn, err error) { + panic("not implemented") + }, + Exchanger: &aghtest.Exchanger{ + OnExchange: func(ip netip.Addr) (host string, ttl time.Duration, err error) { + return tc.host, 0, tc.rdnsErr + }, + }, + PrivateSubnets: netutil.SubnetSetFunc(netutil.IsLocallyServed), + AddressUpdater: &aghtest.AddressUpdater{ + OnUpdateAddress: newOnUpdateAddress(tc.wantUpd, updIPCh, updHostCh, updInfoCh), + }, + UseRDNS: true, + UsePrivateRDNS: tc.usePrivate, + UseWHOIS: false, + }) + testutil.CleanupAndRequireSuccess(t, p.Close) + + p.Process(tc.ip) + + if !tc.wantUpd { + return + } + + gotIP, _ := testutil.RequireReceive(t, updIPCh, testTimeout) + assert.Equal(t, tc.ip, gotIP) + + gotHost, _ := testutil.RequireReceive(t, updHostCh, testTimeout) + assert.Equal(t, tc.host, gotHost) + + gotInfo, _ := testutil.RequireReceive(t, updInfoCh, testTimeout) + assert.Nil(t, gotInfo) + }) + } +} + +// newOnUpdateAddress is a test helper that returns a new OnUpdateAddress +// callback using the provided channels if an update is expected and panicking +// otherwise. +func newOnUpdateAddress( + want bool, + ips chan<- netip.Addr, + hosts chan<- string, + infos chan<- *whois.Info, +) (f func(ip netip.Addr, host string, info *whois.Info)) { + return func(ip netip.Addr, host string, info *whois.Info) { + if !want { + panic("got unexpected update") + } + + ips <- ip + hosts <- host + infos <- info + } +} + +func TestDefaultAddrProc_Process_WHOIS(t *testing.T) { + t.Parallel() + + testCases := []struct { + wantInfo *whois.Info + exchErr error + name string + wantUpd bool + }{{ + wantInfo: &whois.Info{ + City: testWHOISCity, + }, + exchErr: nil, + name: "success", + wantUpd: true, + }, { + wantInfo: nil, + exchErr: nil, + name: "no_info", + wantUpd: false, + }, { + wantInfo: nil, + exchErr: errors.Error("whois error"), + name: "whois_error", + wantUpd: false, + }} + + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + whoisConn := &fakenet.Conn{ + OnClose: func() (err error) { return nil }, + OnRead: func(b []byte) (n int, err error) { + if tc.wantInfo == nil { + return 0, tc.exchErr + } + + data := "city: " + tc.wantInfo.City + "\n" + copy(b, data) + + return len(data), io.EOF + }, + OnSetDeadline: func(_ time.Time) (err error) { return nil }, + OnWrite: func(b []byte) (n int, err error) { return len(b), nil }, + } + + updIPCh := make(chan netip.Addr, 1) + updHostCh := make(chan string, 1) + updInfoCh := make(chan *whois.Info, 1) + + p := client.NewDefaultAddrProc(&client.DefaultAddrProcConfig{ + DialContext: func(_ context.Context, _, _ string) (conn net.Conn, err error) { + return whoisConn, nil + }, + Exchanger: &aghtest.Exchanger{ + OnExchange: func(_ netip.Addr) (_ string, _ time.Duration, _ error) { + panic("not implemented") + }, + }, + PrivateSubnets: netutil.SubnetSetFunc(netutil.IsLocallyServed), + AddressUpdater: &aghtest.AddressUpdater{ + OnUpdateAddress: newOnUpdateAddress(tc.wantUpd, updIPCh, updHostCh, updInfoCh), + }, + UseRDNS: false, + UsePrivateRDNS: false, + UseWHOIS: true, + }) + testutil.CleanupAndRequireSuccess(t, p.Close) + + p.Process(testIP) + + if !tc.wantUpd { + return + } + + gotIP, _ := testutil.RequireReceive(t, updIPCh, testTimeout) + assert.Equal(t, testIP, gotIP) + + gotHost, _ := testutil.RequireReceive(t, updHostCh, testTimeout) + assert.Empty(t, gotHost) + + gotInfo, _ := testutil.RequireReceive(t, updInfoCh, testTimeout) + assert.Equal(t, tc.wantInfo, gotInfo) + }) + } +} + +func TestDefaultAddrProc_Close(t *testing.T) { + t.Parallel() + + p := client.NewDefaultAddrProc(&client.DefaultAddrProcConfig{}) + + err := p.Close() + assert.NoError(t, err) + + err = p.Close() + assert.ErrorIs(t, err, client.ErrClosed) +} diff --git a/internal/client/client.go b/internal/client/client.go new file mode 100644 index 00000000..6cfcec79 --- /dev/null +++ b/internal/client/client.go @@ -0,0 +1,5 @@ +// Package client contains types and logic dealing with AdGuard Home's DNS +// clients. +// +// TODO(a.garipov): Expand. +package client diff --git a/internal/client/client_test.go b/internal/client/client_test.go new file mode 100644 index 00000000..b579f4f8 --- /dev/null +++ b/internal/client/client_test.go @@ -0,0 +1,25 @@ +package client_test + +import ( + "net/netip" + "testing" + "time" + + "github.com/AdguardTeam/golibs/testutil" +) + +func TestMain(m *testing.M) { + testutil.DiscardLogOutput(m) +} + +// testHost is the common hostname for tests. +const testHost = "client.example" + +// testTimeout is the common timeout for tests. +const testTimeout = 1 * time.Second + +// testWHOISCity is the common city for tests. +const testWHOISCity = "Brussels" + +// testIP is the common IP address for tests. +var testIP = netip.MustParseAddr("1.2.3.4") diff --git a/internal/dhcpd/db.go b/internal/dhcpd/db.go index 3a6e98b7..9ec4716d 100644 --- a/internal/dhcpd/db.go +++ b/internal/dhcpd/db.go @@ -9,7 +9,7 @@ import ( "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" - "github.com/google/renameio/maybe" + "github.com/google/renameio/v2/maybe" "golang.org/x/exp/slices" ) diff --git a/internal/dnsforward/config.go b/internal/dnsforward/config.go index aa3f4808..ffd51a78 100644 --- a/internal/dnsforward/config.go +++ b/internal/dnsforward/config.go @@ -13,6 +13,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghalg" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/aghtls" + "github.com/AdguardTeam/AdGuardHome/internal/client" "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/golibs/errors" @@ -270,7 +271,13 @@ type ServerConfig struct { UDPListenAddrs []*net.UDPAddr // UDP listen address TCPListenAddrs []*net.TCPAddr // TCP listen address UpstreamConfig *proxy.UpstreamConfig // Upstream DNS servers config - OnDNSRequest func(d *proxy.DNSContext) + + // AddrProcConf defines the configuration for the client IP processor. + // If nil, [client.EmptyAddrProc] is used. + // + // TODO(a.garipov): The use of [client.EmptyAddrProc] is a crutch for tests. + // Remove that. + AddrProcConf *client.DefaultAddrProcConfig FilteringConfig TLSConfig @@ -298,9 +305,6 @@ type ServerConfig struct { // DNS64Prefixes is a slice of NAT64 prefixes to be used for DNS64. DNS64Prefixes []netip.Prefix - // ResolveClients signals if the RDNS should resolve clients' addresses. - ResolveClients bool - // UsePrivateRDNS defines if the PTR requests for unknown addresses from // locally-served networks should be resolved via private PTR resolvers. UsePrivateRDNS bool @@ -340,6 +344,7 @@ func (s *Server) createProxyConfig() (conf proxy.Config, err error) { UpstreamConfig: srvConf.UpstreamConfig, BeforeRequestHandler: s.beforeRequestHandler, RequestHandler: s.handleDNSRequest, + HTTPSServerName: aghhttp.UserAgent(), EnableEDNSClientSubnet: srvConf.EDNSClientSubnet.Enabled, MaxGoroutines: int(srvConf.MaxGoroutines), UseDNS64: srvConf.UseDNS64, diff --git a/internal/dnsforward/dialcontext.go b/internal/dnsforward/dialcontext.go new file mode 100644 index 00000000..f917f54c --- /dev/null +++ b/internal/dnsforward/dialcontext.go @@ -0,0 +1,57 @@ +package dnsforward + +import ( + "context" + "fmt" + "net" + "time" + + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/log" +) + +// DialContext is an [aghnet.DialContextFunc] that uses s to resolve hostnames. +func (s *Server) DialContext(ctx context.Context, network, addr string) (conn net.Conn, err error) { + log.Debug("dnsforward: dialing %q for network %q", addr, network) + + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + + dialer := &net.Dialer{ + // TODO(a.garipov): Consider making configurable. + Timeout: time.Minute * 5, + } + + if net.ParseIP(host) != nil { + return dialer.DialContext(ctx, network, addr) + } + + addrs, err := s.Resolve(host) + if err != nil { + return nil, fmt.Errorf("resolving %q: %w", host, err) + } + + log.Debug("dnsforward: resolving %q: %v", host, addrs) + + if len(addrs) == 0 { + return nil, fmt.Errorf("no addresses for host %q", host) + } + + var dialErrs []error + for _, a := range addrs { + addr = net.JoinHostPort(a.String(), port) + conn, err = dialer.DialContext(ctx, network, addr) + if err != nil { + dialErrs = append(dialErrs, err) + + continue + } + + return conn, err + } + + // TODO(a.garipov): Use errors.Join in Go 1.20. + return nil, errors.List(fmt.Sprintf("dialing %q", addr), dialErrs...) +} diff --git a/internal/dnsforward/dnsforward.go b/internal/dnsforward/dnsforward.go index 70abd660..730e88f8 100644 --- a/internal/dnsforward/dnsforward.go +++ b/internal/dnsforward/dnsforward.go @@ -14,6 +14,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghalg" "github.com/AdguardTeam/AdGuardHome/internal/aghnet" + "github.com/AdguardTeam/AdGuardHome/internal/client" "github.com/AdguardTeam/AdGuardHome/internal/dhcpd" "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/AdGuardHome/internal/querylog" @@ -99,8 +100,17 @@ type Server struct { // must be a valid domain name plus dots on each side. localDomainSuffix string - ipset ipsetCtx - privateNets netutil.SubnetSet + ipset ipsetCtx + privateNets netutil.SubnetSet + + // addrProc, if not nil, is used to process clients' IP addresses with rDNS, + // WHOIS, etc. + addrProc client.AddressProcessor + + // localResolvers is a DNS proxy instance used to resolve PTR records for + // addresses considered private as per the [privateNets]. + // + // TODO(e.burkov): Remove once the local resolvers logic moved to dnsproxy. localResolvers *proxy.Proxy sysResolvers aghnet.SystemResolvers @@ -170,6 +180,9 @@ const ( // NewServer creates a new instance of the dnsforward.Server // Note: this function must be called only once +// +// TODO(a.garipov): How many constructors and initializers does this thing have? +// Refactor! func NewServer(p DNSCreateParams) (s *Server, err error) { var localDomainSuffix string if p.LocalDomain == "" { @@ -257,14 +270,25 @@ func (s *Server) WriteDiskConfig(c *FilteringConfig) { c.UpstreamDNS = stringutil.CloneSlice(sc.UpstreamDNS) } -// RDNSSettings returns the copy of actual RDNS configuration. -func (s *Server) RDNSSettings() (localPTRResolvers []string, resolveClients, resolvePTR bool) { +// LocalPTRResolvers returns the current local PTR resolver configuration. +func (s *Server) LocalPTRResolvers() (localPTRResolvers []string) { s.serverLock.RLock() defer s.serverLock.RUnlock() - return stringutil.CloneSlice(s.conf.LocalPTRResolvers), - s.conf.ResolveClients, - s.conf.UsePrivateRDNS + return stringutil.CloneSlice(s.conf.LocalPTRResolvers) +} + +// AddrProcConfig returns the current address processing configuration. Only +// fields c.UsePrivateRDNS, c.UseRDNS, and c.UseWHOIS are filled. +func (s *Server) AddrProcConfig() (c *client.DefaultAddrProcConfig) { + s.serverLock.RLock() + defer s.serverLock.RUnlock() + + return &client.DefaultAddrProcConfig{ + UsePrivateRDNS: s.conf.UsePrivateRDNS, + UseRDNS: s.conf.AddrProcConf.UseRDNS, + UseWHOIS: s.conf.AddrProcConf.UseWHOIS, + } } // Resolve - get IP addresses by host name from an upstream server. @@ -292,17 +316,13 @@ const ( var _ rdns.Exchanger = (*Server)(nil) // Exchange implements the [rdns.Exchanger] interface for *Server. -func (s *Server) Exchange(ip netip.Addr) (host string, err error) { +func (s *Server) Exchange(ip netip.Addr) (host string, ttl time.Duration, err error) { s.serverLock.RLock() defer s.serverLock.RUnlock() - if !s.conf.ResolveClients { - return "", nil - } - arpa, err := netutil.IPToReversedAddr(ip.AsSlice()) if err != nil { - return "", fmt.Errorf("reversing ip: %w", err) + return "", 0, fmt.Errorf("reversing ip: %w", err) } arpa = dns.Fqdn(arpa) @@ -318,16 +338,17 @@ func (s *Server) Exchange(ip netip.Addr) (host string, err error) { Qclass: dns.ClassINET, }}, } - ctx := &proxy.DNSContext{ + + dctx := &proxy.DNSContext{ Proto: "udp", Req: req, StartTime: time.Now(), } var resolver *proxy.Proxy - if s.isPrivateIP(ip) { + if s.privateNets.Contains(ip.AsSlice()) { if !s.conf.UsePrivateRDNS { - return "", nil + return "", 0, nil } resolver = s.localResolvers @@ -336,53 +357,48 @@ func (s *Server) Exchange(ip netip.Addr) (host string, err error) { resolver = s.internalProxy } - if err = resolver.Resolve(ctx); err != nil { - return "", err + if err = resolver.Resolve(dctx); err != nil { + return "", 0, err } - return hostFromPTR(ctx.Res) + return hostFromPTR(dctx.Res) } // hostFromPTR returns domain name from the PTR response or error. -func hostFromPTR(resp *dns.Msg) (host string, err error) { +func hostFromPTR(resp *dns.Msg) (host string, ttl time.Duration, err error) { // Distinguish between NODATA response and a failed request. if resp.Rcode != dns.RcodeSuccess && resp.Rcode != dns.RcodeNameError { - return "", fmt.Errorf( + return "", 0, fmt.Errorf( "received %s response: %w", dns.RcodeToString[resp.Rcode], ErrRDNSFailed, ) } + var ttlSec uint32 + for _, ans := range resp.Answer { ptr, ok := ans.(*dns.PTR) - if ok { - return strings.TrimSuffix(ptr.Ptr, "."), nil + if !ok { + continue + } + + if ptr.Hdr.Ttl > ttlSec { + host = ptr.Ptr + ttlSec = ptr.Hdr.Ttl } } - return "", ErrRDNSNoData -} + if host != "" { + // NOTE: Don't use [aghnet.NormalizeDomain] to retain original letter + // case. + host = strings.TrimSuffix(host, ".") + ttl = time.Duration(ttlSec) * time.Second -// isPrivateIP returns true if the ip is private. -func (s *Server) isPrivateIP(ip netip.Addr) (ok bool) { - return s.privateNets.Contains(ip.AsSlice()) -} - -// ShouldResolveClient returns false if ip is a loopback address, or ip is -// private and resolving of private addresses is disabled. -func (s *Server) ShouldResolveClient(ip netip.Addr) (ok bool) { - if ip.IsLoopback() { - return false + return host, ttl, nil } - isPrivate := s.isPrivateIP(ip) - - s.serverLock.RLock() - defer s.serverLock.RUnlock() - - return s.conf.ResolveClients && - (s.conf.UsePrivateRDNS || !isPrivate) + return "", 0, ErrRDNSNoData } // Start starts the DNS server. @@ -457,23 +473,27 @@ func (s *Server) filterOurDNSAddrs(addrs []string) (filtered []string, err error return stringutil.FilterOut(addrs, ourAddrsSet.Has), nil } -// setupResolvers initializes the resolvers for local addresses. For internal -// use only. -func (s *Server) setupResolvers(localAddrs []string) (err error) { +// setupLocalResolvers initializes the resolvers for local addresses. For +// internal use only. +func (s *Server) setupLocalResolvers() (err error) { bootstraps := s.conf.BootstrapDNS - if len(localAddrs) == 0 { - localAddrs = s.sysResolvers.Get() + resolvers := s.conf.LocalPTRResolvers + + if len(resolvers) == 0 { + resolvers = s.sysResolvers.Get() bootstraps = nil + } else { + resolvers = stringutil.FilterOut(resolvers, IsCommentOrEmpty) } - localAddrs, err = s.filterOurDNSAddrs(localAddrs) + resolvers, err = s.filterOurDNSAddrs(resolvers) if err != nil { return err } - log.Debug("dnsforward: upstreams to resolve ptr for local addresses: %v", localAddrs) + log.Debug("dnsforward: upstreams to resolve ptr for local addresses: %v", resolvers) - upsConfig, err := s.prepareUpstreamConfig(localAddrs, nil, &upstream.Options{ + uc, err := s.prepareUpstreamConfig(resolvers, nil, &upstream.Options{ Bootstrap: bootstraps, Timeout: defaultLocalTimeout, // TODO(e.burkov): Should we verify server's certificates? @@ -486,10 +506,17 @@ func (s *Server) setupResolvers(localAddrs []string) (err error) { s.localResolvers = &proxy.Proxy{ Config: proxy.Config{ - UpstreamConfig: upsConfig, + UpstreamConfig: uc, }, } + if s.conf.UsePrivateRDNS && + // Only set the upstream config if there are any upstreams. It's safe + // to put nil into [proxy.Config.PrivateRDNSUpstreamConfig]. + len(uc.Upstreams)+len(uc.DomainReservedUpstreams)+len(uc.SpecifiedDomainUpstreams) > 0 { + s.dnsProxy.PrivateRDNSUpstreamConfig = uc + } + return nil } @@ -539,25 +566,48 @@ func (s *Server) Prepare(conf *ServerConfig) (err error) { return fmt.Errorf("preparing access: %w", err) } - s.registerHandlers() - + // Set the proxy here because [setupLocalResolvers] sets its values. + // // TODO(e.burkov): Remove once the local resolvers logic moved to dnsproxy. - err = s.setupResolvers(s.conf.LocalPTRResolvers) + s.dnsProxy = &proxy.Proxy{Config: proxyConfig} + + err = s.setupLocalResolvers() if err != nil { return fmt.Errorf("setting up resolvers: %w", err) } - if s.conf.UsePrivateRDNS { - proxyConfig.PrivateRDNSUpstreamConfig = s.localResolvers.UpstreamConfig - } - - s.dnsProxy = &proxy.Proxy{Config: proxyConfig} - s.recDetector.clear() + s.setupAddrProc() + + s.registerHandlers() + return nil } +// setupAddrProc initializes the address processor. For internal use only. +func (s *Server) setupAddrProc() { + // TODO(a.garipov): This is a crutch for tests; remove. + if s.conf.AddrProcConf == nil { + s.conf.AddrProcConf = &client.DefaultAddrProcConfig{} + } + if s.conf.AddrProcConf.AddressUpdater == nil { + s.addrProc = client.EmptyAddrProc{} + } else { + c := s.conf.AddrProcConf + c.DialContext = s.DialContext + c.PrivateSubnets = s.privateNets + c.UsePrivateRDNS = s.conf.UsePrivateRDNS + s.addrProc = client.NewDefaultAddrProc(s.conf.AddrProcConf) + + // Clear the initial addresses to not resolve them again. + // + // TODO(a.garipov): Consider ways of removing this once more client + // logic is moved to package client. + c.InitialAddresses = nil + } +} + // validateBlockingMode returns an error if the blocking mode data aren't valid. func validateBlockingMode(mode BlockingMode, blockingIPv4, blockingIPv6 net.IP) (err error) { switch mode { @@ -696,6 +746,11 @@ func (s *Server) Reconfigure(conf *ServerConfig) error { // TODO(a.garipov): This whole piece of API is weird and needs to be remade. if conf == nil { conf = &s.conf + } else { + closeErr := s.addrProc.Close() + if closeErr != nil { + log.Error("dnsforward: closing address processor: %s", closeErr) + } } err = s.Prepare(conf) diff --git a/internal/dnsforward/dnsforward_test.go b/internal/dnsforward/dnsforward_test.go index 705227a1..775a97b5 100644 --- a/internal/dnsforward/dnsforward_test.go +++ b/internal/dnsforward/dnsforward_test.go @@ -1,6 +1,7 @@ package dnsforward import ( + "context" "crypto/ecdsa" "crypto/rand" "crypto/rsa" @@ -39,11 +40,29 @@ func TestMain(m *testing.M) { testutil.DiscardLogOutput(m) } +// testTimeout is the common timeout for tests. +// +// TODO(a.garipov): Use more. +const testTimeout = 1 * time.Second + +// testQuestionTarget is the common question target for tests. +// +// TODO(a.garipov): Use more. +const testQuestionTarget = "target.example" + const ( tlsServerName = "testdns.adguard.com" testMessagesCount = 10 ) +// testClientAddr is the common net.Addr for tests. +// +// TODO(a.garipov): Use more. +var testClientAddr net.Addr = &net.TCPAddr{ + IP: net.IP{1, 2, 3, 4}, + Port: 12345, +} + func startDeferStop(t *testing.T, s *Server) { t.Helper() @@ -53,6 +72,13 @@ func startDeferStop(t *testing.T, s *Server) { testutil.CleanupAndRequireSuccess(t, s.Stop) } +// packageUpstreamVariableMu is used to serialize access to the package-level +// variables of package upstream. +// +// TODO(s.chzhen): Move these parameters to upstream options and remove this +// crutch. +var packageUpstreamVariableMu = &sync.Mutex{} + func createTestServer( t *testing.T, filterConf *filtering.Config, @@ -61,6 +87,9 @@ func createTestServer( ) (s *Server) { t.Helper() + packageUpstreamVariableMu.Lock() + defer packageUpstreamVariableMu.Unlock() + rules := `||nxdomain.example.org ||NULL.example.org^ 127.0.0.1 host.example.org @@ -307,11 +336,9 @@ func TestServer(t *testing.T) { } func TestServer_timeout(t *testing.T) { - const timeout time.Duration = time.Second - t.Run("custom", func(t *testing.T) { srvConf := &ServerConfig{ - UpstreamTimeout: timeout, + UpstreamTimeout: testTimeout, FilteringConfig: FilteringConfig{ BlockingMode: BlockingModeDefault, EDNSClientSubnet: &EDNSClientSubnet{Enabled: false}, @@ -324,7 +351,7 @@ func TestServer_timeout(t *testing.T) { err = s.Prepare(srvConf) require.NoError(t, err) - assert.Equal(t, timeout, s.conf.UpstreamTimeout) + assert.Equal(t, testTimeout, s.conf.UpstreamTimeout) }) t.Run("default", func(t *testing.T) { @@ -441,7 +468,14 @@ func TestServerRace(t *testing.T) { } func TestSafeSearch(t *testing.T) { - resolver := &aghtest.TestResolver{} + resolver := &aghtest.Resolver{ + OnLookupIP: func(_ context.Context, _, host string) (ips []net.IP, err error) { + ip4, ip6 := aghtest.HostToIPs(host) + + return []net.IP{ip4, ip6}, nil + }, + } + safeSearchConf := filtering.SafeSearchConfig{ Enabled: true, Google: true, @@ -480,7 +514,7 @@ func TestSafeSearch(t *testing.T) { client := &dns.Client{} yandexIP := net.IP{213, 180, 193, 56} - googleIP, _ := resolver.HostToIPs("forcesafesearch.google.com") + googleIP, _ := aghtest.HostToIPs("forcesafesearch.google.com") testCases := []struct { host string @@ -545,7 +579,7 @@ func TestInvalidRequest(t *testing.T) { // Send a DNS request without question. _, _, err := (&dns.Client{ - Timeout: 500 * time.Millisecond, + Timeout: testTimeout, }).Exchange(&req, addr) assert.NoErrorf(t, err, "got a response to an invalid query") @@ -928,7 +962,7 @@ func TestBlockedBySafeBrowsing(t *testing.T) { Upstream: aghtest.NewBlockUpstream(hostname, true), }) - ans4, _ := (&aghtest.TestResolver{}).HostToIPs(hostname) + ans4, _ := aghtest.HostToIPs(hostname) filterConf := &filtering.Config{ SafeBrowsingEnabled: true, @@ -1266,25 +1300,57 @@ func TestNewServer(t *testing.T) { } } +// doubleTTL is a helper function that returns a clone of DNS PTR with appended +// copy of first answer record with doubled TTL. +func doubleTTL(msg *dns.Msg) (resp *dns.Msg) { + if msg == nil { + return nil + } + + if len(msg.Answer) == 0 { + return msg + } + + rec := msg.Answer[0] + ptr, ok := rec.(*dns.PTR) + if !ok { + return msg + } + + clone := *ptr + clone.Hdr.Ttl *= 2 + msg.Answer = append(msg.Answer, &clone) + + return msg +} + func TestServer_Exchange(t *testing.T) { const ( onesHost = "one.one.one.one" + twosHost = "two.two.two.two" localDomainHost = "local.domain" + + defaultTTL = time.Second * 60 ) var ( onesIP = netip.MustParseAddr("1.1.1.1") + twosIP = netip.MustParseAddr("2.2.2.2") localIP = netip.MustParseAddr("192.168.1.1") ) - revExtIPv4, err := netutil.IPToReversedAddr(onesIP.AsSlice()) + onesRevExtIPv4, err := netutil.IPToReversedAddr(onesIP.AsSlice()) + require.NoError(t, err) + + twosRevExtIPv4, err := netutil.IPToReversedAddr(twosIP.AsSlice()) require.NoError(t, err) extUpstream := &aghtest.UpstreamMock{ OnAddress: func() (addr string) { return "external.upstream.example" }, OnExchange: func(req *dns.Msg) (resp *dns.Msg, err error) { return aghalg.Coalesce( - aghtest.MatchedResponse(req, dns.TypePTR, revExtIPv4, onesHost), + aghtest.MatchedResponse(req, dns.TypePTR, onesRevExtIPv4, onesHost), + doubleTTL(aghtest.MatchedResponse(req, dns.TypePTR, twosRevExtIPv4, twosHost)), new(dns.Msg).SetRcode(req, dns.RcodeNameError), ), nil }, @@ -1320,53 +1386,65 @@ func TestServer_Exchange(t *testing.T) { }, } - srv.conf.ResolveClients = true srv.conf.UsePrivateRDNS = true - srv.privateNets = netutil.SubnetSetFunc(netutil.IsLocallyServed) testCases := []struct { - name string - want string + req netip.Addr wantErr error locUpstream upstream.Upstream - req netip.Addr + name string + want string + wantTTL time.Duration }{{ name: "external_good", want: onesHost, wantErr: nil, locUpstream: nil, req: onesIP, + wantTTL: defaultTTL, }, { name: "local_good", want: localDomainHost, wantErr: nil, locUpstream: locUpstream, req: localIP, + wantTTL: defaultTTL, }, { name: "upstream_error", want: "", wantErr: aghtest.ErrUpstream, locUpstream: errUpstream, req: localIP, + wantTTL: 0, }, { name: "empty_answer_error", want: "", wantErr: ErrRDNSNoData, locUpstream: locUpstream, req: netip.MustParseAddr("192.168.1.2"), + wantTTL: 0, }, { name: "invalid_answer", want: "", wantErr: ErrRDNSNoData, locUpstream: nonPtrUpstream, req: localIP, + wantTTL: 0, }, { name: "refused", want: "", wantErr: ErrRDNSFailed, locUpstream: refusingUpstream, req: localIP, + wantTTL: 0, + }, { + name: "longest_ttl", + want: twosHost, + wantErr: nil, + locUpstream: nil, + req: twosIP, + wantTTL: defaultTTL * 2, }} for _, tc := range testCases { @@ -1380,73 +1458,20 @@ func TestServer_Exchange(t *testing.T) { } t.Run(tc.name, func(t *testing.T) { - host, eerr := srv.Exchange(tc.req) + host, ttl, eerr := srv.Exchange(tc.req) require.ErrorIs(t, eerr, tc.wantErr) assert.Equal(t, tc.want, host) + assert.Equal(t, tc.wantTTL, ttl) }) } t.Run("resolving_disabled", func(t *testing.T) { srv.conf.UsePrivateRDNS = false - host, eerr := srv.Exchange(localIP) + host, _, eerr := srv.Exchange(localIP) require.NoError(t, eerr) assert.Empty(t, host) }) } - -func TestServer_ShouldResolveClient(t *testing.T) { - srv := &Server{ - privateNets: netutil.SubnetSetFunc(netutil.IsLocallyServed), - } - - testCases := []struct { - ip netip.Addr - want require.BoolAssertionFunc - name string - resolve bool - usePrivate bool - }{{ - name: "default", - ip: netip.MustParseAddr("1.1.1.1"), - want: require.True, - resolve: true, - usePrivate: true, - }, { - name: "no_rdns", - ip: netip.MustParseAddr("1.1.1.1"), - want: require.False, - resolve: false, - usePrivate: true, - }, { - name: "loopback", - ip: netip.MustParseAddr("127.0.0.1"), - want: require.False, - resolve: true, - usePrivate: true, - }, { - name: "private_resolve", - ip: netip.MustParseAddr("192.168.0.1"), - want: require.True, - resolve: true, - usePrivate: true, - }, { - name: "private_no_resolve", - ip: netip.MustParseAddr("192.168.0.1"), - want: require.False, - resolve: true, - usePrivate: false, - }} - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - srv.conf.ResolveClients = tc.resolve - srv.conf.UsePrivateRDNS = tc.usePrivate - - ok := srv.ShouldResolveClient(tc.ip) - tc.want(t, ok) - }) - } -} diff --git a/internal/dnsforward/filter.go b/internal/dnsforward/filter.go index f55e3059..3f35afc2 100644 --- a/internal/dnsforward/filter.go +++ b/internal/dnsforward/filter.go @@ -50,10 +50,10 @@ func (s *Server) beforeRequestHandler( return true, nil } -// getClientRequestFilteringSettings looks up client filtering settings using -// the client's IP address and ID, if any, from dctx. -func (s *Server) getClientRequestFilteringSettings(dctx *dnsContext) *filtering.Settings { - setts := s.dnsFilter.Settings() +// clientRequestFilteringSettings looks up client filtering settings using the +// client's IP address and ID, if any, from dctx. +func (s *Server) clientRequestFilteringSettings(dctx *dnsContext) (setts *filtering.Settings) { + setts = s.dnsFilter.Settings() setts.ProtectionEnabled = dctx.protectionEnabled if s.conf.FilterHandler != nil { ip, _ := netutil.IPAndPortFromAddr(dctx.proxyCtx.Addr) diff --git a/internal/dnsforward/http.go b/internal/dnsforward/http.go index e95459c7..761cbeb4 100644 --- a/internal/dnsforward/http.go +++ b/internal/dnsforward/http.go @@ -124,7 +124,7 @@ func (s *Server) getDNSConfig() (c *jsonDNSConfig) { cacheMinTTL := s.conf.CacheMinTTL cacheMaxTTL := s.conf.CacheMaxTTL cacheOptimistic := s.conf.CacheOptimistic - resolveClients := s.conf.ResolveClients + resolveClients := s.conf.AddrProcConf.UseRDNS usePrivateRDNS := s.conf.UsePrivateRDNS localPTRUpstreams := stringutil.CloneSliceOrEmpty(s.conf.LocalPTRResolvers) @@ -314,8 +314,6 @@ func (s *Server) setConfig(dc *jsonDNSConfig) (shouldRestart bool) { setIfNotNil(&s.conf.ProtectionEnabled, dc.ProtectionEnabled) setIfNotNil(&s.conf.EnableDNSSEC, dc.DNSSECEnabled) setIfNotNil(&s.conf.AAAADisabled, dc.DisableIPv6) - setIfNotNil(&s.conf.ResolveClients, dc.ResolveClients) - setIfNotNil(&s.conf.UsePrivateRDNS, dc.UsePrivateRDNS) return s.setConfigRestartable(dc) } @@ -335,6 +333,9 @@ func setIfNotNil[T any](currentPtr, newPtr *T) (hasSet bool) { // setConfigRestartable sets the parameters which trigger a restart. // shouldRestart is true if the server should be restarted to apply changes. // s.serverLock is expected to be locked. +// +// TODO(a.garipov): Some of these could probably be updated without a restart. +// Inspect and consider refactoring. func (s *Server) setConfigRestartable(dc *jsonDNSConfig) (shouldRestart bool) { for _, hasSet := range []bool{ setIfNotNil(&s.conf.UpstreamDNS, dc.Upstreams), @@ -347,6 +348,8 @@ func (s *Server) setConfigRestartable(dc *jsonDNSConfig) (shouldRestart bool) { setIfNotNil(&s.conf.CacheMinTTL, dc.CacheMinTTL), setIfNotNil(&s.conf.CacheMaxTTL, dc.CacheMaxTTL), setIfNotNil(&s.conf.CacheOptimistic, dc.CacheOptimistic), + setIfNotNil(&s.conf.AddrProcConf.UseRDNS, dc.ResolveClients), + setIfNotNil(&s.conf.UsePrivateRDNS, dc.UsePrivateRDNS), } { shouldRestart = shouldRestart || hasSet if shouldRestart { diff --git a/internal/dnsforward/dns.go b/internal/dnsforward/process.go similarity index 93% rename from internal/dnsforward/dns.go rename to internal/dnsforward/process.go index 6007801f..60feb968 100644 --- a/internal/dnsforward/dns.go +++ b/internal/dnsforward/process.go @@ -30,6 +30,7 @@ type dnsContext struct { setts *filtering.Settings result *filtering.Result + // origResp is the response received from upstream. It is set when the // response is modified by filters. origResp *dns.Msg @@ -48,13 +49,13 @@ type dnsContext struct { // clientID is the ClientID from DoH, DoQ, or DoT, if provided. clientID string + // startTime is the time at which the processing of the request has started. + startTime time.Time + // origQuestion is the question received from the client. It is set // when the request is modified by rewrites. origQuestion dns.Question - // startTime is the time at which the processing of the request has started. - startTime time.Time - // protectionEnabled shows if the filtering is enabled, and if the // server's DNS filter is ready. protectionEnabled bool @@ -160,6 +161,22 @@ func (s *Server) processRecursion(dctx *dnsContext) (rc resultCode) { return resultCodeSuccess } +// mozillaFQDN is the domain used to signal the Firefox browser to not use its +// own DoH server. +// +// See https://support.mozilla.org/en-US/kb/canary-domain-use-application-dnsnet. +const mozillaFQDN = "use-application-dns.net." + +// healthcheckFQDN is a reserved domain-name used for healthchecking. +// +// [Section 6.2 of RFC 6761] states that DNS Registries/Registrars must not +// grant requests to register test names in the normal way to any person or +// entity, making domain names under the .test TLD free to use in internal +// purposes. +// +// [Section 6.2 of RFC 6761]: https://www.rfc-editor.org/rfc/rfc6761.html#section-6.2 +const healthcheckFQDN = "healthcheck.adguardhome.test." + // processInitial terminates the following processing for some requests if // needed and enriches dctx with some client-specific information. // @@ -169,6 +186,8 @@ func (s *Server) processInitial(dctx *dnsContext) (rc resultCode) { defer log.Debug("dnsforward: finished processing initial") pctx := dctx.proxyCtx + s.processClientIP(pctx.Addr) + q := pctx.Req.Question[0] qt := q.Qtype if s.conf.AAAADisabled && qt == dns.TypeAAAA { @@ -177,28 +196,13 @@ func (s *Server) processInitial(dctx *dnsContext) (rc resultCode) { return resultCodeFinish } - if s.conf.OnDNSRequest != nil { - s.conf.OnDNSRequest(pctx) - } - - // Disable Mozilla DoH. - // - // See https://support.mozilla.org/en-US/kb/canary-domain-use-application-dnsnet. - if (qt == dns.TypeA || qt == dns.TypeAAAA) && q.Name == "use-application-dns.net." { + if (qt == dns.TypeA || qt == dns.TypeAAAA) && q.Name == mozillaFQDN { pctx.Res = s.genNXDomain(pctx.Req) return resultCodeFinish } - // Handle a reserved domain healthcheck.adguardhome.test. - // - // [Section 6.2 of RFC 6761] states that DNS Registries/Registrars must not - // grant requests to register test names in the normal way to any person or - // entity, making domain names under test. TLD free to use in internal - // purposes. - // - // [Section 6.2 of RFC 6761]: https://www.rfc-editor.org/rfc/rfc6761.html#section-6.2 - if q.Name == "healthcheck.adguardhome.test." { + if q.Name == healthcheckFQDN { // Generate a NODATA negative response to make nslookup exit with 0. pctx.Res = s.makeResponse(pctx.Req) @@ -213,11 +217,28 @@ func (s *Server) processInitial(dctx *dnsContext) (rc resultCode) { // Get the client-specific filtering settings. dctx.protectionEnabled, _ = s.UpdatedProtectionStatus() - dctx.setts = s.getClientRequestFilteringSettings(dctx) + dctx.setts = s.clientRequestFilteringSettings(dctx) return resultCodeSuccess } +// processClientIP sends the client IP address to s.addrProc, if needed. +func (s *Server) processClientIP(addr net.Addr) { + clientIP := netutil.NetAddrToAddrPort(addr).Addr() + if clientIP == (netip.Addr{}) { + log.Info("dnsforward: warning: bad client addr %q", addr) + + return + } + + // Do not assign s.addrProc to a local variable to then use, since this lock + // also serializes the closure of s.addrProc. + s.serverLock.RLock() + defer s.serverLock.RUnlock() + + s.addrProc.Process(clientIP) +} + func (s *Server) setTableHostToIP(t hostToIPTable) { s.tableHostToIPLock.Lock() defer s.tableHostToIPLock.Unlock() @@ -698,6 +719,18 @@ func (s *Server) processLocalPTR(dctx *dnsContext) (rc resultCode) { if s.conf.UsePrivateRDNS { s.recDetector.add(*pctx.Req) if err := s.localResolvers.Resolve(pctx); err != nil { + // Generate the server failure if the private upstream configuration + // is empty. + // + // TODO(e.burkov): Get rid of this crutch once the local resolvers + // logic is moved to the dnsproxy completely. + if errors.Is(err, upstream.ErrNoUpstreams) { + pctx.Res = s.genServerFailure(pctx.Req) + + // Do not even put into query log. + return resultCodeFinish + } + dctx.err = err return resultCodeError diff --git a/internal/dnsforward/dns_test.go b/internal/dnsforward/process_internal_test.go similarity index 87% rename from internal/dnsforward/dns_test.go rename to internal/dnsforward/process_internal_test.go index 1bcca756..a45ce245 100644 --- a/internal/dnsforward/dns_test.go +++ b/internal/dnsforward/process_internal_test.go @@ -12,6 +12,7 @@ import ( "github.com/AdguardTeam/dnsproxy/upstream" "github.com/AdguardTeam/golibs/netutil" "github.com/AdguardTeam/golibs/testutil" + "github.com/AdguardTeam/urlfilter/rules" "github.com/miekg/dns" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,6 +23,96 @@ const ( ddrTestFQDN = ddrTestDomainName + "." ) +func TestServer_ProcessInitial(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + target string + wantRCode rules.RCode + qType rules.RRType + aaaaDisabled bool + wantRC resultCode + }{{ + name: "success", + target: testQuestionTarget, + wantRCode: -1, + qType: dns.TypeA, + aaaaDisabled: false, + wantRC: resultCodeSuccess, + }, { + name: "aaaa_disabled", + target: testQuestionTarget, + wantRCode: dns.RcodeSuccess, + qType: dns.TypeAAAA, + aaaaDisabled: true, + wantRC: resultCodeFinish, + }, { + name: "aaaa_disabled_a", + target: testQuestionTarget, + wantRCode: -1, + qType: dns.TypeA, + aaaaDisabled: true, + wantRC: resultCodeSuccess, + }, { + name: "mozilla_canary", + target: mozillaFQDN, + wantRCode: dns.RcodeNameError, + qType: dns.TypeA, + aaaaDisabled: false, + wantRC: resultCodeFinish, + }, { + name: "adguardhome_healthcheck", + target: healthcheckFQDN, + wantRCode: dns.RcodeSuccess, + qType: dns.TypeA, + aaaaDisabled: false, + wantRC: resultCodeFinish, + }} + + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + c := ServerConfig{ + FilteringConfig: FilteringConfig{ + AAAADisabled: tc.aaaaDisabled, + EDNSClientSubnet: &EDNSClientSubnet{Enabled: false}, + }, + } + + s := createTestServer(t, &filtering.Config{}, c, nil) + + var gotAddr netip.Addr + s.addrProc = &aghtest.AddressProcessor{ + OnProcess: func(ip netip.Addr) { gotAddr = ip }, + OnClose: func() (err error) { panic("not implemented") }, + } + + dctx := &dnsContext{ + proxyCtx: &proxy.DNSContext{ + Req: createTestMessageWithType(tc.target, tc.qType), + Addr: testClientAddr, + RequestID: 1234, + }, + } + + gotRC := s.processInitial(dctx) + assert.Equal(t, tc.wantRC, gotRC) + assert.Equal(t, netutil.NetAddrToAddrPort(testClientAddr).Addr(), gotAddr) + + if tc.wantRCode > 0 { + gotResp := dctx.proxyCtx.Res + require.NotNil(t, gotResp) + + assert.Equal(t, tc.wantRCode, gotResp.Rcode) + } + }) + } +} + func TestServer_ProcessDDRQuery(t *testing.T) { dohSVCB := &dns.SVCB{ Priority: 1, @@ -64,7 +155,7 @@ func TestServer_ProcessDDRQuery(t *testing.T) { }{{ name: "pass_host", wantRes: resultCodeSuccess, - host: "example.net.", + host: testQuestionTarget, qtype: dns.TypeSVCB, ddrEnabled: true, portDoH: 8043, @@ -234,33 +325,33 @@ func TestServer_ProcessDetermineLocal(t *testing.T) { func TestServer_ProcessDHCPHosts_localRestriction(t *testing.T) { knownIP := netip.MustParseAddr("1.2.3.4") testCases := []struct { + wantIP netip.Addr name string host string - wantIP netip.Addr wantRes resultCode isLocalCli bool }{{ + wantIP: knownIP, name: "local_client_success", host: "example.lan", - wantIP: knownIP, wantRes: resultCodeSuccess, isLocalCli: true, }, { + wantIP: netip.Addr{}, name: "local_client_unknown_host", host: "wronghost.lan", - wantIP: netip.Addr{}, wantRes: resultCodeSuccess, isLocalCli: true, }, { + wantIP: netip.Addr{}, name: "external_client_known_host", host: "example.lan", - wantIP: netip.Addr{}, wantRes: resultCodeFinish, isLocalCli: false, }, { + wantIP: netip.Addr{}, name: "external_client_unknown_host", host: "wronghost.lan", - wantIP: netip.Addr{}, wantRes: resultCodeFinish, isLocalCli: false, }} @@ -332,52 +423,52 @@ func TestServer_ProcessDHCPHosts(t *testing.T) { knownIP := netip.MustParseAddr("1.2.3.4") testCases := []struct { + wantIP netip.Addr name string host string suffix string - wantIP netip.Addr wantRes resultCode qtyp uint16 }{{ + wantIP: netip.Addr{}, name: "success_external", host: examplecom, suffix: defaultLocalDomainSuffix, - wantIP: netip.Addr{}, wantRes: resultCodeSuccess, qtyp: dns.TypeA, }, { + wantIP: netip.Addr{}, name: "success_external_non_a", host: examplecom, suffix: defaultLocalDomainSuffix, - wantIP: netip.Addr{}, wantRes: resultCodeSuccess, qtyp: dns.TypeCNAME, }, { + wantIP: knownIP, name: "success_internal", host: examplelan, suffix: defaultLocalDomainSuffix, - wantIP: knownIP, wantRes: resultCodeSuccess, qtyp: dns.TypeA, }, { + wantIP: netip.Addr{}, name: "success_internal_unknown", host: "example-new.lan", suffix: defaultLocalDomainSuffix, - wantIP: netip.Addr{}, wantRes: resultCodeSuccess, qtyp: dns.TypeA, }, { + wantIP: netip.Addr{}, name: "success_internal_aaaa", host: examplelan, suffix: defaultLocalDomainSuffix, - wantIP: netip.Addr{}, wantRes: resultCodeSuccess, qtyp: dns.TypeAAAA, }, { + wantIP: knownIP, name: "success_custom_suffix", host: "example.custom", suffix: "custom", - wantIP: knownIP, wantRes: resultCodeSuccess, qtyp: dns.TypeA, }} @@ -560,10 +651,8 @@ func TestServer_ProcessLocalPTR_usingResolvers(t *testing.T) { var dnsCtx *dnsContext setup := func(use bool) { proxyCtx = &proxy.DNSContext{ - Addr: &net.TCPAddr{ - IP: net.IP{127, 0, 0, 1}, - }, - Req: createTestMessageWithType(reqAddr, dns.TypePTR), + Addr: testClientAddr, + Req: createTestMessageWithType(reqAddr, dns.TypePTR), } dnsCtx = &dnsContext{ proxyCtx: proxyCtx, diff --git a/internal/dnsforward/upstreams.go b/internal/dnsforward/upstreams.go index cbd92b36..ceec1cb7 100644 --- a/internal/dnsforward/upstreams.go +++ b/internal/dnsforward/upstreams.go @@ -42,11 +42,13 @@ func (s *Server) loadUpstreams() (upstreams []string, err error) { // prepareUpstreamSettings sets upstream DNS server settings. func (s *Server) prepareUpstreamSettings() (err error) { - // We're setting a customized set of RootCAs. The reason is that Go default - // mechanism of loading TLS roots does not always work properly on some - // routers so we're loading roots manually and pass it here. + // Use a customized set of RootCAs, because Go's default mechanism of + // loading TLS roots does not always work properly on some routers so we're + // loading roots manually and pass it here. // // See [aghtls.SystemRootCAs]. + // + // TODO(a.garipov): Investigate if that's true. upstream.RootCAs = s.conf.TLSv12Roots upstream.CipherSuites = s.conf.TLSCiphers @@ -190,7 +192,7 @@ func (s *Server) resolveUpstreamsWithHosts( // extractUpstreamHost returns the hostname of addr without port with an // assumption that any address passed here has already been successfully parsed -// by [upstream.AddressToUpstream]. This function eesentially mirrors the logic +// by [upstream.AddressToUpstream]. This function essentially mirrors the logic // of [upstream.AddressToUpstream], see TODO on [replaceUpstreamsWithHosts]. func extractUpstreamHost(addr string) (host string) { var err error diff --git a/internal/filtering/filter.go b/internal/filtering/filter.go index fa512c29..88e8a0fc 100644 --- a/internal/filtering/filter.go +++ b/internal/filtering/filter.go @@ -11,6 +11,7 @@ import ( "time" "github.com/AdguardTeam/AdGuardHome/internal/aghalg" + "github.com/AdguardTeam/AdGuardHome/internal/aghrenameio" "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" @@ -83,53 +84,53 @@ func (d *DNSFilter) filterSetProperties( filters = d.WhitelistFilters } - i := slices.IndexFunc(filters, func(filt FilterYAML) bool { return filt.URL == listURL }) + i := slices.IndexFunc(filters, func(flt FilterYAML) bool { return flt.URL == listURL }) if i == -1 { return false, errFilterNotExist } - filt := &filters[i] + flt := &filters[i] log.Debug( "filtering: set name to %q, url to %s, enabled to %t for filter %s", newList.Name, newList.URL, newList.Enabled, - filt.URL, + flt.URL, ) defer func(oldURL, oldName string, oldEnabled bool, oldUpdated time.Time, oldRulesCount int) { if err != nil { - filt.URL = oldURL - filt.Name = oldName - filt.Enabled = oldEnabled - filt.LastUpdated = oldUpdated - filt.RulesCount = oldRulesCount + flt.URL = oldURL + flt.Name = oldName + flt.Enabled = oldEnabled + flt.LastUpdated = oldUpdated + flt.RulesCount = oldRulesCount } - }(filt.URL, filt.Name, filt.Enabled, filt.LastUpdated, filt.RulesCount) + }(flt.URL, flt.Name, flt.Enabled, flt.LastUpdated, flt.RulesCount) - filt.Name = newList.Name + flt.Name = newList.Name - if filt.URL != newList.URL { + if flt.URL != newList.URL { if d.filterExistsLocked(newList.URL) { return false, errFilterExists } shouldRestart = true - filt.URL = newList.URL - filt.LastUpdated = time.Time{} - filt.unload() + flt.URL = newList.URL + flt.LastUpdated = time.Time{} + flt.unload() } - if filt.Enabled != newList.Enabled { - filt.Enabled = newList.Enabled + if flt.Enabled != newList.Enabled { + flt.Enabled = newList.Enabled shouldRestart = true } - if filt.Enabled { + if flt.Enabled { if shouldRestart { // Download the filter contents. - shouldRestart, err = d.update(filt) + shouldRestart, err = d.update(flt) } } else { // TODO(e.burkov): The validation of the contents of the new URL is @@ -137,7 +138,7 @@ func (d *DNSFilter) filterSetProperties( // possible to set a bad rules source, but the validation should still // kick in when the filter is enabled. Consider changing this behavior // to be stricter. - filt.unload() + flt.unload() } return shouldRestart, err @@ -250,24 +251,24 @@ func assignUniqueFilterID() int64 { // Sets up a timer that will be checking for filters updates periodically func (d *DNSFilter) periodicallyRefreshFilters() { const maxInterval = 1 * 60 * 60 - intval := 5 // use a dynamically increasing time interval + ivl := 5 // use a dynamically increasing time interval for { isNetErr, ok := false, false if d.FiltersUpdateIntervalHours != 0 { _, isNetErr, ok = d.tryRefreshFilters(true, true, false) if ok && !isNetErr { - intval = maxInterval + ivl = maxInterval } } if isNetErr { - intval *= 2 - if intval > maxInterval { - intval = maxInterval + ivl *= 2 + if ivl > maxInterval { + ivl = maxInterval } } - time.Sleep(time.Duration(intval) * time.Second) + time.Sleep(time.Duration(ivl) * time.Second) } } @@ -329,20 +330,20 @@ func (d *DNSFilter) refreshFiltersArray(filters *[]FilterYAML, force bool) (int, return 0, nil, nil, false } - nfail := 0 + failNum := 0 for i := range updateFilters { uf := &updateFilters[i] updated, err := d.update(uf) updateFlags = append(updateFlags, updated) if err != nil { - nfail++ - log.Info("filtering: updating filter from url %q: %s\n", uf.URL, err) + failNum++ + log.Error("filtering: updating filter from url %q: %s\n", uf.URL, err) continue } } - if nfail == len(updateFilters) { + if failNum == len(updateFilters) { return 0, nil, nil, true } @@ -464,48 +465,6 @@ func (d *DNSFilter) update(filter *FilterYAML) (b bool, err error) { return b, err } -// finalizeUpdate closes and gets rid of temporary file f with filter's content -// according to updated. It also saves new values of flt's name, rules number -// and checksum if succeeded. -func (d *DNSFilter) finalizeUpdate( - file *os.File, - flt *FilterYAML, - updated bool, - res *rulelist.ParseResult, -) (err error) { - tmpFileName := file.Name() - - // Close the file before renaming it because it's required on Windows. - // - // See https://github.com/adguardTeam/adGuardHome/issues/1553. - err = file.Close() - if err != nil { - return fmt.Errorf("closing temporary file: %w", err) - } - - if !updated { - log.Debug("filtering: filter %d from url %q has no changes, skipping", flt.ID, flt.URL) - - return os.Remove(tmpFileName) - } - - fltPath := flt.Path(d.DataDir) - - log.Info("filtering: saving contents of filter %d into %q", flt.ID, fltPath) - - // Don't use renameio or maybe packages, since those will require loading - // the whole filter content to the memory on Windows. - err = os.Rename(tmpFileName, fltPath) - if err != nil { - return errors.WithDeferred(err, os.Remove(tmpFileName)) - } - - flt.Name = aghalg.Coalesce(flt.Name, res.Title) - flt.checksum, flt.RulesCount = res.Checksum, res.RulesCount - - return nil -} - // updateIntl updates the flt rewriting it's actual file. It returns true if // the actual update has been performed. func (d *DNSFilter) updateIntl(flt *FilterYAML) (ok bool, err error) { @@ -513,63 +472,22 @@ func (d *DNSFilter) updateIntl(flt *FilterYAML) (ok bool, err error) { var res *rulelist.ParseResult - var tmpFile *os.File - tmpFile, err = os.CreateTemp(filepath.Join(d.DataDir, filterDir), "") - if err != nil { - return false, err - } - defer func() { - finErr := d.finalizeUpdate(tmpFile, flt, ok, res) - if ok && finErr == nil { - log.Info( - "filtering: updated filter %d: %d bytes, %d rules", - flt.ID, - res.BytesWritten, - res.RulesCount, - ) - - return - } - - err = errors.WithDeferred(err, finErr) - }() - // Change the default 0o600 permission to something more acceptable by end // users. // // See https://github.com/AdguardTeam/AdGuardHome/issues/3198. - if err = tmpFile.Chmod(0o644); err != nil { - return false, fmt.Errorf("changing file mode: %w", err) + tmpFile, err := aghrenameio.NewPendingFile(flt.Path(d.DataDir), 0o644) + if err != nil { + return false, err } + defer func() { err = d.finalizeUpdate(tmpFile, flt, res, err, ok) }() - var r io.Reader - if !filepath.IsAbs(flt.URL) { - var resp *http.Response - resp, err = d.HTTPClient.Get(flt.URL) - if err != nil { - log.Info("filtering: requesting filter from %q: %s, skipping", flt.URL, err) - - return false, err - } - defer func() { err = errors.WithDeferred(err, resp.Body.Close()) }() - - if resp.StatusCode != http.StatusOK { - log.Info("filtering got status code %d from %q, skipping", resp.StatusCode, flt.URL) - - return false, fmt.Errorf("got status code %d, want %d", resp.StatusCode, http.StatusOK) - } - - r = resp.Body - } else { - var f *os.File - f, err = os.Open(flt.URL) - if err != nil { - return false, fmt.Errorf("open file: %w", err) - } - defer func() { err = errors.WithDeferred(err, f.Close()) }() - - r = f + r, err := d.reader(flt.URL) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return false, err } + defer func() { err = errors.WithDeferred(err, r.Close()) }() bufPtr := d.bufPool.Get().(*[]byte) defer d.bufPool.Put(bufPtr) @@ -580,6 +498,78 @@ func (d *DNSFilter) updateIntl(flt *FilterYAML) (ok bool, err error) { return res.Checksum != flt.checksum && err == nil, err } +// finalizeUpdate closes and gets rid of temporary file f with filter's content +// according to updated. It also saves new values of flt's name, rules number +// and checksum if succeeded. +func (d *DNSFilter) finalizeUpdate( + file aghrenameio.PendingFile, + flt *FilterYAML, + res *rulelist.ParseResult, + returned error, + updated bool, +) (err error) { + id := flt.ID + if !updated { + if returned == nil { + log.Debug("filtering: filter %d from url %q has no changes, skipping", id, flt.URL) + } + + return errors.WithDeferred(returned, file.Cleanup()) + } + + log.Info("filtering: saving contents of filter %d into %q", id, flt.Path(d.DataDir)) + + err = file.CloseReplace() + if err != nil { + return fmt.Errorf("finalizing update: %w", err) + } + + rulesCount := res.RulesCount + log.Info("filtering: updated filter %d: %d bytes, %d rules", id, res.BytesWritten, rulesCount) + + flt.Name = aghalg.Coalesce(flt.Name, res.Title) + flt.checksum = res.Checksum + flt.RulesCount = rulesCount + + return nil +} + +// reader returns an io.ReadCloser reading filtering-rule list data form either +// a file on the filesystem or the filter's HTTP URL. +func (d *DNSFilter) reader(fltURL string) (r io.ReadCloser, err error) { + if !filepath.IsAbs(fltURL) { + r, err = d.readerFromURL(fltURL) + if err != nil { + return nil, fmt.Errorf("reading from url: %w", err) + } + + return r, nil + } + + r, err = os.Open(fltURL) + if err != nil { + return nil, fmt.Errorf("opening file: %w", err) + } + + return r, nil +} + +// readerFromURL returns an io.ReadCloser reading filtering-rule list data form +// the filter's URL. +func (d *DNSFilter) readerFromURL(fltURL string) (r io.ReadCloser, err error) { + resp, err := d.HTTPClient.Get(fltURL) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("got status code %d, want %d", resp.StatusCode, http.StatusOK) + } + + return resp.Body, nil +} + // loads filter contents from the file in dataDir func (d *DNSFilter) load(flt *FilterYAML) (err error) { fileName := flt.Path(d.DataDir) diff --git a/internal/filtering/filtering.go b/internal/filtering/filtering.go index f80b2220..b6249cee 100644 --- a/internal/filtering/filtering.go +++ b/internal/filtering/filtering.go @@ -943,7 +943,7 @@ func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) { d = &DNSFilter{ bufPool: &sync.Pool{ New: func() (buf any) { - bufVal := make([]byte, rulelist.MaxRuleLen) + bufVal := make([]byte, rulelist.DefaultRuleBufSize) return &bufVal }, diff --git a/internal/filtering/rulelist/parser.go b/internal/filtering/rulelist/parser.go index 0bf5dba8..24d19b9c 100644 --- a/internal/filtering/rulelist/parser.go +++ b/internal/filtering/rulelist/parser.go @@ -6,9 +6,9 @@ import ( "fmt" "hash/crc32" "io" - "unicode" "github.com/AdguardTeam/golibs/errors" + "golang.org/x/exp/slices" ) // Parser is a filtering-rule parser that collects data, such as the checksum @@ -48,19 +48,29 @@ type ParseResult struct { // nil. func (p *Parser) Parse(dst io.Writer, src io.Reader, buf []byte) (r *ParseResult, err error) { s := bufio.NewScanner(src) - s.Buffer(buf, MaxRuleLen) - lineIdx := 0 + // Don't use [DefaultRuleBufSize] as the maximum size, since some + // filtering-rule lists compressed by e.g. HostlistsCompiler can have very + // large lines. The buffer optimization still works for the more common + // case of reasonably-sized lines. + // + // See https://github.com/AdguardTeam/AdGuardHome/issues/6003. + s.Buffer(buf, bufio.MaxScanTokenSize) + + // Use a one-based index for lines and columns, since these errors end up in + // the frontend, and users are more familiar with one-based line and column + // indexes. + lineNum := 1 for s.Scan() { var n int - n, err = p.processLine(dst, s.Bytes(), lineIdx) + n, err = p.processLine(dst, s.Bytes(), lineNum) p.written += n if err != nil { // Don't wrap the error, because it's informative enough as is. return p.result(), err } - lineIdx++ + lineNum++ } r = p.result() @@ -81,7 +91,7 @@ func (p *Parser) result() (r *ParseResult) { // processLine processes a single line. It may write to dst, and if it does, n // is the number of bytes written. -func (p *Parser) processLine(dst io.Writer, line []byte, lineIdx int) (n int, err error) { +func (p *Parser) processLine(dst io.Writer, line []byte, lineNum int) (n int, err error) { trimmed := bytes.TrimSpace(line) if p.written == 0 && isHTMLLine(trimmed) { return 0, ErrHTML @@ -95,9 +105,10 @@ func (p *Parser) processLine(dst io.Writer, line []byte, lineIdx int) (n int, er } if badIdx != -1 { return 0, fmt.Errorf( - "line at index %d: character at index %d: non-printable character", - lineIdx, - badIdx+bytes.Index(line, trimmed), + "line %d: character %d: likely binary character %q", + lineNum, + badIdx+bytes.Index(line, trimmed)+1, + trimmed[badIdx], ) } @@ -130,41 +141,37 @@ func hasPrefixFold(b, prefix []byte) (ok bool) { } // parseLine returns true if the parsed line is a filtering rule. line is -// assumed to be trimmed of whitespace characters. nonPrintIdx is the index of -// the first non-printable character, if any; if there are none, nonPrintIdx is -// -1. +// assumed to be trimmed of whitespace characters. badIdx is the index of the +// first character that may indicate that this is a binary file, or -1 if none. // // A line is considered a rule if it's not empty, not a comment, and contains // only printable characters. -func parseLine(line []byte) (nonPrintIdx int, isRule bool) { +func parseLine(line []byte) (badIdx int, isRule bool) { if len(line) == 0 || line[0] == '#' || line[0] == '!' { return -1, false } - nonPrintIdx = bytes.IndexFunc(line, isNotPrintable) + badIdx = slices.IndexFunc(line, likelyBinary) - return nonPrintIdx, nonPrintIdx == -1 + return badIdx, badIdx == -1 } -// isNotPrintable returns true if r is not a printable character that can be -// contained in a filtering rule. -func isNotPrintable(r rune) (ok bool) { - // Tab isn't included into Unicode's graphic symbols, so include it here - // explicitly. - return r != '\t' && !unicode.IsGraphic(r) +// likelyBinary returns true if b is likely to be a byte from a binary file. +func likelyBinary(b byte) (ok bool) { + return (b < ' ' || b == 0x7f) && b != '\n' && b != '\r' && b != '\t' } // parseLineTitle is like [parseLine] but additionally looks for a title. line // is assumed to be trimmed of whitespace characters. -func (p *Parser) parseLineTitle(line []byte) (nonPrintIdx int, isRule bool) { +func (p *Parser) parseLineTitle(line []byte) (badIdx int, isRule bool) { if len(line) == 0 || line[0] == '#' { return -1, false } if line[0] != '!' { - nonPrintIdx = bytes.IndexFunc(line, isNotPrintable) + badIdx = slices.IndexFunc(line, likelyBinary) - return nonPrintIdx, nonPrintIdx == -1 + return badIdx, badIdx == -1 } const titlePattern = "! Title: " diff --git a/internal/filtering/rulelist/parser_test.go b/internal/filtering/rulelist/parser_test.go index 5e912988..5554458d 100644 --- a/internal/filtering/rulelist/parser_test.go +++ b/internal/filtering/rulelist/parser_test.go @@ -6,10 +6,10 @@ import ( "strings" "testing" - "github.com/AdguardTeam/AdGuardHome/internal/aghtest" "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/testutil" + "github.com/AdguardTeam/golibs/testutil/fakeio" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -17,6 +17,9 @@ import ( func TestParser_Parse(t *testing.T) { t.Parallel() + longRule := strings.Repeat("a", rulelist.DefaultRuleBufSize+1) + "\n" + tooLongRule := strings.Repeat("a", bufio.MaxScanTokenSize+1) + "\n" + testCases := []struct { name string in string @@ -74,26 +77,42 @@ func TestParser_Parse(t *testing.T) { wantTitle: "Test Title", wantRulesNum: 1, wantWritten: len(testRuleTextBlocked), + }, { + name: "cosmetic_with_zwnj", + in: testRuleTextCosmetic, + wantDst: testRuleTextCosmetic, + wantErrMsg: "", + wantTitle: "", + wantRulesNum: 1, + wantWritten: len(testRuleTextCosmetic), }, { name: "bad_char", in: "! Title: Test Title \n" + testRuleTextBlocked + ">>>\x7F<<<", wantDst: testRuleTextBlocked, - wantErrMsg: "line at index 2: " + - "character at index 3: " + - "non-printable character", + wantErrMsg: "line 3: " + + "character 4: " + + "likely binary character '\\x7f'", wantTitle: "Test Title", wantRulesNum: 1, wantWritten: len(testRuleTextBlocked), }, { name: "too_long", - in: strings.Repeat("a", rulelist.MaxRuleLen+1), + in: tooLongRule, wantDst: "", - wantErrMsg: "scanning filter contents: " + bufio.ErrTooLong.Error(), + wantErrMsg: "scanning filter contents: bufio.Scanner: token too long", wantTitle: "", wantRulesNum: 0, wantWritten: 0, + }, { + name: "longer_than_default", + in: longRule, + wantDst: longRule, + wantErrMsg: "", + wantTitle: "", + wantRulesNum: 1, + wantWritten: len(longRule), }, { name: "bad_tab_and_comment", in: testRuleTextBadTab, @@ -118,7 +137,7 @@ func TestParser_Parse(t *testing.T) { t.Parallel() dst := &bytes.Buffer{} - buf := make([]byte, rulelist.MaxRuleLen) + buf := make([]byte, rulelist.DefaultRuleBufSize) p := rulelist.NewParser() r, err := p.Parse(dst, strings.NewReader(tc.in), buf) @@ -140,12 +159,12 @@ func TestParser_Parse(t *testing.T) { func TestParser_Parse_writeError(t *testing.T) { t.Parallel() - dst := &aghtest.Writer{ + dst := &fakeio.Writer{ OnWrite: func(b []byte) (n int, err error) { return 1, errors.Error("test error") }, } - buf := make([]byte, rulelist.MaxRuleLen) + buf := make([]byte, rulelist.DefaultRuleBufSize) p := rulelist.NewParser() r, err := p.Parse(dst, strings.NewReader(testRuleTextBlocked), buf) @@ -165,7 +184,7 @@ func TestParser_Parse_checksums(t *testing.T) { "# Another comment.\n" ) - buf := make([]byte, rulelist.MaxRuleLen) + buf := make([]byte, rulelist.DefaultRuleBufSize) p := rulelist.NewParser() r, err := p.Parse(&bytes.Buffer{}, strings.NewReader(withoutComments), buf) @@ -192,7 +211,7 @@ var ( func BenchmarkParser_Parse(b *testing.B) { dst := &bytes.Buffer{} src := strings.NewReader(strings.Repeat(testRuleTextBlocked, 1000)) - buf := make([]byte, rulelist.MaxRuleLen) + buf := make([]byte, rulelist.DefaultRuleBufSize) p := rulelist.NewParser() b.ReportAllocs() @@ -204,6 +223,14 @@ func BenchmarkParser_Parse(b *testing.B) { require.NoError(b, errSink) require.NotNil(b, resSink) + + // Most recent result, on a ThinkPad X13 with a Ryzen Pro 7 CPU: + // + // goos: linux + // goarch: amd64 + // pkg: github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist + // cpu: AMD Ryzen 7 PRO 4750U with Radeon Graphics + // BenchmarkParser_Parse-16 100000000 128.0 ns/op 48 B/op 1 allocs/op } func FuzzParser_Parse(f *testing.F) { @@ -215,15 +242,17 @@ func FuzzParser_Parse(f *testing.F) { "! Comment", "! Title ", "! Title XXX", + testRuleTextBadTab, + testRuleTextBlocked, + testRuleTextCosmetic, testRuleTextEtcHostsTab, testRuleTextHTML, - testRuleTextBlocked, - testRuleTextBadTab, "1.2.3.4", "1.2.3.4 etc-hosts.example", ">>>\x00<<<", ">>>\x7F<<<", - strings.Repeat("a", n+1), + strings.Repeat("a", rulelist.DefaultRuleBufSize+1), + strings.Repeat("a", bufio.MaxScanTokenSize+1), } for _, tc := range testCases { diff --git a/internal/filtering/rulelist/rulelist.go b/internal/filtering/rulelist/rulelist.go index 1a6236c5..464650a1 100644 --- a/internal/filtering/rulelist/rulelist.go +++ b/internal/filtering/rulelist/rulelist.go @@ -4,8 +4,6 @@ // TODO(a.garipov): Expand. package rulelist -// MaxRuleLen is the maximum length of a line with a filtering rule, in bytes. -// -// TODO(a.garipov): Consider changing this to a rune length, like AdGuardDNS -// does. -const MaxRuleLen = 1024 +// DefaultRuleBufSize is the default length of a buffer used to read a line with +// a filtering rule, in bytes. +const DefaultRuleBufSize = 1024 diff --git a/internal/filtering/rulelist/rulelist_test.go b/internal/filtering/rulelist/rulelist_test.go index 0c3a3b84..aec6f33b 100644 --- a/internal/filtering/rulelist/rulelist_test.go +++ b/internal/filtering/rulelist/rulelist_test.go @@ -7,8 +7,13 @@ const testTimeout = 1 * time.Second // Common texts for tests. const ( - testRuleTextHTML = "\n" - testRuleTextBlocked = "||blocked.example^\n" testRuleTextBadTab = "||bad-tab-and-comment.example^\t# A comment.\n" + testRuleTextBlocked = "||blocked.example^\n" testRuleTextEtcHostsTab = "0.0.0.0 tab..example^\t# A comment.\n" + testRuleTextHTML = "\n" + + // testRuleTextCosmetic is a cosmetic rule with a zero-width non-joiner. + // + // See https://github.com/AdguardTeam/AdGuardHome/issues/6003. + testRuleTextCosmetic = "||cosmetic.example## :has-text(/\u200c/i)\n" ) diff --git a/internal/filtering/safesearch/safesearch_internal_test.go b/internal/filtering/safesearch/safesearch_internal_test.go index c87a9ad5..909265ee 100644 --- a/internal/filtering/safesearch/safesearch_internal_test.go +++ b/internal/filtering/safesearch/safesearch_internal_test.go @@ -89,37 +89,34 @@ func TestSafeSearchCacheGoogle(t *testing.T) { assert.False(t, res.IsFiltered) assert.Empty(t, res.Rules) - resolver := &aghtest.TestResolver{} + resolver := &aghtest.Resolver{ + OnLookupIP: func(_ context.Context, _, host string) (ips []net.IP, err error) { + ip4, ip6 := aghtest.HostToIPs(host) + + return []net.IP{ip4, ip6}, nil + }, + } + ss = newForTest(t, defaultSafeSearchConf) ss.resolver = resolver // Lookup for safesearch domain. rewrite := ss.searchHost(domain, testQType) - ips, err := resolver.LookupIP(context.Background(), "ip", rewrite.NewCNAME) - require.NoError(t, err) - - var foundIP net.IP - for _, ip := range ips { - if ip.To4() != nil { - foundIP = ip - - break - } - } + wantIP, _ := aghtest.HostToIPs(rewrite.NewCNAME) res, err = ss.CheckHost(domain, testQType) require.NoError(t, err) require.Len(t, res.Rules, 1) - assert.True(t, res.Rules[0].IP.Equal(foundIP)) + assert.True(t, res.Rules[0].IP.Equal(wantIP)) // Check cache. cachedValue, isFound := ss.getCachedResult(domain, testQType) require.True(t, isFound) require.Len(t, cachedValue.Rules, 1) - assert.True(t, cachedValue.Rules[0].IP.Equal(foundIP)) + assert.True(t, cachedValue.Rules[0].IP.Equal(wantIP)) } const googleHost = "www.google.com" diff --git a/internal/filtering/safesearch/safesearch_test.go b/internal/filtering/safesearch/safesearch_test.go index 12860c5d..c62dd6e4 100644 --- a/internal/filtering/safesearch/safesearch_test.go +++ b/internal/filtering/safesearch/safesearch_test.go @@ -92,8 +92,15 @@ func TestDefault_CheckHost_yandexAAAA(t *testing.T) { } func TestDefault_CheckHost_google(t *testing.T) { - resolver := &aghtest.TestResolver{} - ip, _ := resolver.HostToIPs("forcesafesearch.google.com") + resolver := &aghtest.Resolver{ + OnLookupIP: func(_ context.Context, _, host string) (ips []net.IP, err error) { + ip4, ip6 := aghtest.HostToIPs(host) + + return []net.IP{ip4, ip6}, nil + }, + } + + wantIP, _ := aghtest.HostToIPs("forcesafesearch.google.com") conf := testConf conf.CustomResolver = resolver @@ -119,7 +126,7 @@ func TestDefault_CheckHost_google(t *testing.T) { require.Len(t, res.Rules, 1) - assert.Equal(t, ip, res.Rules[0].IP) + assert.Equal(t, wantIP, res.Rules[0].IP) assert.EqualValues(t, filtering.SafeSearchListID, res.Rules[0].FilterListID) }) } diff --git a/internal/filtering/servicelist.go b/internal/filtering/servicelist.go index 56988193..305baa52 100644 --- a/internal/filtering/servicelist.go +++ b/internal/filtering/servicelist.go @@ -1505,7 +1505,6 @@ var blockedServices = []blockedService{{ "||aus.social^", "||awscommunity.social^", "||climatejustice.social^", - "||cupoftea.social^", "||cyberplace.social^", "||defcon.social^", "||det.social^", @@ -1589,6 +1588,7 @@ var blockedServices = []blockedService{{ "||techhub.social^", "||theblower.au^", "||tkz.one^", + "||todon.eu^", "||toot.aquilenet.fr^", "||toot.community^", "||toot.funami.tech^", @@ -1661,6 +1661,7 @@ var blockedServices = []blockedService{{ "||nintendo.jp^", "||nintendo.net^", "||nintendo.nl^", + "||nintendo.pt^", "||nintendoswitch.cn^", "||nintendowifi.net^", }, @@ -2160,6 +2161,20 @@ var blockedServices = []blockedService{{ Rules: []string{ "||voot.com^", }, +}, { + ID: "wargaming", + Name: "Wargaming", + IconSVG: []byte(""), + Rules: []string{ + "||wargaming.com^", + "||wargaming.net^", + "||wgcdn.co^", + "||wgcrowd.io^", + "||worldoftanks.com^", + "||worldofwarplanes.com^", + "||worldofwarships.eu^", + "||wotblitz.com^", + }, }, { ID: "wechat", Name: "WeChat", diff --git a/internal/home/clients.go b/internal/home/clients.go index 1cd3add0..049710bc 100644 --- a/internal/home/clients.go +++ b/internal/home/clients.go @@ -10,6 +10,7 @@ import ( "time" "github.com/AdguardTeam/AdGuardHome/internal/aghnet" + "github.com/AdguardTeam/AdGuardHome/internal/client" "github.com/AdguardTeam/AdGuardHome/internal/dhcpd" "github.com/AdguardTeam/AdGuardHome/internal/dhcpsvc" "github.com/AdguardTeam/AdGuardHome/internal/dnsforward" @@ -141,7 +142,7 @@ func (clients *clientsContainer) handleHostsUpdates() { } } -// webHandlersRegistered prevents a [clientsContainer] from regisering its web +// webHandlersRegistered prevents a [clientsContainer] from registering its web // handlers more than once. // // TODO(a.garipov): Refactor HTTP handler registration logic. @@ -743,11 +744,9 @@ func (clients *clientsContainer) Update(prev, c *Client) (err error) { return nil } -// setWHOISInfo sets the WHOIS information for a client. +// setWHOISInfo sets the WHOIS information for a client. clients.lock is +// expected to be locked. func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *whois.Info) { - clients.lock.Lock() - defer clients.lock.Unlock() - _, ok := clients.findLocked(ip.String()) if ok { log.Debug("clients: client for %s is already created, ignore whois info", ip) @@ -774,9 +773,11 @@ func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *whois.Info) { rc.WHOIS = wi } -// AddHost adds a new IP-hostname pairing. The priorities of the sources are +// addHost adds a new IP-hostname pairing. The priorities of the sources are // taken into account. ok is true if the pairing was added. -func (clients *clientsContainer) AddHost( +// +// TODO(a.garipov): Only used in internal tests. Consider removing. +func (clients *clientsContainer) addHost( ip netip.Addr, host string, src clientSource, @@ -787,6 +788,32 @@ func (clients *clientsContainer) AddHost( return clients.addHostLocked(ip, host, src) } +// type check +var _ client.AddressUpdater = (*clientsContainer)(nil) + +// UpdateAddress implements the [client.AddressUpdater] interface for +// *clientsContainer +func (clients *clientsContainer) UpdateAddress(ip netip.Addr, host string, info *whois.Info) { + // Common fast path optimization. + if host == "" && info == nil { + return + } + + clients.lock.Lock() + defer clients.lock.Unlock() + + if host != "" { + ok := clients.addHostLocked(ip, host, ClientSourceRDNS) + if !ok { + log.Debug("clients: host for client %q already set with higher priority source", ip) + } + } + + if info != nil { + clients.setWHOISInfo(ip, info) + } +} + // addHostLocked adds a new IP-hostname pairing. clients.lock is expected to be // locked. func (clients *clientsContainer) addHostLocked( diff --git a/internal/home/clients_test.go b/internal/home/clients_internal_test.go similarity index 94% rename from internal/home/clients_test.go rename to internal/home/clients_internal_test.go index 9ad819ec..d3ff2a57 100644 --- a/internal/home/clients_test.go +++ b/internal/home/clients_internal_test.go @@ -168,13 +168,13 @@ func TestClients(t *testing.T) { t.Run("addhost_success", func(t *testing.T) { ip := netip.MustParseAddr("1.1.1.1") - ok := clients.AddHost(ip, "host", ClientSourceARP) + ok := clients.addHost(ip, "host", ClientSourceARP) assert.True(t, ok) - ok = clients.AddHost(ip, "host2", ClientSourceARP) + ok = clients.addHost(ip, "host2", ClientSourceARP) assert.True(t, ok) - ok = clients.AddHost(ip, "host3", ClientSourceHostsFile) + ok = clients.addHost(ip, "host3", ClientSourceHostsFile) assert.True(t, ok) assert.Equal(t, clients.clientSource(ip), ClientSourceHostsFile) @@ -182,18 +182,18 @@ func TestClients(t *testing.T) { t.Run("dhcp_replaces_arp", func(t *testing.T) { ip := netip.MustParseAddr("1.2.3.4") - ok := clients.AddHost(ip, "from_arp", ClientSourceARP) + ok := clients.addHost(ip, "from_arp", ClientSourceARP) assert.True(t, ok) assert.Equal(t, clients.clientSource(ip), ClientSourceARP) - ok = clients.AddHost(ip, "from_dhcp", ClientSourceDHCP) + ok = clients.addHost(ip, "from_dhcp", ClientSourceDHCP) assert.True(t, ok) assert.Equal(t, clients.clientSource(ip), ClientSourceDHCP) }) t.Run("addhost_fail", func(t *testing.T) { ip := netip.MustParseAddr("1.1.1.1") - ok := clients.AddHost(ip, "host1", ClientSourceRDNS) + ok := clients.addHost(ip, "host1", ClientSourceRDNS) assert.False(t, ok) }) } @@ -216,7 +216,7 @@ func TestClientsWHOIS(t *testing.T) { t.Run("existing_auto-client", func(t *testing.T) { ip := netip.MustParseAddr("1.1.1.1") - ok := clients.AddHost(ip, "host", ClientSourceRDNS) + ok := clients.addHost(ip, "host", ClientSourceRDNS) assert.True(t, ok) clients.setWHOISInfo(ip, whois) @@ -259,7 +259,7 @@ func TestClientsAddExisting(t *testing.T) { assert.True(t, ok) // Now add an auto-client with the same IP. - ok = clients.AddHost(ip, "test", ClientSourceRDNS) + ok = clients.addHost(ip, "test", ClientSourceRDNS) assert.True(t, ok) }) diff --git a/internal/home/config.go b/internal/home/config.go index 60cbe621..361fdebb 100644 --- a/internal/home/config.go +++ b/internal/home/config.go @@ -20,7 +20,7 @@ import ( "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/timeutil" - "github.com/google/renameio/maybe" + "github.com/google/renameio/v2/maybe" "golang.org/x/exp/slices" yaml "gopkg.in/yaml.v3" ) @@ -590,7 +590,13 @@ func (c *configuration) write() (err error) { s.WriteDiskConfig(&c) dns := &config.DNS dns.FilteringConfig = c - dns.LocalPTRResolvers, config.Clients.Sources.RDNS, dns.UsePrivateRDNS = s.RDNSSettings() + + dns.LocalPTRResolvers = s.LocalPTRResolvers() + + addrProcConf := s.AddrProcConfig() + config.Clients.Sources.RDNS = addrProcConf.UseRDNS + config.Clients.Sources.WHOIS = addrProcConf.UseWHOIS + dns.UsePrivateRDNS = addrProcConf.UsePrivateRDNS } if Context.dhcpServer != nil { diff --git a/internal/home/control.go b/internal/home/control.go index 48afcf71..a2414b35 100644 --- a/internal/home/control.go +++ b/internal/home/control.go @@ -176,12 +176,16 @@ func handleStatus(w http.ResponseWriter, r *http.Request) { // ------------------------ // registration of handlers // ------------------------ -func registerControlHandlers() { +func registerControlHandlers(web *webAPI) { + Context.mux.HandleFunc( + "/control/version.json", + postInstall(optionalAuth(web.handleVersionJSON)), + ) + httpRegister(http.MethodPost, "/control/update", web.handleUpdate) + httpRegister(http.MethodGet, "/control/status", handleStatus) httpRegister(http.MethodPost, "/control/i18n/change_language", handleI18nChangeLanguage) httpRegister(http.MethodGet, "/control/i18n/current_language", handleI18nCurrentLanguage) - Context.mux.HandleFunc("/control/version.json", postInstall(optionalAuth(handleVersionJSON))) - httpRegister(http.MethodPost, "/control/update", handleUpdate) httpRegister(http.MethodGet, "/control/profile", handleGetProfile) httpRegister(http.MethodPut, "/control/profile/update", handlePutProfile) diff --git a/internal/home/controlinstall.go b/internal/home/controlinstall.go index c9503fa0..a5be3354 100644 --- a/internal/home/controlinstall.go +++ b/internal/home/controlinstall.go @@ -448,7 +448,7 @@ func (web *webAPI) handleInstallConfigure(w http.ResponseWriter, r *http.Request web.conf.BindHost = req.Web.IP web.conf.BindPort = req.Web.Port - registerControlHandlers() + registerControlHandlers(web) aghhttp.OK(w) if f, ok := w.(http.Flusher); ok { diff --git a/internal/home/controlupdate.go b/internal/home/controlupdate.go index 434286ba..5238134c 100644 --- a/internal/home/controlupdate.go +++ b/internal/home/controlupdate.go @@ -29,9 +29,9 @@ type temporaryError interface { // handleVersionJSON is the handler for the POST /control/version.json HTTP API. // // TODO(a.garipov): Find out if this API used with a GET method by anyone. -func handleVersionJSON(w http.ResponseWriter, r *http.Request) { +func (web *webAPI) handleVersionJSON(w http.ResponseWriter, r *http.Request) { resp := &versionResponse{} - if Context.disableUpdate { + if web.conf.disableUpdate { resp.Disabled = true _ = aghhttp.WriteJSONResponse(w, r, resp) @@ -52,7 +52,7 @@ func handleVersionJSON(w http.ResponseWriter, r *http.Request) { } } - err = requestVersionInfo(resp, req.Recheck) + err = web.requestVersionInfo(resp, req.Recheck) if err != nil { // Don't wrap the error, because it's informative enough as is. aghhttp.Error(r, w, http.StatusBadGateway, "%s", err) @@ -73,9 +73,10 @@ func handleVersionJSON(w http.ResponseWriter, r *http.Request) { // requestVersionInfo sets the VersionInfo field of resp if it can reach the // update server. -func requestVersionInfo(resp *versionResponse, recheck bool) (err error) { +func (web *webAPI) requestVersionInfo(resp *versionResponse, recheck bool) (err error) { + updater := web.conf.updater for i := 0; i != 3; i++ { - resp.VersionInfo, err = Context.updater.VersionInfo(recheck) + resp.VersionInfo, err = updater.VersionInfo(recheck) if err != nil { var terr temporaryError if errors.As(err, &terr) && terr.Temporary() { @@ -95,7 +96,7 @@ func requestVersionInfo(resp *versionResponse, recheck bool) (err error) { } if err != nil { - vcu := Context.updater.VersionCheckURL() + vcu := updater.VersionCheckURL() return fmt.Errorf("getting version info from %s: %w", vcu, err) } @@ -104,8 +105,9 @@ func requestVersionInfo(resp *versionResponse, recheck bool) (err error) { } // handleUpdate performs an update to the latest available version procedure. -func handleUpdate(w http.ResponseWriter, r *http.Request) { - if Context.updater.NewVersion() == "" { +func (web *webAPI) handleUpdate(w http.ResponseWriter, r *http.Request) { + updater := web.conf.updater + if updater.NewVersion() == "" { aghhttp.Error(r, w, http.StatusBadRequest, "/update request isn't allowed now") return @@ -122,7 +124,7 @@ func handleUpdate(w http.ResponseWriter, r *http.Request) { return } - err = Context.updater.Update(false) + err = updater.Update(false) if err != nil { aghhttp.Error(r, w, http.StatusInternalServerError, "%s", err) @@ -137,7 +139,7 @@ func handleUpdate(w http.ResponseWriter, r *http.Request) { // The background context is used because the underlying functions wrap it // with timeout and shut down the server, which handles current request. It // also should be done in a separate goroutine for the same reason. - go finishUpdate(context.Background(), execPath) + go finishUpdate(context.Background(), execPath, web.conf.runningAsService) } // versionResponse is the response for /control/version.json endpoint. @@ -178,7 +180,7 @@ func tlsConfUsesPrivilegedPorts(c *tlsConfigSettings) (ok bool) { } // finishUpdate completes an update procedure. -func finishUpdate(ctx context.Context, execPath string) { +func finishUpdate(ctx context.Context, execPath string, runningAsService bool) { var err error log.Info("stopping all tasks") @@ -187,7 +189,7 @@ func finishUpdate(ctx context.Context, execPath string) { cleanupAlways() if runtime.GOOS == "windows" { - if Context.runningAsService { + if runningAsService { // NOTE: We can't restart the service via "kardianos/service" // package, because it kills the process first we can't start a new // instance, because Windows doesn't allow it. diff --git a/internal/home/dns.go b/internal/home/dns.go index fbbda423..87cb70fd 100644 --- a/internal/home/dns.go +++ b/internal/home/dns.go @@ -13,14 +13,12 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghalg" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/aghnet" + "github.com/AdguardTeam/AdGuardHome/internal/client" "github.com/AdguardTeam/AdGuardHome/internal/dhcpd" "github.com/AdguardTeam/AdGuardHome/internal/dnsforward" "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/AdGuardHome/internal/querylog" - "github.com/AdguardTeam/AdGuardHome/internal/rdns" "github.com/AdguardTeam/AdGuardHome/internal/stats" - "github.com/AdguardTeam/AdGuardHome/internal/whois" - "github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/netutil" @@ -135,7 +133,7 @@ func initDNSServer( return fmt.Errorf("preparing set of private subnets: %w", err) } - p := dnsforward.DNSCreateParams{ + Context.dnsServer, err = dnsforward.NewServer(dnsforward.DNSCreateParams{ DNSFilter: filters, Stats: sts, QueryLog: qlog, @@ -143,9 +141,7 @@ func initDNSServer( Anonymizer: anonymizer, LocalDomain: config.DHCP.LocalDomainName, DHCPServer: dhcpSrv, - } - - Context.dnsServer, err = dnsforward.NewServer(p) + }) if err != nil { closeDNSServer() @@ -154,134 +150,23 @@ func initDNSServer( Context.clients.dnsServer = Context.dnsServer - dnsConf, err := generateServerConfig(tlsConf, httpReg) + dnsConf, err := newServerConfig(tlsConf, httpReg) if err != nil { closeDNSServer() - return fmt.Errorf("generateServerConfig: %w", err) + return fmt.Errorf("newServerConfig: %w", err) } - err = Context.dnsServer.Prepare(&dnsConf) + err = Context.dnsServer.Prepare(dnsConf) if err != nil { closeDNSServer() return fmt.Errorf("dnsServer.Prepare: %w", err) } - initRDNS() - initWHOIS() - return nil } -const ( - // defaultQueueSize is the size of queue of IPs for rDNS and WHOIS - // processing. - defaultQueueSize = 255 - - // defaultCacheSize is the maximum size of the cache for rDNS and WHOIS - // processing. It must be greater than zero. - defaultCacheSize = 10_000 - - // defaultIPTTL is the Time to Live duration for IP addresses cached by - // rDNS and WHOIS. - defaultIPTTL = 1 * time.Hour -) - -// initRDNS initializes the rDNS. -func initRDNS() { - Context.rdnsCh = make(chan netip.Addr, defaultQueueSize) - - // TODO(s.chzhen): Add ability to disable it on dns server configuration - // update in [dnsforward] package. - r := rdns.New(&rdns.Config{ - Exchanger: Context.dnsServer, - CacheSize: defaultCacheSize, - CacheTTL: defaultIPTTL, - }) - - go processRDNS(r) -} - -// processRDNS processes reverse DNS lookup queries. It is intended to be used -// as a goroutine. -func processRDNS(r rdns.Interface) { - defer log.OnPanic("rdns") - - for ip := range Context.rdnsCh { - ok := Context.dnsServer.ShouldResolveClient(ip) - if !ok { - continue - } - - host, changed := r.Process(ip) - if host == "" || !changed { - continue - } - - ok = Context.clients.AddHost(ip, host, ClientSourceRDNS) - if ok { - continue - } - - log.Debug( - "dns: can't set rdns info for client %q: already set with higher priority source", - ip, - ) - } -} - -// initWHOIS initializes the WHOIS. -// -// TODO(s.chzhen): Consider making configurable. -func initWHOIS() { - const ( - // defaultTimeout is the timeout for WHOIS requests. - defaultTimeout = 5 * time.Second - - // defaultMaxConnReadSize is an upper limit in bytes for reading from - // net.Conn. - defaultMaxConnReadSize = 64 * 1024 - - // defaultMaxRedirects is the maximum redirects count. - defaultMaxRedirects = 5 - - // defaultMaxInfoLen is the maximum length of whois.Info fields. - defaultMaxInfoLen = 250 - ) - - Context.whoisCh = make(chan netip.Addr, defaultQueueSize) - - var w whois.Interface - - if config.Clients.Sources.WHOIS { - w = whois.New(&whois.Config{ - DialContext: customDialContext, - ServerAddr: whois.DefaultServer, - Port: whois.DefaultPort, - Timeout: defaultTimeout, - CacheSize: defaultCacheSize, - MaxConnReadSize: defaultMaxConnReadSize, - MaxRedirects: defaultMaxRedirects, - MaxInfoLen: defaultMaxInfoLen, - CacheTTL: defaultIPTTL, - }) - } else { - w = whois.Empty{} - } - - go func() { - defer log.OnPanic("whois") - - for ip := range Context.whoisCh { - info, changed := w.Process(context.Background(), ip) - if info != nil && changed { - Context.clients.setWHOISInfo(ip, info) - } - } - }() -} - // parseSubnetSet parses a slice of subnets. If the slice is empty, it returns // a subnet set that matches all locally served networks, see // [netutil.IsLocallyServed]. @@ -312,17 +197,6 @@ func isRunning() bool { return Context.dnsServer != nil && Context.dnsServer.IsRunning() } -func onDNSRequest(pctx *proxy.DNSContext) { - ip := netutil.NetAddrToAddrPort(pctx.Addr).Addr() - if ip == (netip.Addr{}) { - // This would be quite weird if we get here. - return - } - - Context.rdnsCh <- ip - Context.whoisCh <- ip -} - func ipsToTCPAddrs(ips []netip.Addr, port int) (tcpAddrs []*net.TCPAddr) { if ips == nil { return nil @@ -349,23 +223,41 @@ func ipsToUDPAddrs(ips []netip.Addr, port int) (udpAddrs []*net.UDPAddr) { return udpAddrs } -func generateServerConfig( +func newServerConfig( tlsConf *tlsConfigSettings, httpReg aghhttp.RegisterFunc, -) (newConf dnsforward.ServerConfig, err error) { +) (newConf *dnsforward.ServerConfig, err error) { dnsConf := config.DNS hosts := aghalg.CoalesceSlice(dnsConf.BindHosts, []netip.Addr{netutil.IPv4Localhost()}) - newConf = dnsforward.ServerConfig{ + + newConf = &dnsforward.ServerConfig{ UDPListenAddrs: ipsToUDPAddrs(hosts, dnsConf.Port), TCPListenAddrs: ipsToTCPAddrs(hosts, dnsConf.Port), FilteringConfig: dnsConf.FilteringConfig, ConfigModified: onConfigModified, HTTPRegister: httpReg, - OnDNSRequest: onDNSRequest, UseDNS64: config.DNS.UseDNS64, DNS64Prefixes: config.DNS.DNS64Prefixes, } + var initialAddresses []netip.Addr + // Context.stats may be nil here if initDNSServer is called from + // [cmdlineUpdate]. + if sts := Context.stats; sts != nil { + const initialClientsNum = 100 + initialAddresses = Context.stats.TopClientsIP(initialClientsNum) + } + + // Do not set DialContext, PrivateSubnets, and UsePrivateRDNS, because they + // are set by [dnsforward.Server.Prepare]. + newConf.AddrProcConf = &client.DefaultAddrProcConfig{ + Exchanger: Context.dnsServer, + AddressUpdater: &Context.clients, + InitialAddresses: initialAddresses, + UseRDNS: config.Clients.Sources.RDNS, + UseWHOIS: config.Clients.Sources.WHOIS, + } + if tlsConf.Enabled { newConf.TLSConfig = tlsConf.TLSConfig newConf.TLSConfig.ServerName = tlsConf.ServerName @@ -385,9 +277,9 @@ func generateServerConfig( if tlsConf.PortDNSCrypt != 0 { newConf.DNSCryptConfig, err = newDNSCrypt(hosts, *tlsConf) if err != nil { - // Don't wrap the error, because it's already - // wrapped by newDNSCrypt. - return dnsforward.ServerConfig{}, err + // Don't wrap the error, because it's already wrapped by + // newDNSCrypt. + return nil, err } } } @@ -401,7 +293,6 @@ func generateServerConfig( newConf.LocalPTRResolvers = dnsConf.LocalPTRResolvers newConf.UpstreamTimeout = dnsConf.UpstreamTimeout.Duration - newConf.ResolveClients = config.Clients.Sources.RDNS newConf.UsePrivateRDNS = dnsConf.UsePrivateRDNS newConf.ServeHTTP3 = dnsConf.ServeHTTP3 newConf.UseHTTP3Upstreams = dnsConf.UseHTTP3Upstreams @@ -556,27 +447,19 @@ func startDNSServer() error { Context.stats.Start() Context.queryLog.Start() - const topClientsNumber = 100 // the number of clients to get - for _, ip := range Context.stats.TopClientsIP(topClientsNumber) { - Context.rdnsCh <- ip - Context.whoisCh <- ip - } - return nil } func reconfigureDNSServer() (err error) { - var newConf dnsforward.ServerConfig - tlsConf := &tlsConfigSettings{} Context.tls.WriteDiskConfig(tlsConf) - newConf, err = generateServerConfig(tlsConf, httpRegister) + newConf, err := newServerConfig(tlsConf, httpRegister) if err != nil { return fmt.Errorf("generating forwarding dns server config: %w", err) } - err = Context.dnsServer.Reconfigure(&newConf) + err = Context.dnsServer.Reconfigure(newConf) if err != nil { return fmt.Errorf("starting forwarding dns server: %w", err) } diff --git a/internal/home/home.go b/internal/home/home.go index bdc0f86c..60eb63dc 100644 --- a/internal/home/home.go +++ b/internal/home/home.go @@ -3,14 +3,12 @@ package home import ( "context" - "crypto/tls" "crypto/x509" "fmt" "io/fs" "net" "net/http" "net/netip" - "net/url" "os" "os/signal" "path/filepath" @@ -66,40 +64,24 @@ type homeContext struct { // configuration files, for example /etc/hosts. etcHosts *aghnet.HostsContainer - updater *updater.Updater - // mux is our custom http.ServeMux. mux *http.ServeMux // Runtime properties // -- - configFilename string // Config filename (can be overridden via the command line arguments) - workDir string // Location of our directory, used to protect against CWD being somewhere else - pidFileName string // PID file name. Empty if no PID file was created. - controlLock sync.Mutex - tlsRoots *x509.CertPool // list of root CAs for TLSv1.2 - client *http.Client - appSignalChannel chan os.Signal // Channel for receiving OS signals by the console app - - // rdnsCh is the channel for receiving IPs for rDNS processing. - rdnsCh chan netip.Addr - - // whoisCh is the channel for receiving IPs for WHOIS processing. - whoisCh chan netip.Addr + configFilename string // Config filename (can be overridden via the command line arguments) + workDir string // Location of our directory, used to protect against CWD being somewhere else + pidFileName string // PID file name. Empty if no PID file was created. + controlLock sync.Mutex + tlsRoots *x509.CertPool // list of root CAs for TLSv1.2 // tlsCipherIDs are the ID of the cipher suites that AdGuard Home must use. tlsCipherIDs []uint16 - // disableUpdate, if true, tells AdGuard Home to not check for updates. - disableUpdate bool - // firstRun, if true, tells AdGuard Home to only start the web interface // service, and only serve the first-run APIs. firstRun bool - - // runningAsService flag is set to true when options are passed from the service runner - runningAsService bool } // getDataDir returns path to the directory where we store databases and filters @@ -122,11 +104,11 @@ func Main(clientBuildFS fs.FS) { // package flag. opts := loadCmdLineOpts() - Context.appSignalChannel = make(chan os.Signal) - signal.Notify(Context.appSignalChannel, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT) + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT) go func() { for { - sig := <-Context.appSignalChannel + sig := <-signals log.Info("Received signal %q", sig) switch sig { case syscall.SIGHUP: @@ -141,7 +123,7 @@ func Main(clientBuildFS fs.FS) { }() if opts.serviceControlAction != "" { - handleServiceControlAction(opts, clientBuildFS) + handleServiceControlAction(opts, clientBuildFS, signals) return } @@ -153,74 +135,48 @@ func Main(clientBuildFS fs.FS) { // setupContext initializes [Context] fields. It also reads and upgrades // config file if necessary. func setupContext(opts options) (err error) { - setupContextFlags(opts) + Context.firstRun = detectFirstRun() Context.tlsRoots = aghtls.SystemRootCAs() - Context.client = &http.Client{ - Timeout: time.Minute * 5, - Transport: &http.Transport{ - DialContext: customDialContext, - Proxy: getHTTPProxy, - TLSClientConfig: &tls.Config{ - RootCAs: Context.tlsRoots, - CipherSuites: Context.tlsCipherIDs, - MinVersion: tls.VersionTLS12, - }, - }, - } - Context.mux = http.NewServeMux() - if !Context.firstRun { - // Do the upgrade if necessary. - err = upgradeConfig() + if Context.firstRun { + log.Info("This is the first time AdGuard Home is launched") + checkPermissions() + + return nil + } + + // Do the upgrade if necessary. + err = upgradeConfig() + if err != nil { + // Don't wrap the error, because it's informative enough as is. + return err + } + + if err = parseConfig(); err != nil { + log.Error("parsing configuration file: %s", err) + + os.Exit(1) + } + + if opts.checkConfig { + log.Info("configuration file is ok") + + os.Exit(0) + } + + if !opts.noEtcHosts && config.Clients.Sources.HostsFile { + err = setupHostsContainer() if err != nil { // Don't wrap the error, because it's informative enough as is. return err } - - if err = parseConfig(); err != nil { - log.Error("parsing configuration file: %s", err) - - os.Exit(1) - } - - if opts.checkConfig { - log.Info("configuration file is ok") - - os.Exit(0) - } - - if !opts.noEtcHosts && config.Clients.Sources.HostsFile { - err = setupHostsContainer() - if err != nil { - // Don't wrap the error, because it's informative enough as is. - return err - } - } } return nil } -// setupContextFlags sets global flags and prints their status to the log. -func setupContextFlags(opts options) { - Context.firstRun = detectFirstRun() - if Context.firstRun { - log.Info("This is the first time AdGuard Home is launched") - checkPermissions() - } - - Context.runningAsService = opts.runningAsService - // Don't print the runningAsService flag, since that has already been done - // in [run]. - - Context.disableUpdate = opts.disableUpdate || version.Channel() == version.ChannelDevelopment - if Context.disableUpdate { - log.Info("AdGuard Home updates are disabled") - } -} - // logIfUnsupported logs a formatted warning if the error is one of the // unsupported errors and returns nil. If err is nil, logIfUnsupported returns // nil. Otherwise, it returns err. @@ -325,7 +281,7 @@ func initContextClients() (err error) { return err } - //lint:ignore SA1019 Migration is not over. + //lint:ignore SA1019 Migration is not over. config.DHCP.WorkDir = Context.workDir config.DHCP.DataDir = Context.getDataDir() config.DHCP.HTTPRegister = httpRegister @@ -340,18 +296,6 @@ func initContextClients() (err error) { return fmt.Errorf("initing dhcp: %w", err) } - Context.updater = updater.NewUpdater(&updater.Config{ - Client: Context.client, - Version: version.Version(), - Channel: version.Channel(), - GOARCH: runtime.GOARCH, - GOOS: runtime.GOOS, - GOARM: version.GOARM(), - GOMIPS: version.GOMIPS(), - WorkDir: Context.workDir, - ConfName: config.getConfigFilename(), - }) - var arpdb aghnet.ARPDB if config.Clients.Sources.ARP { arpdb = aghnet.NewARPDB() @@ -433,7 +377,7 @@ func setupDNSFilteringConf(conf *filtering.Config) (err error) { conf.Filters = slices.Clone(config.Filters) conf.WhitelistFilters = slices.Clone(config.WhitelistFilters) conf.UserRules = slices.Clone(config.UserRules) - conf.HTTPClient = Context.client + conf.HTTPClient = httpClient() cacheTime := time.Duration(conf.CacheTime) * time.Minute @@ -515,7 +459,7 @@ func checkPorts() (err error) { return nil } -func initWeb(opts options, clientBuildFS fs.FS) (web *webAPI, err error) { +func initWeb(opts options, clientBuildFS fs.FS, upd *updater.Updater) (web *webAPI, err error) { var clientFS fs.FS if opts.localFrontend { log.Info("warning: using local frontend files") @@ -528,8 +472,16 @@ func initWeb(opts options, clientBuildFS fs.FS) (web *webAPI, err error) { } } - webConf := webConfig{ - firstRun: Context.firstRun, + disableUpdate := opts.disableUpdate || version.Channel() == version.ChannelDevelopment + if disableUpdate { + log.Info("AdGuard Home updates are disabled") + } + + webConf := &webConfig{ + updater: upd, + + clientFS: clientFS, + BindHost: config.HTTPConfig.Address.Addr(), BindPort: int(config.HTTPConfig.Address.Port()), @@ -537,12 +489,13 @@ func initWeb(opts options, clientBuildFS fs.FS) (web *webAPI, err error) { ReadHeaderTimeout: readHdrTimeout, WriteTimeout: writeTimeout, - clientFS: clientFS, - - serveHTTP3: config.DNS.ServeHTTP3, + firstRun: Context.firstRun, + disableUpdate: disableUpdate, + runningAsService: opts.runningAsService, + serveHTTP3: config.DNS.ServeHTTP3, } - web = newWebAPI(&webConf) + web = newWebAPI(webConf) if web == nil { return nil, fmt.Errorf("initializing web: %w", err) } @@ -593,9 +546,21 @@ func run(opts options, clientBuildFS fs.FS) { err = setupOpts(opts) fatalOnError(err) + upd := updater.NewUpdater(&updater.Config{ + Client: config.DNS.DnsfilterConf.HTTPClient, + Version: version.Version(), + Channel: version.Channel(), + GOARCH: runtime.GOARCH, + GOOS: runtime.GOOS, + GOARM: version.GOARM(), + GOMIPS: version.GOMIPS(), + WorkDir: Context.workDir, + ConfName: config.getConfigFilename(), + }) + // TODO(e.burkov): This could be made earlier, probably as the option's // effect. - cmdlineUpdate(opts) + cmdlineUpdate(opts, upd) if !Context.firstRun { // Save the updated config. @@ -624,7 +589,7 @@ func run(opts options, clientBuildFS fs.FS) { onConfigModified() } - Context.web, err = initWeb(opts, clientBuildFS) + Context.web, err = initWeb(opts, clientBuildFS, upd) fatalOnError(err) if !Context.firstRun { @@ -634,10 +599,10 @@ func run(opts options, clientBuildFS fs.FS) { Context.tls.start() go func() { - sErr := startDNSServer() - if sErr != nil { + startErr := startDNSServer() + if startErr != nil { closeDNSServer() - fatalOnError(sErr) + fatalOnError(startErr) } }() @@ -996,62 +961,6 @@ func detectFirstRun() bool { return errors.Is(err, os.ErrNotExist) } -// Connect to a remote server resolving hostname using our own DNS server. -// -// TODO(e.burkov): This messy logic should be decomposed and clarified. -// -// TODO(a.garipov): Support network. -func customDialContext(ctx context.Context, network, addr string) (conn net.Conn, err error) { - log.Debug("home: customdial: dialing addr %q for network %s", addr, network) - - host, port, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - - dialer := &net.Dialer{ - Timeout: time.Minute * 5, - } - - if net.ParseIP(host) != nil || config.DNS.Port == 0 { - return dialer.DialContext(ctx, network, addr) - } - - addrs, err := Context.dnsServer.Resolve(host) - if err != nil { - return nil, fmt.Errorf("resolving %q: %w", host, err) - } - - log.Debug("dnsServer.Resolve: %q: %v", host, addrs) - - if len(addrs) == 0 { - return nil, fmt.Errorf("couldn't lookup host: %q", host) - } - - var dialErrs []error - for _, a := range addrs { - addr = net.JoinHostPort(a.String(), port) - conn, err = dialer.DialContext(ctx, network, addr) - if err != nil { - dialErrs = append(dialErrs, err) - - continue - } - - return conn, err - } - - return nil, errors.List(fmt.Sprintf("couldn't dial to %s", addr), dialErrs...) -} - -func getHTTPProxy(_ *http.Request) (*url.URL, error) { - if config.ProxyURL == "" { - return nil, nil - } - - return url.Parse(config.ProxyURL) -} - // jsonError is a generic JSON error response. // // TODO(a.garipov): Merge together with the implementations in [dhcpd] and other @@ -1062,7 +971,7 @@ type jsonError struct { } // cmdlineUpdate updates current application and exits. -func cmdlineUpdate(opts options) { +func cmdlineUpdate(opts options, upd *updater.Updater) { if !opts.performUpdate { return } @@ -1077,10 +986,9 @@ func cmdlineUpdate(opts options) { log.Info("cmdline update: performing update") - updater := Context.updater - info, err := updater.VersionInfo(true) + info, err := upd.VersionInfo(true) if err != nil { - vcu := updater.VersionCheckURL() + vcu := upd.VersionCheckURL() log.Error("getting version info from %s: %s", vcu, err) os.Exit(1) @@ -1092,7 +1000,7 @@ func cmdlineUpdate(opts options) { os.Exit(0) } - err = updater.Update(Context.firstRun) + err = upd.Update(Context.firstRun) fatalOnError(err) err = restartService() diff --git a/internal/home/httpclient.go b/internal/home/httpclient.go new file mode 100644 index 00000000..ae41d6ac --- /dev/null +++ b/internal/home/httpclient.go @@ -0,0 +1,47 @@ +package home + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "net/url" + "time" +) + +// httpClient returns a new HTTP client that uses the AdGuard Home's own DNS +// server for resolving hostnames. The resulting client should not be used +// until [Context.dnsServer] is initialized. +// +// TODO(a.garipov, e.burkov): This is rather messy. Refactor. +func httpClient() (c *http.Client) { + // Do not use Context.dnsServer.DialContext directly in the struct literal + // below, since Context.dnsServer may be nil when this function is called. + dialContext := func(ctx context.Context, network, addr string) (conn net.Conn, err error) { + return Context.dnsServer.DialContext(ctx, network, addr) + } + + return &http.Client{ + // TODO(a.garipov): Make configurable. + Timeout: time.Minute * 5, + Transport: &http.Transport{ + DialContext: dialContext, + Proxy: httpProxy, + TLSClientConfig: &tls.Config{ + RootCAs: Context.tlsRoots, + CipherSuites: Context.tlsCipherIDs, + MinVersion: tls.VersionTLS12, + }, + }, + } +} + +// httpProxy returns parses and returns an HTTP proxy URL from the config, if +// any. +func httpProxy(_ *http.Request) (u *url.URL, err error) { + if config.ProxyURL == "" { + return nil, nil + } + + return url.Parse(config.ProxyURL) +} diff --git a/internal/home/pprof.go b/internal/home/pprof.go deleted file mode 100644 index b8ef8e74..00000000 --- a/internal/home/pprof.go +++ /dev/null @@ -1,39 +0,0 @@ -package home - -import ( - "net/http" - "net/http/pprof" - "runtime" - - "github.com/AdguardTeam/golibs/log" -) - -// startPprof launches the debug and profiling server on addr. -func startPprof(addr string) { - runtime.SetBlockProfileRate(1) - runtime.SetMutexProfileFraction(1) - - mux := http.NewServeMux() - - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - - // See profileSupportsDelta in src/net/http/pprof/pprof.go. - mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs")) - mux.Handle("/debug/pprof/block", pprof.Handler("block")) - mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) - mux.Handle("/debug/pprof/heap", pprof.Handler("heap")) - mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex")) - mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) - - go func() { - defer log.OnPanic("pprof server") - - log.Info("pprof: listening on %q", addr) - err := http.ListenAndServe(addr, mux) - log.Info("pprof server errors: %v", err) - }() -} diff --git a/internal/home/service.go b/internal/home/service.go index 3ec44138..e98aa030 100644 --- a/internal/home/service.go +++ b/internal/home/service.go @@ -33,9 +33,13 @@ const ( // daemon. type program struct { clientBuildFS fs.FS + signals chan os.Signal opts options } +// type check +var _ service.Interface = (*program)(nil) + // Start implements service.Interface interface for *program. func (p *program) Start(_ service.Service) (err error) { // Start should not block. Do the actual work async. @@ -48,14 +52,14 @@ func (p *program) Start(_ service.Service) (err error) { } // Stop implements service.Interface interface for *program. -func (p *program) Stop(_ service.Service) error { - // Stop should not block. Return with a few seconds. - if Context.appSignalChannel == nil { - os.Exit(0) +func (p *program) Stop(_ service.Service) (err error) { + select { + case p.signals <- syscall.SIGINT: + // Go on. + default: + // Stop should not block. } - Context.appSignalChannel <- syscall.SIGINT - return nil } @@ -194,7 +198,7 @@ func restartService() (err error) { // - run: This is a special command that is not supposed to be used directly // it is specified when we register a service, and it indicates to the app // that it is being run as a service/daemon. -func handleServiceControlAction(opts options, clientBuildFS fs.FS) { +func handleServiceControlAction(opts options, clientBuildFS fs.FS, signals chan os.Signal) { // Call chooseSystem explicitly to introduce OpenBSD support for service // package. It's a noop for other GOOS values. chooseSystem() @@ -226,7 +230,11 @@ func handleServiceControlAction(opts options, clientBuildFS fs.FS) { } configureService(svcConfig) - s, err := service.New(&program{clientBuildFS: clientBuildFS, opts: runOpts}, svcConfig) + s, err := service.New(&program{ + clientBuildFS: clientBuildFS, + signals: signals, + opts: runOpts, + }, svcConfig) if err != nil { log.Fatalf("service: initializing service: %s", err) } diff --git a/internal/home/upgrade.go b/internal/home/upgrade.go index 96b46b77..be9e2873 100644 --- a/internal/home/upgrade.go +++ b/internal/home/upgrade.go @@ -17,7 +17,7 @@ import ( "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/netutil" "github.com/AdguardTeam/golibs/timeutil" - "github.com/google/renameio/maybe" + "github.com/google/renameio/v2/maybe" "golang.org/x/crypto/bcrypt" yaml "gopkg.in/yaml.v3" ) diff --git a/internal/home/web.go b/internal/home/web.go index d53c946b..6649dfe2 100644 --- a/internal/home/web.go +++ b/internal/home/web.go @@ -6,16 +6,18 @@ import ( "io/fs" "net/http" "net/netip" + "runtime" "sync" "time" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/aghnet" + "github.com/AdguardTeam/AdGuardHome/internal/updater" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/netutil" + "github.com/AdguardTeam/golibs/pprofutil" "github.com/NYTimes/gziphandler" - "github.com/quic-go/quic-go" "github.com/quic-go/quic-go/http3" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" @@ -33,6 +35,8 @@ const ( ) type webConfig struct { + updater *updater.Updater + clientFS fs.FS BindHost netip.Addr @@ -52,6 +56,13 @@ type webConfig struct { firstRun bool + // disableUpdate, if true, tells AdGuard Home to not check for updates. + disableUpdate bool + + // runningAsService flag is set to true when options are passed from the + // service runner. + runningAsService bool + serveHTTP3 bool } @@ -102,7 +113,7 @@ func newWebAPI(conf *webConfig) (w *webAPI) { Context.mux.Handle("/install.html", preInstallHandler(clientFS)) w.registerInstallHandlers() } else { - registerControlHandlers() + registerControlHandlers(w) } w.httpsServer.cond = sync.NewCond(&w.httpsServer.condLock) @@ -295,8 +306,27 @@ func (web *webAPI) mustStartHTTP3(address string) { log.Debug("web: starting http/3 server") err := web.httpsServer.server3.ListenAndServe() - if !errors.Is(err, quic.ErrServerClosed) { + if !errors.Is(err, http.ErrServerClosed) { cleanupAlways() log.Fatalf("web: http3: %s", err) } } + +// startPprof launches the debug and profiling server on addr. +func startPprof(addr string) { + runtime.SetBlockProfileRate(1) + runtime.SetMutexProfileFraction(1) + + mux := http.NewServeMux() + pprofutil.RoutePprof(mux) + + go func() { + defer log.OnPanic("pprof server") + + log.Info("pprof: listening on %q", addr) + err := http.ListenAndServe(addr, mux) + if !errors.Is(err, http.ErrServerClosed) { + log.Error("pprof: shutting down: %s", err) + } + }() +} diff --git a/internal/rdns/rdns.go b/internal/rdns/rdns.go index e352da52..b33e212c 100644 --- a/internal/rdns/rdns.go +++ b/internal/rdns/rdns.go @@ -7,6 +7,7 @@ import ( "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/mathutil" "github.com/bluele/gcache" ) @@ -17,7 +18,7 @@ type Interface interface { Process(ip netip.Addr) (host string, changed bool) } -// Empty is an empty [Inteface] implementation which does nothing. +// Empty is an empty [Interface] implementation which does nothing. type Empty struct{} // type check @@ -32,7 +33,7 @@ func (Empty) Process(_ netip.Addr) (host string, changed bool) { type Exchanger interface { // Exchange tries to resolve the ip in a suitable way, i.e. either as local // or as external. - Exchange(ip netip.Addr) (host string, err error) + Exchange(ip netip.Addr) (host string, ttl time.Duration, err error) } // Config is the configuration structure for Default. @@ -82,13 +83,16 @@ func (r *Default) Process(ip netip.Addr) (host string, changed bool) { return fromCache, false } - host, err := r.exchanger.Exchange(ip) + host, ttl, err := r.exchanger.Exchange(ip) if err != nil { log.Debug("rdns: resolving %q: %s", ip, err) } + // TODO(s.chzhen): Use built-in function max in Go 1.21. + ttl = mathutil.Max(ttl, r.cacheTTL) + item := &cacheItem{ - expiry: time.Now().Add(r.cacheTTL), + expiry: time.Now().Add(ttl), host: host, } diff --git a/internal/rdns/rdns_test.go b/internal/rdns/rdns_test.go index 8694eba3..61130ec5 100644 --- a/internal/rdns/rdns_test.go +++ b/internal/rdns/rdns_test.go @@ -5,25 +5,13 @@ import ( "testing" "time" + "github.com/AdguardTeam/AdGuardHome/internal/aghtest" "github.com/AdguardTeam/AdGuardHome/internal/rdns" "github.com/AdguardTeam/golibs/netutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// fakeRDNSExchanger is a mock [rdns.Exchanger] implementation for tests. -type fakeRDNSExchanger struct { - OnExchange func(ip netip.Addr) (host string, err error) -} - -// type check -var _ rdns.Exchanger = (*fakeRDNSExchanger)(nil) - -// Exchange implements [rdns.Exchanger] interface for *fakeRDNSExchanger. -func (e *fakeRDNSExchanger) Exchange(ip netip.Addr) (host string, err error) { - return e.OnExchange(ip) -} - func TestDefault_Process(t *testing.T) { ip1 := netip.MustParseAddr("1.2.3.4") revAddr1, err := netutil.IPToReversedAddr(ip1.AsSlice()) @@ -67,21 +55,21 @@ func TestDefault_Process(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { hit := 0 - onExchange := func(ip netip.Addr) (host string, err error) { + onExchange := func(ip netip.Addr) (host string, ttl time.Duration, err error) { hit++ switch ip { case ip1: - return revAddr1, nil + return revAddr1, 0, nil case ip2: - return revAddr2, nil + return revAddr2, 0, nil case localIP: - return localRevAddr1, nil + return localRevAddr1, 0, nil default: - return "", nil + return "", 0, nil } } - exchanger := &fakeRDNSExchanger{ + exchanger := &aghtest.Exchanger{ OnExchange: onExchange, } diff --git a/internal/tools/go.mod b/internal/tools/go.mod index 47b2e330..2dd18576 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -9,9 +9,9 @@ require ( github.com/kisielk/errcheck v1.6.3 github.com/kyoh86/looppointer v0.2.1 github.com/securego/gosec/v2 v2.16.0 - github.com/uudashr/gocognit v1.0.6 + github.com/uudashr/gocognit v1.0.7 golang.org/x/tools v0.11.0 - golang.org/x/vuln v0.2.0 + golang.org/x/vuln v1.0.0 // TODO(a.garipov): Return to tagged releases once a new one appears. honnef.co/go/tools v0.5.0-0.dev.0.20230709092525-bc759185c5ee mvdan.cc/gofumpt v0.5.0 @@ -22,12 +22,12 @@ require ( github.com/BurntSushi/toml v1.3.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/gookit/color v1.5.3 // indirect + github.com/gookit/color v1.5.4 // indirect github.com/kyoh86/nolint v0.0.1 // indirect github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect - golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22 // indirect + golang.org/x/exp/typeparams v0.0.0-20230725093048-515e97ebf090 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.10.0 // indirect diff --git a/internal/tools/go.sum b/internal/tools/go.sum index d2244861..56a70261 100644 --- a/internal/tools/go.sum +++ b/internal/tools/go.sum @@ -16,8 +16,8 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gookit/color v1.5.3 h1:twfIhZs4QLCtimkP7MOxlF3A0U/5cDPseRT9M/+2SCE= -github.com/gookit/color v1.5.3/go.mod h1:NUzwzeehUfl7GIb36pqId+UGmRfQcU/WiiyTTeNjHtE= +github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= +github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/gordonklaus/ineffassign v0.0.0-20230610083614-0e73809eb601 h1:mrEEilTAUmaAORhssPPkxj84TsHrPMLBGW2Z4SoTxm8= github.com/gordonklaus/ineffassign v0.0.0-20230610083614-0e73809eb601/go.mod h1:Qcp2HIAYhR7mNUVSIxZww3Guk4it82ghYcEXIAk+QT0= github.com/kisielk/errcheck v1.6.3 h1:dEKh+GLHcWm2oN34nMvDzn1sqI0i0WxPvrgiJA5JuM8= @@ -38,9 +38,9 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/securego/gosec/v2 v2.16.0 h1:Pi0JKoasQQ3NnoRao/ww/N/XdynIB9NRYYZT5CyOs5U= github.com/securego/gosec/v2 v2.16.0/go.mod h1:xvLcVZqUfo4aAQu56TNv7/Ltz6emAOQAEsrZrt7uGlI= github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/uudashr/gocognit v1.0.6 h1:2Cgi6MweCsdB6kpcVQp7EW4U23iBFQWfTXiWlyp842Y= -github.com/uudashr/gocognit v1.0.6/go.mod h1:nAIUuVBnYU7pcninia3BHOvQkpQCeO76Uscky5BOwcY= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/uudashr/gocognit v1.0.7 h1:e9aFXgKgUJrQ5+bs61zBigmj7bFJ/5cC6HmMahVzuDo= +github.com/uudashr/gocognit v1.0.7/go.mod h1:nAIUuVBnYU7pcninia3BHOvQkpQCeO76Uscky5BOwcY= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -52,8 +52,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22 h1:e8iSCQYXZ4EB6q3kIfy2fgPFTvDbozqzRe4OuIOyrL4= -golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20230725093048-515e97ebf090 h1:qOYhjyK9OeXREdh7Zrta8JRvnmnFIzhkosQpp+852Ag= +golang.org/x/exp/typeparams v0.0.0-20230725093048-515e97ebf090/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= @@ -98,8 +98,8 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= -golang.org/x/vuln v0.2.0 h1:Dlz47lW0pvPHU7tnb10S8vbMn9GnV2B6eyT7Tem5XBI= -golang.org/x/vuln v0.2.0/go.mod h1:V0eyhHwaAaHrt42J9bgrN6rd12f6GU4T0Lu0ex2wDg4= +golang.org/x/vuln v1.0.0 h1:tYLAU3jD9LQr98Y+3el06lWyGMCnvzw06PIWP3LIy7g= +golang.org/x/vuln v1.0.0/go.mod h1:V0eyhHwaAaHrt42J9bgrN6rd12f6GU4T0Lu0ex2wDg4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/whois/whois.go b/internal/whois/whois.go index 2a64bdd1..ae01304b 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -13,6 +13,7 @@ import ( "time" "github.com/AdguardTeam/AdGuardHome/internal/aghio" + "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/netutil" @@ -48,9 +49,8 @@ func (Empty) Process(_ context.Context, _ netip.Addr) (info *Info, changed bool) // Config is the configuration structure for Default. type Config struct { - // DialContext specifies the dial function for creating unencrypted TCP - // connections. - DialContext func(ctx context.Context, network, addr string) (conn net.Conn, err error) + // DialContext is used to create TCP connections to WHOIS servers. + DialContext aghnet.DialContextFunc // ServerAddr is the address of the WHOIS server. ServerAddr string @@ -86,9 +86,8 @@ type Default struct { // resolve the same IP. cache gcache.Cache - // dialContext connects to a remote server resolving hostname using our own - // DNS server and unecrypted TCP connection. - dialContext func(ctx context.Context, network, addr string) (conn net.Conn, err error) + // dialContext is used to create TCP connections to WHOIS servers. + dialContext aghnet.DialContextFunc // serverAddr is the address of the WHOIS server. serverAddr string @@ -215,7 +214,7 @@ func (w *Default) query(ctx context.Context, target, serverAddr string) (data [] return nil, err } - _ = conn.SetReadDeadline(time.Now().Add(w.timeout)) + _ = conn.SetDeadline(time.Now().Add(w.timeout)) _, err = io.WriteString(conn, target+"\r\n") if err != nil { // Don't wrap the error since it's informative enough as is. @@ -310,7 +309,7 @@ func (w *Default) requestInfo( kv, err := w.queryAll(ctx, ip.String()) if err != nil { - log.Debug("whois: quering about %q: %s", ip, err) + log.Debug("whois: querying %q: %s", ip, err) return nil, true } diff --git a/internal/whois/whois_test.go b/internal/whois/whois_test.go index 2fe32255..109ee1e9 100644 --- a/internal/whois/whois_test.go +++ b/internal/whois/whois_test.go @@ -113,20 +113,14 @@ func TestDefault_Process(t *testing.T) { return copy(b, tc.data), io.EOF }, - OnWrite: func(b []byte) (n int, err error) { - return len(b), nil - }, - OnClose: func() (err error) { - return nil - }, - OnSetReadDeadline: func(t time.Time) (err error) { - return nil - }, + OnWrite: func(b []byte) (n int, err error) { return len(b), nil }, + OnClose: func() (err error) { return nil }, + OnSetDeadline: func(t time.Time) (err error) { return nil }, } w := whois.New(&whois.Config{ Timeout: 5 * time.Second, - DialContext: func(_ context.Context, _, addr string) (_ net.Conn, _ error) { + DialContext: func(_ context.Context, _, _ string) (_ net.Conn, _ error) { hit = 0 return fakeConn, nil diff --git a/scripts/README.md b/scripts/README.md index d9ef6851..5ce27e80 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -269,25 +269,31 @@ Optional environment: ### Usage - * `go run main.go help`: print usage. + * `go run ./scripts/translations help`: print usage. - * `go run main.go download [-n ]`: download and save all translations. - `n` is optional flag where count is a number of concurrent downloads. + * `go run ./scripts/translations download [-n ]`: download and save + all translations. `n` is optional flag where count is a number of + concurrent downloads. - * `go run main.go upload`: upload the base `en` locale. + * `go run ./scripts/translations upload`: upload the base `en` locale. - * `go run main.go summary`: show the current locales summary. + * `go run ./scripts/translations summary`: show the current locales summary. - * `go run main.go unused`: show the list of unused strings. + * `go run ./scripts/translations unused`: show the list of unused strings. - * `go run main.go auto-add`: add locales with additions to the git and - restore locales with deletions. + * `go run ./scripts/translations auto-add`: add locales with additions to the + git and restore locales with deletions. After the download you'll find the output locales in the `client/src/__locales/` directory. Optional environment: + * `DOWNLOAD_LANGUAGES`: set a list of specific languages to `download`. For + example `ar be bg`. If it set to `blocker` then script will download only + those languages, which need to be fully translated (`de en es fr it ja ko + pt-br pt-pt ru zh-cn zh-tw`). + * `UPLOAD_LANGUAGE`: set an alternative language for `upload`. * `TWOSKY_URI`: set an alternative URL for `download` or `upload`. diff --git a/scripts/make/go-lint.sh b/scripts/make/go-lint.sh index 286a87f3..77df4af7 100644 --- a/scripts/make/go-lint.sh +++ b/scripts/make/go-lint.sh @@ -176,15 +176,30 @@ run_linter gocognit --over 10\ ./internal/aghchan/\ ./internal/aghhttp/\ ./internal/aghio/\ + ./internal/aghrenameio/\ + ./internal/client/\ + ./internal/dhcpsvc\ ./internal/filtering/hashprefix/\ ./internal/filtering/rulelist/\ ./internal/next/\ ./internal/rdns/\ + ./internal/schedule/\ ./internal/tools/\ ./internal/version/\ ./internal/whois/\ + ./scripts/\ ; +# TODO(a.garipov): move these to the group above. +run_linter gocognit --over 20 ./internal/aghnet/ ./internal/querylog/ +run_linter gocognit --over 19 ./internal/dnsforward/ ./internal/home/ +run_linter gocognit --over 18 ./internal/aghtls/ +run_linter gocognit --over 17 ./internal/filtering ./internal/filtering/rewrite/ +run_linter gocognit --over 15 ./internal/aghos/ ./internal/dhcpd/ +run_linter gocognit --over 14 ./internal/stats/ +run_linter gocognit --over 12 ./internal/updater/ +run_linter gocognit --over 11 ./internal/aghtest/ + run_linter ineffassign ./... run_linter unparam ./... @@ -209,13 +224,16 @@ run_linter gosec --quiet\ ./internal/aghio\ ./internal/aghnet\ ./internal/aghos\ + ./internal/aghrenameio/\ ./internal/aghtest\ + ./internal/client\ ./internal/dhcpd\ ./internal/dhcpsvc\ ./internal/dnsforward\ ./internal/filtering/hashprefix/\ ./internal/filtering/rulelist/\ ./internal/next\ + ./internal/rdns\ ./internal/schedule\ ./internal/stats\ ./internal/tools\ diff --git a/scripts/translations/download.go b/scripts/translations/download.go new file mode 100644 index 00000000..b9b809c9 --- /dev/null +++ b/scripts/translations/download.go @@ -0,0 +1,177 @@ +package main + +import ( + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/AdguardTeam/AdGuardHome/internal/aghio" + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/log" + "golang.org/x/exp/slices" +) + +// download and save all translations. +func (c *twoskyClient) download() (err error) { + var numWorker int + + flagSet := flag.NewFlagSet("download", flag.ExitOnError) + flagSet.Usage = func() { + usage("download command error") + } + flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads") + + err = flagSet.Parse(os.Args[2:]) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return err + } + + if numWorker < 1 { + usage("count must be positive") + } + + downloadURI := c.uri.JoinPath("download") + + client := &http.Client{ + Timeout: 10 * time.Second, + } + + wg := &sync.WaitGroup{} + failed := &sync.Map{} + uriCh := make(chan *url.URL, len(c.langs)) + + for i := 0; i < numWorker; i++ { + wg.Add(1) + go downloadWorker(wg, failed, client, uriCh) + } + + for _, lang := range c.langs { + uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang) + + uriCh <- uri + } + + close(uriCh) + wg.Wait() + + printFailedLocales(failed) + + return nil +} + +// printFailedLocales prints sorted list of failed downloads, if any. +func printFailedLocales(failed *sync.Map) { + keys := []string{} + failed.Range(func(k, _ any) bool { + s, ok := k.(string) + if !ok { + panic("unexpected type") + } + + keys = append(keys, s) + + return true + }) + + if len(keys) == 0 { + return + } + + slices.Sort(keys) + log.Info("failed locales: %s", strings.Join(keys, " ")) +} + +// downloadWorker downloads translations by received urls and saves them. +// Where failed is a map for storing failed downloads. +func downloadWorker( + wg *sync.WaitGroup, + failed *sync.Map, + client *http.Client, + uriCh <-chan *url.URL, +) { + defer wg.Done() + + for uri := range uriCh { + q := uri.Query() + code := q.Get("language") + + err := saveToFile(client, uri, code) + if err != nil { + log.Error("download: worker: %s", err) + failed.Store(code, struct{}{}) + } + } +} + +// saveToFile downloads translation by url and saves it to a file, or returns +// error. +func saveToFile(client *http.Client, uri *url.URL, code string) (err error) { + data, err := getTranslation(client, uri.String()) + if err != nil { + log.Info("%s", data) + + return fmt.Errorf("getting translation: %s", err) + } + + name := filepath.Join(localesDir, code+".json") + err = os.WriteFile(name, data, 0o664) + if err != nil { + return fmt.Errorf("writing file: %s", err) + } + + fmt.Println(name) + + return nil +} + +// getTranslation returns received translation data and error. If err is not +// nil, data may contain a response from server for inspection. +func getTranslation(client *http.Client, url string) (data []byte, err error) { + resp, err := client.Get(url) + if err != nil { + return nil, fmt.Errorf("requesting: %w", err) + } + + defer log.OnCloserError(resp.Body, log.ERROR) + + if resp.StatusCode != http.StatusOK { + err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode)) + + // Go on and download the body for inspection. + } + + limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit) + if lrErr != nil { + // Generally shouldn't happen, since the only error returned by + // [aghio.LimitReader] is an argument error. + panic(fmt.Errorf("limit reading: %w", lrErr)) + } + + data, readErr := io.ReadAll(limitReader) + + return data, errors.WithDeferred(err, readErr) +} + +// translationURL returns a new url.URL with provided query parameters. +func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) { + uri = &url.URL{} + *uri = *oldURL + + q := uri.Query() + q.Set("format", "json") + q.Set("filename", baseFile) + q.Set("project", projectID) + q.Set("language", string(lang)) + + uri.RawQuery = q.Encode() + + return uri +} diff --git a/scripts/translations/main.go b/scripts/translations/main.go index 0d4e7871..800c36ee 100644 --- a/scripts/translations/main.go +++ b/scripts/translations/main.go @@ -6,25 +6,16 @@ import ( "bufio" "bytes" "encoding/json" - "flag" "fmt" - "io" - "mime/multipart" - "net/http" - "net/textproto" "net/url" "os" "os/exec" "path/filepath" "strings" - "sync" "time" - "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" - "github.com/AdguardTeam/AdGuardHome/internal/aghio" "github.com/AdguardTeam/AdGuardHome/internal/aghos" "github.com/AdguardTeam/golibs/errors" - "github.com/AdguardTeam/golibs/httphdr" "github.com/AdguardTeam/golibs/log" "golang.org/x/exp/maps" "golang.org/x/exp/slices" @@ -38,9 +29,26 @@ const ( srcDir = "./client/src" twoskyURI = "https://twosky.int.agrd.dev/api/v1" - readLimit = 1 * 1024 * 1024 + readLimit = 1 * 1024 * 1024 + uploadTimeout = 10 * time.Second ) +// blockerLangCodes is the codes of languages which need to be fully translated. +var blockerLangCodes = []langCode{ + "de", + "en", + "es", + "fr", + "it", + "ja", + "ko", + "pt-br", + "pt-pt", + "ru", + "zh-cn", + "zh-tw", +} + // langCode is a language code. type langCode string @@ -62,31 +70,26 @@ func main() { usage("") } - uriStr := os.Getenv("TWOSKY_URI") - if uriStr == "" { - uriStr = twoskyURI - } - - uri, err := url.Parse(uriStr) + conf, err := readTwoskyConfig() check(err) - projectID := os.Getenv("TWOSKY_PROJECT_ID") - if projectID == "" { - projectID = defaultProjectID - } - - conf, err := readTwoskyConf() - check(err) + var cli *twoskyClient switch os.Args[1] { case "summary": err = summary(conf.Languages) case "download": - err = download(uri, projectID, conf.Languages) + cli, err = conf.toClient() + check(err) + + err = cli.download() case "unused": err = unused(conf.LocalizableFiles[0]) case "upload": - err = upload(uri, projectID, conf.BaseLangcode) + cli, err = conf.toClient() + check(err) + + err = cli.upload() case "auto-add": err = autoAdd(conf.LocalizableFiles[0]) default: @@ -133,51 +136,133 @@ Commands: os.Exit(0) } -// twoskyConf is the configuration structure for localization. -type twoskyConf struct { +// twoskyConfig is the configuration structure for localization. +type twoskyConfig struct { Languages languages `json:"languages"` ProjectID string `json:"project_id"` BaseLangcode langCode `json:"base_locale"` LocalizableFiles []string `json:"localizable_files"` } -// readTwoskyConf returns configuration. -func readTwoskyConf() (t twoskyConf, err error) { - defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }() +// readTwoskyConfig returns twosky configuration. +func readTwoskyConfig() (t *twoskyConfig, err error) { + defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }() b, err := os.ReadFile(twoskyConfFile) if err != nil { // Don't wrap the error since it's informative enough as is. - return twoskyConf{}, err + return nil, err } - var tsc []twoskyConf + var tsc []twoskyConfig err = json.Unmarshal(b, &tsc) if err != nil { err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err) - return twoskyConf{}, err + return nil, err } if len(tsc) == 0 { err = fmt.Errorf("%q is empty", twoskyConfFile) - return twoskyConf{}, err + return nil, err } conf := tsc[0] for _, lang := range conf.Languages { if lang == "" { - return twoskyConf{}, errors.Error("language is empty") + return nil, errors.Error("language is empty") } } if len(conf.LocalizableFiles) == 0 { - return twoskyConf{}, errors.Error("no localizable files specified") + return nil, errors.Error("no localizable files specified") } - return conf, nil + return &conf, nil +} + +// twoskyClient is the twosky client with methods for download and upload +// translations. +type twoskyClient struct { + // uri is the base URL. + uri *url.URL + + // projectID is the name of the project. + projectID string + + // baseLang is the base language code. + baseLang langCode + + // langs is the list of codes of languages to download. + langs []langCode +} + +// toClient reads values from environment variables or defaults, validates +// them, and returns the twosky client. +func (t *twoskyConfig) toClient() (cli *twoskyClient, err error) { + defer func() { err = errors.Annotate(err, "filling config: %w") }() + + uriStr := os.Getenv("TWOSKY_URI") + if uriStr == "" { + uriStr = twoskyURI + } + uri, err := url.Parse(uriStr) + if err != nil { + return nil, err + } + + projectID := os.Getenv("TWOSKY_PROJECT_ID") + if projectID == "" { + projectID = defaultProjectID + } + + baseLang := t.BaseLangcode + uLangStr := os.Getenv("UPLOAD_LANGUAGE") + if uLangStr != "" { + baseLang = langCode(uLangStr) + } + + langs := maps.Keys(t.Languages) + dlLangStr := os.Getenv("DOWNLOAD_LANGUAGES") + if dlLangStr == "blocker" { + langs = blockerLangCodes + } else if dlLangStr != "" { + var dlLangs []langCode + dlLangs, err = validateLanguageStr(dlLangStr, t.Languages) + if err != nil { + return nil, err + } + + langs = dlLangs + } + + return &twoskyClient{ + uri: uri, + projectID: projectID, + baseLang: baseLang, + langs: langs, + }, nil +} + +// validateLanguageStr validates languages codes that contain in the str and +// returns them or error. +func validateLanguageStr(str string, all languages) (langs []langCode, err error) { + codes := strings.Fields(str) + langs = make([]langCode, 0, len(codes)) + + for _, k := range codes { + lc := langCode(k) + _, ok := all[lc] + if !ok { + return nil, fmt.Errorf("validating languages: unexpected language code %q", k) + } + + langs = append(langs, lc) + } + + return langs, nil } // readLocales reads file with name fn and returns a map, where key is text @@ -227,169 +312,47 @@ func summary(langs languages) (err error) { f := float64(len(loc)) * 100 / size - fmt.Printf("%s\t %6.2f %%\n", lang, f) + blocker := "" + + // N is small enough to not raise performance questions. + ok := slices.Contains(blockerLangCodes, lang) + if ok { + blocker = " (blocker)" + } + + fmt.Printf("%s\t %6.2f %%%s\n", lang, f, blocker) } return nil } -// download and save all translations. uri is the base URL. projectID is the -// name of the project. -func download(uri *url.URL, projectID string, langs languages) (err error) { - var numWorker int - - flagSet := flag.NewFlagSet("download", flag.ExitOnError) - flagSet.Usage = func() { - usage("download command error") - } - flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads") - - err = flagSet.Parse(os.Args[2:]) - if err != nil { - // Don't wrap the error since it's informative enough as is. - return err - } - - if numWorker < 1 { - usage("count must be positive") - } - - downloadURI := uri.JoinPath("download") - - client := &http.Client{ - Timeout: 10 * time.Second, - } - - wg := &sync.WaitGroup{} - uriCh := make(chan *url.URL, len(langs)) - - for i := 0; i < numWorker; i++ { - wg.Add(1) - go downloadWorker(wg, client, uriCh) - } - - for lang := range langs { - uri = translationURL(downloadURI, defaultBaseFile, projectID, lang) - - uriCh <- uri - } - - close(uriCh) - wg.Wait() - - return nil -} - -// downloadWorker downloads translations by received urls and saves them. -func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.URL) { - defer wg.Done() - - for uri := range uriCh { - data, err := getTranslation(client, uri.String()) - if err != nil { - log.Error("download worker: getting translation: %s", err) - log.Info("download worker: error response:\n%s", data) - - continue - } - - q := uri.Query() - code := q.Get("language") - - // Fix some TwoSky weirdnesses. - // - // TODO(a.garipov): Remove when those are fixed. - code = strings.ToLower(code) - - name := filepath.Join(localesDir, code+".json") - err = os.WriteFile(name, data, 0o664) - if err != nil { - log.Error("download worker: writing file: %s", err) - - continue - } - - fmt.Println(name) - } -} - -// getTranslation returns received translation data and error. If err is not -// nil, data may contain a response from server for inspection. -func getTranslation(client *http.Client, url string) (data []byte, err error) { - resp, err := client.Get(url) - if err != nil { - return nil, fmt.Errorf("requesting: %w", err) - } - - defer log.OnCloserError(resp.Body, log.ERROR) - - if resp.StatusCode != http.StatusOK { - err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode)) - - // Go on and download the body for inspection. - } - - limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit) - if lrErr != nil { - // Generally shouldn't happen, since the only error returned by - // [aghio.LimitReader] is an argument error. - panic(fmt.Errorf("limit reading: %w", lrErr)) - } - - data, readErr := io.ReadAll(limitReader) - - return data, errors.WithDeferred(err, readErr) -} - -// translationURL returns a new url.URL with provided query parameters. -func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) { - uri = &url.URL{} - *uri = *oldURL - - // Fix some TwoSky weirdnesses. - // - // TODO(a.garipov): Remove when those are fixed. - switch lang { - case "si-lk": - lang = "si-LK" - case "zh-hk": - lang = "zh-HK" - default: - // Go on. - } - - q := uri.Query() - q.Set("format", "json") - q.Set("filename", baseFile) - q.Set("project", projectID) - q.Set("language", string(lang)) - - uri.RawQuery = q.Encode() - - return uri -} - // unused prints unused text labels. func unused(basePath string) (err error) { + defer func() { err = errors.Annotate(err, "unused: %w") }() + baseLoc, err := readLocales(basePath) if err != nil { - return fmt.Errorf("unused: %w", err) + return err } locDir := filepath.Clean(localesDir) + js, err := findJS(locDir) + if err != nil { + return err + } - fileNames := []string{} - err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error { + return findUnused(js, baseLoc) +} + +// findJS returns list of JavaScript and JSON files or error. +func findJS(locDir string) (fileNames []string, err error) { + walkFn := func(name string, _ os.FileInfo, err error) error { if err != nil { log.Info("warning: accessing a path %q: %s", name, err) return nil } - if info.IsDir() { - return nil - } - if strings.HasPrefix(name, locDir) { return nil } @@ -400,13 +363,14 @@ func unused(basePath string) (err error) { } return nil - }) - - if err != nil { - return fmt.Errorf("filepath walking %q: %w", srcDir, err) } - return findUnused(fileNames, baseLoc) + err = filepath.Walk(srcDir, walkFn) + if err != nil { + return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err) + } + + return fileNames, nil } // findUnused prints unused text labels from fileNames. @@ -445,118 +409,6 @@ func findUnused(fileNames []string, loc locales) (err error) { return nil } -// upload base translation. uri is the base URL. projectID is the name of the -// project. baseLang is the base language code. -func upload(uri *url.URL, projectID string, baseLang langCode) (err error) { - defer func() { err = errors.Annotate(err, "upload: %w") }() - - uploadURI := uri.JoinPath("upload") - - lang := baseLang - - langStr := os.Getenv("UPLOAD_LANGUAGE") - if langStr != "" { - lang = langCode(langStr) - } - - basePath := filepath.Join(localesDir, defaultBaseFile) - - formData := map[string]string{ - "format": "json", - "language": string(lang), - "filename": defaultBaseFile, - "project": projectID, - } - - buf, cType, err := prepareMultipartMsg(formData, basePath) - if err != nil { - return fmt.Errorf("preparing multipart msg: %w", err) - } - - err = send(uploadURI.String(), cType, buf) - if err != nil { - return fmt.Errorf("sending multipart msg: %w", err) - } - - return nil -} - -// prepareMultipartMsg prepares translation data for upload. -func prepareMultipartMsg( - formData map[string]string, - basePath string, -) (buf *bytes.Buffer, cType string, err error) { - buf = &bytes.Buffer{} - w := multipart.NewWriter(buf) - var fw io.Writer - - for k, v := range formData { - err = w.WriteField(k, v) - if err != nil { - return nil, "", fmt.Errorf("writing field: %w", err) - } - } - - file, err := os.Open(basePath) - if err != nil { - return nil, "", fmt.Errorf("opening file: %w", err) - } - - defer func() { - err = errors.WithDeferred(err, file.Close()) - }() - - h := make(textproto.MIMEHeader) - h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON) - - d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile) - h.Set(httphdr.ContentDisposition, d) - - fw, err = w.CreatePart(h) - if err != nil { - return nil, "", fmt.Errorf("creating part: %w", err) - } - - _, err = io.Copy(fw, file) - if err != nil { - return nil, "", fmt.Errorf("copying: %w", err) - } - - err = w.Close() - if err != nil { - return nil, "", fmt.Errorf("closing writer: %w", err) - } - - return buf, w.FormDataContentType(), nil -} - -// send POST request to uriStr. -func send(uriStr, cType string, buf *bytes.Buffer) (err error) { - var client http.Client - - req, err := http.NewRequest(http.MethodPost, uriStr, buf) - if err != nil { - return fmt.Errorf("bad request: %w", err) - } - - req.Header.Set(httphdr.ContentType, cType) - - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("client post form: %w", err) - } - - defer func() { - err = errors.WithDeferred(err, resp.Body.Close()) - }() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode)) - } - - return nil -} - // autoAdd adds locales with additions to the git and restores locales with // deletions. func autoAdd(basePath string) (err error) { @@ -572,28 +424,48 @@ func autoAdd(basePath string) (err error) { return errors.Error("base locale contains deletions") } - var ( - args []string - code int - out []byte - ) - - if len(adds) > 0 { - args = append([]string{"add"}, adds...) - code, out, err = aghos.RunCommand("git", args...) - - if err != nil || code != 0 { - return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err) - } + err = handleAdds(adds) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil } - if len(dels) > 0 { - args = append([]string{"restore"}, dels...) - code, out, err = aghos.RunCommand("git", args...) + err = handleDels(dels) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil + } - if err != nil || code != 0 { - return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err) - } + return nil +} + +// handleAdds adds locales with additions to the git. +func handleAdds(locales []string) (err error) { + if len(locales) == 0 { + return nil + } + + args := append([]string{"add"}, locales...) + code, out, err := aghos.RunCommand("git", args...) + + if err != nil || code != 0 { + return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err) + } + + return nil +} + +// handleDels restores locales with deletions. +func handleDels(locales []string) (err error) { + if len(locales) == 0 { + return nil + } + + args := append([]string{"restore"}, locales...) + code, out, err := aghos.RunCommand("git", args...) + + if err != nil || code != 0 { + return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err) } return nil diff --git a/scripts/translations/upload.go b/scripts/translations/upload.go new file mode 100644 index 00000000..b9cfd4bf --- /dev/null +++ b/scripts/translations/upload.go @@ -0,0 +1,120 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/textproto" + "os" + "path/filepath" + + "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/httphdr" + "github.com/AdguardTeam/golibs/mapsutil" +) + +// upload base translation. +func (c *twoskyClient) upload() (err error) { + defer func() { err = errors.Annotate(err, "upload: %w") }() + + uploadURI := c.uri.JoinPath("upload") + basePath := filepath.Join(localesDir, defaultBaseFile) + + formData := map[string]string{ + "format": "json", + "language": string(c.baseLang), + "filename": defaultBaseFile, + "project": c.projectID, + } + + buf, cType, err := prepareMultipartMsg(formData, basePath) + if err != nil { + return fmt.Errorf("preparing multipart msg: %w", err) + } + + err = send(uploadURI.String(), cType, buf) + if err != nil { + return fmt.Errorf("sending multipart msg: %w", err) + } + + return nil +} + +// prepareMultipartMsg prepares translation data for upload. +func prepareMultipartMsg( + formData map[string]string, + basePath string, +) (buf *bytes.Buffer, cType string, err error) { + buf = &bytes.Buffer{} + w := multipart.NewWriter(buf) + var fw io.Writer + + err = mapsutil.OrderedRangeError(formData, w.WriteField) + if err != nil { + return nil, "", fmt.Errorf("writing field: %w", err) + } + + file, err := os.Open(basePath) + if err != nil { + return nil, "", fmt.Errorf("opening file: %w", err) + } + + defer func() { + err = errors.WithDeferred(err, file.Close()) + }() + + h := make(textproto.MIMEHeader) + h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON) + + d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile) + h.Set(httphdr.ContentDisposition, d) + + fw, err = w.CreatePart(h) + if err != nil { + return nil, "", fmt.Errorf("creating part: %w", err) + } + + _, err = io.Copy(fw, file) + if err != nil { + return nil, "", fmt.Errorf("copying: %w", err) + } + + err = w.Close() + if err != nil { + return nil, "", fmt.Errorf("closing writer: %w", err) + } + + return buf, w.FormDataContentType(), nil +} + +// send POST request to uriStr. +func send(uriStr, cType string, buf *bytes.Buffer) (err error) { + client := http.Client{ + Timeout: uploadTimeout, + } + + req, err := http.NewRequest(http.MethodPost, uriStr, buf) + if err != nil { + return fmt.Errorf("bad request: %w", err) + } + + req.Header.Set(httphdr.ContentType, cType) + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("client post form: %w", err) + } + + defer func() { + err = errors.WithDeferred(err, resp.Body.Close()) + }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode)) + } + + return nil +} diff --git a/scripts/vetted-filters/main.go b/scripts/vetted-filters/main.go index e0076878..c960f200 100644 --- a/scripts/vetted-filters/main.go +++ b/scripts/vetted-filters/main.go @@ -12,7 +12,7 @@ import ( "time" "github.com/AdguardTeam/golibs/log" - "github.com/google/renameio/maybe" + "github.com/google/renameio/v2/maybe" ) func main() {