From 91f3e29c08b1ac6074f0a789af1a355cd244a695 Mon Sep 17 00:00:00 2001 From: Eugene Burkov Date: Fri, 30 Jun 2023 12:41:10 +0300 Subject: [PATCH] Pull request 1891: 5902-bootstrap-hosts Merge in DNS/adguard-home from 5902-bootstrap-hosts to master Updates #5902. Squashed commit of the following: commit fcc65d3a8d7566acc361f54b18d1af85045225e2 Merge: 0c336af07 1fd6cf1a2 Author: Eugene Burkov Date: Fri Jun 30 12:29:06 2023 +0300 Merge branch 'master' into 5902-bootstrap-hosts commit 0c336af07d2864533e1f10029b4321d7cd210a47 Author: Eugene Burkov Date: Thu Jun 29 15:40:28 2023 +0300 all: imp & simplify commit 45aae90035b98b30199cc7fc92991528f4e968c0 Author: Eugene Burkov Date: Wed Jun 28 20:24:43 2023 +0300 all: imp code, docs commit e3dbb5bfe5dfbde7af00f39adcc15e9711e5feb0 Merge: a33a8e93c 2069eddf9 Author: Eugene Burkov Date: Wed Jun 28 18:27:36 2023 +0300 Merge branch 'master' into 5902-bootstrap-hosts commit a33a8e93cb36f7d0c4472e524e44de6ff0ab6653 Author: Eugene Burkov Date: Wed Jun 28 13:27:11 2023 +0300 aghos: add type check commit 781a3a248871df2ea37a936c8d6b0b11e2d2f3a4 Author: Eugene Burkov Date: Wed Jun 28 13:09:37 2023 +0300 all: log changes commit 4575368655356f84992fad2bfb78cbc1c88da25a Merge: 636c440fc cf7c12c97 Author: Eugene Burkov Date: Wed Jun 28 13:08:11 2023 +0300 Merge branch 'master' into 5902-bootstrap-hosts commit 636c440fca9cbdfd5c12b7f89432fb9323e01d86 Author: Eugene Burkov Date: Wed Jun 28 13:06:32 2023 +0300 all: imp tests commit 0eff7a747e32216d78abf9db9460cb9d48f31f96 Author: Eugene Burkov Date: Mon Jun 26 18:40:22 2023 +0300 dnsforward: imp code commit 7489a30971e3c76b8f62fd4ca11a977eeabe2cf5 Author: Eugene Burkov Date: Mon Jun 26 17:04:10 2023 +0300 all: resolve upstreams with hosts --- CHANGELOG.md | 3 + internal/aghnet/hostscontainer.go | 19 +- internal/dnsforward/config.go | 97 ---------- internal/dnsforward/dnsforward.go | 21 +- internal/dnsforward/http.go | 146 +++++++------- internal/dnsforward/http_test.go | 108 ++++++++--- internal/dnsforward/upstreams.go | 311 ++++++++++++++++++++++++++++++ internal/filtering/filtering.go | 2 +- 8 files changed, 502 insertions(+), 205 deletions(-) create mode 100644 internal/dnsforward/upstreams.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bf94940..33baeb22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -143,6 +143,8 @@ In this release, the schema version has changed from 20 to 23. ### Fixed +- Using of `/etc/hosts` file to resolve the hostnames of upstream DNS servers + ([#5902]). - Excessive error logging when using DNS-over-QUIC ([#5285]). - Cannot set `bind_host` in AdGuardHome.yaml (docker version)([#4231], [#4235]). - The blocklists can now be deleted properly ([#5700]). @@ -157,6 +159,7 @@ In this release, the schema version has changed from 20 to 23. [#4235]: https://github.com/AdguardTeam/AdGuardHome/pull/4235 [#5285]: https://github.com/AdguardTeam/AdGuardHome/issues/5285 [#5700]: https://github.com/AdguardTeam/AdGuardHome/issues/5700 +[#5902]: https://github.com/AdguardTeam/AdGuardHome/issues/5902 [#5910]: https://github.com/AdguardTeam/AdGuardHome/issues/5910 [#5913]: https://github.com/AdguardTeam/AdGuardHome/issues/5913 diff --git a/internal/aghnet/hostscontainer.go b/internal/aghnet/hostscontainer.go index 2f4f20b8..2fecbc6f 100644 --- a/internal/aghnet/hostscontainer.go +++ b/internal/aghnet/hostscontainer.go @@ -56,15 +56,20 @@ func (rm *requestMatcher) MatchRequest( ) (res *urlfilter.DNSResult, ok bool) { switch req.DNSType { case dns.TypeA, dns.TypeAAAA, dns.TypePTR: - log.Debug("%s: handling the request for %s", hostsContainerPrefix, req.Hostname) + log.Debug( + "%s: handling %s request for %s", + hostsContainerPrefix, + dns.Type(req.DNSType), + req.Hostname, + ) + + rm.stateLock.RLock() + defer rm.stateLock.RUnlock() + + return rm.engine.MatchRequest(req) default: return nil, false } - - rm.stateLock.RLock() - defer rm.stateLock.RUnlock() - - return rm.engine.MatchRequest(req) } // Translate returns the source hosts-syntax rule for the generated dnsrewrite @@ -96,6 +101,8 @@ const hostsContainerPrefix = "hosts container" // HostsContainer stores the relevant hosts database provided by the OS and // processes both A/AAAA and PTR DNS requests for those. +// +// TODO(e.burkov): Improve API and move to golibs. type HostsContainer struct { // requestMatcher matches the requests and translates the rules. It's // embedded to implement MatchRequest and Translate for *HostsContainer. diff --git a/internal/dnsforward/config.go b/internal/dnsforward/config.go index 994b91e9..aa3f4808 100644 --- a/internal/dnsforward/config.go +++ b/internal/dnsforward/config.go @@ -15,7 +15,6 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghtls" "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/dnsproxy/proxy" - "github.com/AdguardTeam/dnsproxy/upstream" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/netutil" @@ -436,102 +435,6 @@ func (s *Server) initDefaultSettings() { } } -// UpstreamHTTPVersions returns the HTTP versions for upstream configuration -// depending on configuration. -func UpstreamHTTPVersions(http3 bool) (v []upstream.HTTPVersion) { - if !http3 { - return upstream.DefaultHTTPVersions - } - - return []upstream.HTTPVersion{ - upstream.HTTPVersion3, - upstream.HTTPVersion2, - upstream.HTTPVersion11, - } -} - -// prepareUpstreamSettings - prepares upstream DNS server settings -func (s *Server) prepareUpstreamSettings() error { - // We're setting a customized set of RootCAs. The reason is that Go default - // mechanism of loading TLS roots does not always work properly on some - // routers so we're loading roots manually and pass it here. - // - // See [aghtls.SystemRootCAs]. - upstream.RootCAs = s.conf.TLSv12Roots - upstream.CipherSuites = s.conf.TLSCiphers - - // Load upstreams either from the file, or from the settings - var upstreams []string - if s.conf.UpstreamDNSFileName != "" { - data, err := os.ReadFile(s.conf.UpstreamDNSFileName) - if err != nil { - return fmt.Errorf("reading upstream from file: %w", err) - } - - upstreams = stringutil.SplitTrimmed(string(data), "\n") - - log.Debug("dns: using %d upstream servers from file %s", len(upstreams), s.conf.UpstreamDNSFileName) - } else { - upstreams = s.conf.UpstreamDNS - } - - httpVersions := UpstreamHTTPVersions(s.conf.UseHTTP3Upstreams) - upstreams = stringutil.FilterOut(upstreams, IsCommentOrEmpty) - upstreamConfig, err := proxy.ParseUpstreamsConfig( - upstreams, - &upstream.Options{ - Bootstrap: s.conf.BootstrapDNS, - Timeout: s.conf.UpstreamTimeout, - HTTPVersions: httpVersions, - PreferIPv6: s.conf.BootstrapPreferIPv6, - }, - ) - if err != nil { - return fmt.Errorf("parsing upstream config: %w", err) - } - - if len(upstreamConfig.Upstreams) == 0 { - log.Info("warning: no default upstream servers specified, using %v", defaultDNS) - var uc *proxy.UpstreamConfig - uc, err = proxy.ParseUpstreamsConfig( - defaultDNS, - &upstream.Options{ - Bootstrap: s.conf.BootstrapDNS, - Timeout: s.conf.UpstreamTimeout, - HTTPVersions: httpVersions, - PreferIPv6: s.conf.BootstrapPreferIPv6, - }, - ) - if err != nil { - return fmt.Errorf("parsing default upstreams: %w", err) - } - - upstreamConfig.Upstreams = uc.Upstreams - } - - s.conf.UpstreamConfig = upstreamConfig - - return nil -} - -// setProxyUpstreamMode sets the upstream mode and related settings in conf -// based on provided parameters. -func setProxyUpstreamMode( - conf *proxy.Config, - allServers bool, - fastestAddr bool, - fastestTimeout time.Duration, -) { - if allServers { - conf.UpstreamMode = proxy.UModeParallel - } else if fastestAddr { - conf.UpstreamMode = proxy.UModeFastestAddr - conf.FastestPingTimeout = fastestTimeout - } else { - conf.UpstreamMode = proxy.UModeLoadBalance - } -} - // prepareIpsetListSettings reads and prepares the ipset configuration either // from a file or from the data in the configuration file. func (s *Server) prepareIpsetListSettings() (err error) { diff --git a/internal/dnsforward/dnsforward.go b/internal/dnsforward/dnsforward.go index 34f01884..a3f9fa73 100644 --- a/internal/dnsforward/dnsforward.go +++ b/internal/dnsforward/dnsforward.go @@ -466,19 +466,15 @@ func (s *Server) setupResolvers(localAddrs []string) (err error) { log.Debug("dnsforward: upstreams to resolve ptr for local addresses: %v", localAddrs) - var upsConfig *proxy.UpstreamConfig - upsConfig, err = proxy.ParseUpstreamsConfig( - localAddrs, - &upstream.Options{ - Bootstrap: bootstraps, - Timeout: defaultLocalTimeout, - // TODO(e.burkov): Should we verify server's certificates? + upsConfig, err := s.prepareUpstreamConfig(localAddrs, nil, &upstream.Options{ + Bootstrap: bootstraps, + Timeout: defaultLocalTimeout, + // TODO(e.burkov): Should we verify server's certificates? - PreferIPv6: s.conf.BootstrapPreferIPv6, - }, - ) + PreferIPv6: s.conf.BootstrapPreferIPv6, + }) if err != nil { - return fmt.Errorf("parsing upstreams: %w", err) + return fmt.Errorf("parsing private upstreams: %w", err) } s.localResolvers = &proxy.Proxy{ @@ -510,7 +506,8 @@ func (s *Server) Prepare(conf *ServerConfig) (err error) { err = s.prepareUpstreamSettings() if err != nil { - return fmt.Errorf("preparing upstream settings: %w", err) + // Don't wrap the error, because it's informative enough as is. + return err } var proxyConfig proxy.Config diff --git a/internal/dnsforward/http.go b/internal/dnsforward/http.go index 3cf28e4b..e95459c7 100644 --- a/internal/dnsforward/http.go +++ b/internal/dnsforward/http.go @@ -633,61 +633,70 @@ func (err domainSpecificTestError) Error() (msg string) { return fmt.Sprintf("WARNING: %s", err.error) } -// checkDNS checks the upstream server defined by upstreamConfigStr using -// healthCheck for actually exchange messages. It uses bootstrap to resolve the -// upstream's address. -func checkDNS( - upstreamConfigStr string, - bootstrap []string, - bootstrapPrefIPv6 bool, - timeout time.Duration, - healthCheck healthCheckFunc, -) (err error) { - if IsCommentOrEmpty(upstreamConfigStr) { - return nil +// parseUpstreamLine parses line and creates the [upstream.Upstream] using opts +// and information from [s.dnsFilter.EtcHosts]. It returns an error if the line +// is not a valid upstream line, see [upstream.AddressToUpstream]. It's a +// caller's responsibility to close u. +func (s *Server) parseUpstreamLine( + line string, + opts *upstream.Options, +) (u upstream.Upstream, specific bool, err error) { + // Separate upstream from domains list. + upstreamAddr, domains, err := separateUpstream(line) + if err != nil { + return nil, false, fmt.Errorf("wrong upstream format: %w", err) } - // Separate upstream from domains list. - upstreamAddr, domains, err := separateUpstream(upstreamConfigStr) - if err != nil { - return fmt.Errorf("wrong upstream format: %w", err) - } + specific = len(domains) > 0 useDefault, err := validateUpstream(upstreamAddr, domains) if err != nil { - return fmt.Errorf("wrong upstream format: %w", err) + return nil, specific, fmt.Errorf("wrong upstream format: %w", err) } else if useDefault { - return nil - } - - if len(bootstrap) == 0 { - bootstrap = defaultBootstrap + return nil, specific, nil } log.Debug("dnsforward: checking if upstream %q works", upstreamAddr) - u, err := upstream.AddressToUpstream(upstreamAddr, &upstream.Options{ - Bootstrap: bootstrap, - Timeout: timeout, - PreferIPv6: bootstrapPrefIPv6, - }) + opts = &upstream.Options{ + Bootstrap: opts.Bootstrap, + Timeout: opts.Timeout, + PreferIPv6: opts.PreferIPv6, + } + + if s.dnsFilter != nil && s.dnsFilter.EtcHosts != nil { + resolved := s.resolveUpstreamHost(extractUpstreamHost(upstreamAddr)) + sortNetIPAddrs(resolved, opts.PreferIPv6) + opts.ServerIPAddrs = resolved + } + u, err = upstream.AddressToUpstream(upstreamAddr, opts) if err != nil { - return fmt.Errorf("failed to choose upstream for %q: %w", upstreamAddr, err) + return nil, specific, fmt.Errorf("creating upstream for %q: %w", upstreamAddr, err) + } + + return u, specific, nil +} + +func (s *Server) checkDNS(line string, opts *upstream.Options, check healthCheckFunc) (err error) { + if IsCommentOrEmpty(line) { + return nil + } + + var u upstream.Upstream + var specific bool + defer func() { + if err != nil && specific { + err = domainSpecificTestError{error: err} + } + }() + + u, specific, err = s.parseUpstreamLine(line, opts) + if err != nil || u == nil { + return err } defer func() { err = errors.WithDeferred(err, u.Close()) }() - if err = healthCheck(u); err != nil { - err = fmt.Errorf("upstream %q fails to exchange: %w", upstreamAddr, err) - if domains != nil { - return domainSpecificTestError{error: err} - } - - return err - } - - log.Debug("dnsforward: upstream %q is ok", upstreamAddr) - - return nil + return check(u) } func (s *Server) handleTestUpstreamDNS(w http.ResponseWriter, r *http.Request) { @@ -699,47 +708,54 @@ func (s *Server) handleTestUpstreamDNS(w http.ResponseWriter, r *http.Request) { return } - result := map[string]string{} - bootstraps := req.BootstrapDNS - bootstrapPrefIPv6 := s.conf.BootstrapPreferIPv6 - timeout := s.conf.UpstreamTimeout + opts := &upstream.Options{ + Bootstrap: req.BootstrapDNS, + Timeout: s.conf.UpstreamTimeout, + PreferIPv6: s.conf.BootstrapPreferIPv6, + } + if len(opts.Bootstrap) == 0 { + opts.Bootstrap = defaultBootstrap + } type upsCheckResult = struct { - res string + err error host string } + req.Upstreams = stringutil.FilterOut(req.Upstreams, IsCommentOrEmpty) + req.PrivateUpstreams = stringutil.FilterOut(req.PrivateUpstreams, IsCommentOrEmpty) + upsNum := len(req.Upstreams) + len(req.PrivateUpstreams) + result := make(map[string]string, upsNum) resCh := make(chan upsCheckResult, upsNum) - checkUps := func(ups string, healthCheck healthCheckFunc) { - res := upsCheckResult{ - host: ups, - } - defer func() { resCh <- res }() - - checkErr := checkDNS(ups, bootstraps, bootstrapPrefIPv6, timeout, healthCheck) - if checkErr != nil { - res.res = checkErr.Error() - } else { - res.res = "OK" - } - } - for _, ups := range req.Upstreams { - go checkUps(ups, checkDNSUpstreamExc) + go func(ups string) { + resCh <- upsCheckResult{ + host: ups, + err: s.checkDNS(ups, opts, checkDNSUpstreamExc), + } + }(ups) } for _, ups := range req.PrivateUpstreams { - go checkUps(ups, checkPrivateUpstreamExc) + go func(ups string) { + resCh <- upsCheckResult{ + host: ups, + err: s.checkDNS(ups, opts, checkPrivateUpstreamExc), + } + }(ups) } for i := 0; i < upsNum; i++ { - pair := <-resCh // TODO(e.burkov): The upstreams used for both common and private // resolving should be reported separately. - result[pair.host] = pair.res + pair := <-resCh + if pair.err != nil { + result[pair.host] = pair.err.Error() + } else { + result[pair.host] = "OK" + } } - close(resCh) _ = aghhttp.WriteJSONResponse(w, r, result) } diff --git a/internal/dnsforward/http_test.go b/internal/dnsforward/http_test.go index c9846ae4..38d8a766 100644 --- a/internal/dnsforward/http_test.go +++ b/internal/dnsforward/http_test.go @@ -13,10 +13,12 @@ import ( "path/filepath" "strings" "testing" + "testing/fstest" "time" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/aghnet" + "github.com/AdguardTeam/AdGuardHome/internal/aghtest" "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/golibs/httphdr" "github.com/AdguardTeam/golibs/netutil" @@ -280,6 +282,10 @@ func TestIsCommentOrEmpty(t *testing.T) { } func TestValidateUpstreams(t *testing.T) { + const sdnsStamp = `sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_J` + + `S3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczE` + + `uYWRndWFyZC5jb20` + testCases := []struct { name string wantErr string @@ -300,7 +306,7 @@ func TestValidateUpstreams(t *testing.T) { "[//]tls://1.1.1.1", "[/www.host.com/]#", "[/host.com/google.com/]8.8.8.8", - "[/host/]sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20", + "[/host/]" + sdnsStamp, }, }, { name: "with_default", @@ -310,7 +316,7 @@ func TestValidateUpstreams(t *testing.T) { "[//]tls://1.1.1.1", "[/www.host.com/]#", "[/host.com/google.com/]8.8.8.8", - "[/host/]sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20", + "[/host/]" + sdnsStamp, "8.8.8.8", }, }, { @@ -326,9 +332,10 @@ func TestValidateUpstreams(t *testing.T) { wantErr: `validating upstream "123.3.7m": not an ip:port`, set: []string{"123.3.7m"}, }, { - name: "invalid", - wantErr: `bad upstream for domain "[/host.com]tls://dns.adguard.com": missing separator`, - set: []string{"[/host.com]tls://dns.adguard.com"}, + name: "invalid", + wantErr: `bad upstream for domain "[/host.com]tls://dns.adguard.com": ` + + `missing separator`, + set: []string{"[/host.com]tls://dns.adguard.com"}, }, { name: "invalid", wantErr: `validating upstream "[host.ru]#": not an ip:port`, @@ -340,14 +347,14 @@ func TestValidateUpstreams(t *testing.T) { "1.1.1.1", "tls://1.1.1.1", "https://dns.adguard.com/dns-query", - "sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20", + sdnsStamp, "udp://dns.google", "udp://8.8.8.8", "[/host.com/]1.1.1.1", "[//]tls://1.1.1.1", "[/www.host.com/]#", "[/host.com/google.com/]8.8.8.8", - "[/host/]sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20", + "[/host/]" + sdnsStamp, "[/пример.рф/]8.8.8.8", }, }, { @@ -418,27 +425,28 @@ func TestValidateUpstreamsPrivate(t *testing.T) { } } -func newLocalUpstreamListener(t *testing.T, port int, handler dns.Handler) (real net.Addr) { +func newLocalUpstreamListener(t *testing.T, port uint16, handler dns.Handler) (real netip.AddrPort) { + t.Helper() + startCh := make(chan struct{}) upsSrv := &dns.Server{ - Addr: netip.AddrPortFrom(netutil.IPv4Localhost(), uint16(port)).String(), + Addr: netip.AddrPortFrom(netutil.IPv4Localhost(), port).String(), Net: "tcp", Handler: handler, NotifyStartedFunc: func() { close(startCh) }, } go func() { - t := testutil.PanicT{} - err := upsSrv.ListenAndServe() - require.NoError(t, err) + require.NoError(testutil.PanicT{}, err) }() + <-startCh testutil.CleanupAndRequireSuccess(t, upsSrv.Shutdown) - return upsSrv.Listener.Addr() + return testutil.RequireTypeAssert[*net.TCPAddr](t, upsSrv.Listener.Addr()).AddrPort() } -func TestServer_handleTestUpstreaDNS(t *testing.T) { +func TestServer_HandleTestUpstreamDNS(t *testing.T) { goodHandler := dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) { err := w.WriteMsg(new(dns.Msg).SetReply(m)) require.NoError(testutil.PanicT{}, err) @@ -457,9 +465,38 @@ func TestServer_handleTestUpstreaDNS(t *testing.T) { Host: newLocalUpstreamListener(t, 0, badHandler).String(), }).String() - const upsTimeout = 100 * time.Millisecond + const ( + upsTimeout = 100 * time.Millisecond - srv := createTestServer(t, &filtering.Config{}, ServerConfig{ + hostsFileName = "hosts" + upstreamHost = "custom.localhost" + ) + + hostsListener := newLocalUpstreamListener(t, 0, goodHandler) + hostsUps := (&url.URL{ + Scheme: "tcp", + Host: netutil.JoinHostPort(upstreamHost, int(hostsListener.Port())), + }).String() + + hc, err := aghnet.NewHostsContainer( + filtering.SysHostsListID, + fstest.MapFS{ + hostsFileName: &fstest.MapFile{ + Data: []byte(hostsListener.Addr().String() + " " + upstreamHost), + }, + }, + &aghtest.FSWatcher{ + OnEvents: func() (e <-chan struct{}) { return nil }, + OnAdd: func(_ string) (err error) { return nil }, + OnClose: func() (err error) { return nil }, + }, + hostsFileName, + ) + require.NoError(t, err) + + srv := createTestServer(t, &filtering.Config{ + EtcHosts: hc, + }, ServerConfig{ UDPListenAddrs: []*net.UDPAddr{{}}, TCPListenAddrs: []*net.TCPAddr{{}}, UpstreamTimeout: upsTimeout, @@ -486,8 +523,7 @@ func TestServer_handleTestUpstreaDNS(t *testing.T) { "upstream_dns": []string{badUps}, }, wantResp: map[string]any{ - badUps: `upstream "` + badUps + `" fails to exchange: ` + - `couldn't communicate with upstream: exchanging with ` + + badUps: `couldn't communicate with upstream: exchanging with ` + badUps + ` over tcp: dns: id mismatch`, }, name: "broken", @@ -497,20 +533,40 @@ func TestServer_handleTestUpstreaDNS(t *testing.T) { }, wantResp: map[string]any{ goodUps: "OK", - badUps: `upstream "` + badUps + `" fails to exchange: ` + - `couldn't communicate with upstream: exchanging with ` + + badUps: `couldn't communicate with upstream: exchanging with ` + badUps + ` over tcp: dns: id mismatch`, }, name: "both", + }, { + body: map[string]any{ + "upstream_dns": []string{"[/domain.example/]" + badUps}, + }, + wantResp: map[string]any{ + "[/domain.example/]" + badUps: `WARNING: couldn't communicate ` + + `with upstream: exchanging with ` + badUps + ` over tcp: ` + + `dns: id mismatch`, + }, + name: "domain_specific_error", + }, { + body: map[string]any{ + "upstream_dns": []string{hostsUps}, + }, + wantResp: map[string]any{ + hostsUps: "OK", + }, + name: "etc_hosts", }} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - reqBody, err := json.Marshal(tc.body) + var reqBody []byte + reqBody, err = json.Marshal(tc.body) require.NoError(t, err) w := httptest.NewRecorder() - r, err := http.NewRequest(http.MethodPost, "", bytes.NewReader(reqBody)) + + var r *http.Request + r, err = http.NewRequest(http.MethodPost, "", bytes.NewReader(reqBody)) require.NoError(t, err) srv.handleTestUpstreamDNS(w, r) @@ -538,11 +594,15 @@ func TestServer_handleTestUpstreaDNS(t *testing.T) { req := map[string]any{ "upstream_dns": []string{sleepyUps}, } - reqBody, err := json.Marshal(req) + + var reqBody []byte + reqBody, err = json.Marshal(req) require.NoError(t, err) w := httptest.NewRecorder() - r, err := http.NewRequest(http.MethodPost, "", bytes.NewReader(reqBody)) + + var r *http.Request + r, err = http.NewRequest(http.MethodPost, "", bytes.NewReader(reqBody)) require.NoError(t, err) srv.handleTestUpstreamDNS(w, r) diff --git a/internal/dnsforward/upstreams.go b/internal/dnsforward/upstreams.go new file mode 100644 index 00000000..cbd92b36 --- /dev/null +++ b/internal/dnsforward/upstreams.go @@ -0,0 +1,311 @@ +package dnsforward + +import ( + "bytes" + "fmt" + "net" + "net/url" + "os" + "strings" + "time" + + "github.com/AdguardTeam/dnsproxy/proxy" + "github.com/AdguardTeam/dnsproxy/upstream" + "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/netutil" + "github.com/AdguardTeam/golibs/stringutil" + "github.com/AdguardTeam/urlfilter" + "github.com/miekg/dns" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" +) + +// loadUpstreams parses upstream DNS servers from the configured file or from +// the configuration itself. +func (s *Server) loadUpstreams() (upstreams []string, err error) { + if s.conf.UpstreamDNSFileName == "" { + return stringutil.FilterOut(s.conf.UpstreamDNS, IsCommentOrEmpty), nil + } + + var data []byte + data, err = os.ReadFile(s.conf.UpstreamDNSFileName) + if err != nil { + return nil, fmt.Errorf("reading upstream from file: %w", err) + } + + upstreams = stringutil.SplitTrimmed(string(data), "\n") + + log.Debug("dnsforward: got %d upstreams in %q", len(upstreams), s.conf.UpstreamDNSFileName) + + return stringutil.FilterOut(upstreams, IsCommentOrEmpty), nil +} + +// prepareUpstreamSettings sets upstream DNS server settings. +func (s *Server) prepareUpstreamSettings() (err error) { + // We're setting a customized set of RootCAs. The reason is that Go default + // mechanism of loading TLS roots does not always work properly on some + // routers so we're loading roots manually and pass it here. + // + // See [aghtls.SystemRootCAs]. + upstream.RootCAs = s.conf.TLSv12Roots + upstream.CipherSuites = s.conf.TLSCiphers + + // Load upstreams either from the file, or from the settings + var upstreams []string + upstreams, err = s.loadUpstreams() + if err != nil { + return fmt.Errorf("loading upstreams: %w", err) + } + + s.conf.UpstreamConfig, err = s.prepareUpstreamConfig(upstreams, defaultDNS, &upstream.Options{ + Bootstrap: s.conf.BootstrapDNS, + Timeout: s.conf.UpstreamTimeout, + HTTPVersions: UpstreamHTTPVersions(s.conf.UseHTTP3Upstreams), + PreferIPv6: s.conf.BootstrapPreferIPv6, + }) + if err != nil { + return fmt.Errorf("preparing upstream config: %w", err) + } + + return nil +} + +// prepareUpstreamConfig sets upstream configuration based on upstreams and +// configuration of s. +func (s *Server) prepareUpstreamConfig( + upstreams []string, + defaultUpstreams []string, + opts *upstream.Options, +) (uc *proxy.UpstreamConfig, err error) { + uc, err = proxy.ParseUpstreamsConfig(upstreams, opts) + if err != nil { + return nil, fmt.Errorf("parsing upstream config: %w", err) + } + + if len(uc.Upstreams) == 0 && defaultUpstreams != nil { + log.Info("dnsforward: warning: no default upstreams specified, using %v", defaultUpstreams) + var defaultUpstreamConfig *proxy.UpstreamConfig + defaultUpstreamConfig, err = proxy.ParseUpstreamsConfig(defaultUpstreams, opts) + if err != nil { + return nil, fmt.Errorf("parsing default upstreams: %w", err) + } + + uc.Upstreams = defaultUpstreamConfig.Upstreams + } + + if s.dnsFilter != nil && s.dnsFilter.EtcHosts != nil { + err = s.replaceUpstreamsWithHosts(uc, opts) + if err != nil { + return nil, fmt.Errorf("resolving upstreams with hosts: %w", err) + } + } + + return uc, nil +} + +// replaceUpstreamsWithHosts replaces unique upstreams with their resolved +// versions based on the system hosts file. +// +// TODO(e.burkov): This should be performed inside dnsproxy, which should +// actually consider /etc/hosts. See TODO on [aghnet.HostsContainer]. +func (s *Server) replaceUpstreamsWithHosts( + upsConf *proxy.UpstreamConfig, + opts *upstream.Options, +) (err error) { + resolved := map[string]*upstream.Options{} + + err = s.resolveUpstreamsWithHosts(resolved, upsConf.Upstreams, opts) + if err != nil { + return fmt.Errorf("resolving upstreams: %w", err) + } + + hosts := maps.Keys(upsConf.DomainReservedUpstreams) + // TODO(e.burkov): Think of extracting sorted range into an util function. + slices.Sort(hosts) + for _, host := range hosts { + err = s.resolveUpstreamsWithHosts(resolved, upsConf.DomainReservedUpstreams[host], opts) + if err != nil { + return fmt.Errorf("resolving upstreams reserved for %s: %w", host, err) + } + } + + hosts = maps.Keys(upsConf.SpecifiedDomainUpstreams) + slices.Sort(hosts) + for _, host := range hosts { + err = s.resolveUpstreamsWithHosts(resolved, upsConf.SpecifiedDomainUpstreams[host], opts) + if err != nil { + return fmt.Errorf("resolving upstreams specific for %s: %w", host, err) + } + } + + return nil +} + +// resolveUpstreamsWithHosts resolves the IP addresses of each of the upstreams +// and replaces those both in upstreams and resolved. Upstreams that failed to +// resolve are placed to resolved as-is. This function only returns error of +// upstreams closing. +func (s *Server) resolveUpstreamsWithHosts( + resolved map[string]*upstream.Options, + upstreams []upstream.Upstream, + opts *upstream.Options, +) (err error) { + for i := range upstreams { + u := upstreams[i] + addr := u.Address() + host := extractUpstreamHost(addr) + + withIPs, ok := resolved[host] + if !ok { + ips := s.resolveUpstreamHost(host) + if len(ips) == 0 { + resolved[host] = nil + + return nil + } + + sortNetIPAddrs(ips, opts.PreferIPv6) + + withIPs = opts.Clone() + withIPs.ServerIPAddrs = ips + resolved[host] = withIPs + } else if withIPs == nil { + continue + } + + if err = u.Close(); err != nil { + return fmt.Errorf("closing upstream %s: %w", addr, err) + } + + upstreams[i], err = upstream.AddressToUpstream(addr, withIPs) + if err != nil { + return fmt.Errorf("replacing upstream %s with resolved %s: %w", addr, host, err) + } + + log.Debug("dnsforward: using %s for %s", withIPs.ServerIPAddrs, upstreams[i].Address()) + } + + return nil +} + +// extractUpstreamHost returns the hostname of addr without port with an +// assumption that any address passed here has already been successfully parsed +// by [upstream.AddressToUpstream]. This function eesentially mirrors the logic +// of [upstream.AddressToUpstream], see TODO on [replaceUpstreamsWithHosts]. +func extractUpstreamHost(addr string) (host string) { + var err error + if strings.Contains(addr, "://") { + var u *url.URL + u, err = url.Parse(addr) + if err != nil { + log.Debug("dnsforward: parsing upstream %s: %s", addr, err) + + return addr + } + + return u.Hostname() + } + + // Probably, plain UDP upstream defined by address or address:port. + host, err = netutil.SplitHost(addr) + if err != nil { + return addr + } + + return host +} + +// resolveUpstreamHost returns the version of ups with IP addresses from the +// system hosts file placed into its options. +func (s *Server) resolveUpstreamHost(host string) (addrs []net.IP) { + req := &urlfilter.DNSRequest{ + Hostname: host, + DNSType: dns.TypeA, + } + aRes, _ := s.dnsFilter.EtcHosts.MatchRequest(req) + + req.DNSType = dns.TypeAAAA + aaaaRes, _ := s.dnsFilter.EtcHosts.MatchRequest(req) + + var ips []net.IP + for _, rw := range append(aRes.DNSRewrites(), aaaaRes.DNSRewrites()...) { + dr := rw.DNSRewrite + if dr == nil || dr.Value == nil { + continue + } + + if ip, ok := dr.Value.(net.IP); ok { + ips = append(ips, ip) + } + } + + return ips +} + +// sortNetIPAddrs sorts addrs in accordance with the protocol preferences. +// Invalid addresses are sorted near the end. +// +// TODO(e.burkov): This function taken from dnsproxy, which also already +// contains a few similar functions. Think of moving to golibs. +func sortNetIPAddrs(addrs []net.IP, preferIPv6 bool) { + l := len(addrs) + if l <= 1 { + return + } + + slices.SortStableFunc(addrs, func(addrA, addrB net.IP) (sortsBefore bool) { + switch len(addrA) { + case net.IPv4len, net.IPv6len: + switch len(addrB) { + case net.IPv4len, net.IPv6len: + // Go on. + default: + return true + } + default: + return false + } + + if aIs4, bIs4 := addrA.To4() != nil, addrB.To4() != nil; aIs4 != bIs4 { + if aIs4 { + return !preferIPv6 + } + + return preferIPv6 + } + + return bytes.Compare(addrA, addrB) < 0 + }) +} + +// UpstreamHTTPVersions returns the HTTP versions for upstream configuration +// depending on configuration. +func UpstreamHTTPVersions(http3 bool) (v []upstream.HTTPVersion) { + if !http3 { + return upstream.DefaultHTTPVersions + } + + return []upstream.HTTPVersion{ + upstream.HTTPVersion3, + upstream.HTTPVersion2, + upstream.HTTPVersion11, + } +} + +// setProxyUpstreamMode sets the upstream mode and related settings in conf +// based on provided parameters. +func setProxyUpstreamMode( + conf *proxy.Config, + allServers bool, + fastestAddr bool, + fastestTimeout time.Duration, +) { + if allServers { + conf.UpstreamMode = proxy.UModeParallel + } else if fastestAddr { + conf.UpstreamMode = proxy.UModeFastestAddr + conf.FastestPingTimeout = fastestTimeout + } else { + conf.UpstreamMode = proxy.UModeLoadBalance + } +} diff --git a/internal/filtering/filtering.go b/internal/filtering/filtering.go index 7cad6c99..e67c3a39 100644 --- a/internal/filtering/filtering.go +++ b/internal/filtering/filtering.go @@ -519,7 +519,7 @@ func (d *DNSFilter) matchSysHosts( dnsres, _ := d.EtcHosts.MatchRequest(&urlfilter.DNSRequest{ Hostname: host, SortedClientTags: setts.ClientTags, - // TODO(e.burkov): Wait for urlfilter update to pass net.IP. + // TODO(e.burkov): Wait for urlfilter update to pass netip.Addr. ClientIP: setts.ClientIP.String(), ClientName: setts.ClientName, DNSType: qtype,