From 4b4036fa6a777fb1fc039ee98076a8153d80e284 Mon Sep 17 00:00:00 2001 From: Eugene Burkov Date: Tue, 22 Aug 2023 16:45:11 +0300 Subject: [PATCH] Pull request 1964: AG-23599 use hostsfile Merge in DNS/adguard-home from AG-23599-use-hostsfile to master Squashed commit of the following: commit 4766e67a9d5faa4bc89a2a935d187ce4829f7214 Merge: 38369360b 762e5be97 Author: Eugene Burkov Date: Tue Aug 22 16:33:54 2023 +0300 Merge branch 'master' into AG-23599-use-hostsfile commit 38369360b7d0e5c9ec373c5a06bac8792ca9cd69 Author: Eugene Burkov Date: Mon Aug 21 18:09:15 2023 +0300 filtering: imp tests commit 1c4d4a9f9639f048173e1c949f39f9ecb6ed0347 Author: Eugene Burkov Date: Mon Aug 21 14:00:10 2023 +0300 filtering: imp cognit, cyclo commit c50c33d7240c2812a715759fabf140e02184b729 Author: Eugene Burkov Date: Mon Aug 21 12:57:31 2023 +0300 filtering: imp code commit 92203b16719a717a2946c0401e166b1b38ddb7bc Author: Eugene Burkov Date: Fri Aug 18 17:39:11 2023 +0300 all: imp code, docs commit 523e8cd50f9136feede657385b7274fa6ba64131 Author: Eugene Burkov Date: Thu Aug 17 15:14:02 2023 +0300 all: fix ipv6 commit 6ce4537132615cbdc34a0b1f326fedd2b63c355d Author: Eugene Burkov Date: Thu Aug 17 14:17:27 2023 +0300 all: rm urlfilter from hosts commit d6666e851680c7e586325ea5970e0356ab919074 Author: Eugene Burkov Date: Wed Aug 16 15:09:52 2023 +0300 WIP commit 4a2732960558bef6636d3c428bad4c7c830016ca Author: Eugene Burkov Date: Wed Aug 16 14:47:13 2023 +0300 all: use hostsfile --- internal/aghnet/hostscontainer.go | 407 ++++---------- .../aghnet/hostscontainer_internal_test.go | 68 --- internal/aghnet/hostscontainer_test.go | 497 ++++++++---------- internal/dnsforward/dnsforward_test.go | 2 +- internal/dnsforward/http.go | 8 +- internal/dnsforward/http_test.go | 1 - internal/dnsforward/upstreams.go | 42 +- internal/filtering/dnsrewrite.go | 60 +++ internal/filtering/dnsrewrite_test.go | 158 ++++++ internal/filtering/filtering.go | 106 ++-- internal/filtering/filtering_test.go | 2 +- internal/filtering/rewrites.go | 30 ++ internal/home/clients.go | 10 +- internal/home/home.go | 1 - 14 files changed, 639 insertions(+), 753 deletions(-) diff --git a/internal/aghnet/hostscontainer.go b/internal/aghnet/hostscontainer.go index f2e57c46..79f89594 100644 --- a/internal/aghnet/hostscontainer.go +++ b/internal/aghnet/hostscontainer.go @@ -1,25 +1,19 @@ package aghnet import ( - "bufio" "fmt" "io" "io/fs" "net/netip" "path" - "strings" - "sync" + "sync/atomic" "github.com/AdguardTeam/AdGuardHome/internal/aghos" "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/hostsfile" "github.com/AdguardTeam/golibs/log" - "github.com/AdguardTeam/golibs/netutil" - "github.com/AdguardTeam/golibs/stringutil" - "github.com/AdguardTeam/urlfilter" - "github.com/AdguardTeam/urlfilter/filterlist" - "github.com/AdguardTeam/urlfilter/rules" - "github.com/miekg/dns" "golang.org/x/exp/maps" + "golang.org/x/exp/slices" ) // DefaultHostsPaths returns the slice of paths default for the operating system @@ -29,95 +23,51 @@ func DefaultHostsPaths() (paths []string) { return defaultHostsPaths() } -// requestMatcher combines the logic for matching requests and translating the -// appropriate rules. -type requestMatcher struct { - // stateLock protects all the fields of requestMatcher. - stateLock *sync.RWMutex - - // rulesStrg stores the rules obtained from the hosts' file. - rulesStrg *filterlist.RuleStorage - // engine serves rulesStrg. - engine *urlfilter.DNSEngine - - // translator maps generated $dnsrewrite rules into hosts-syntax rules. - // - // TODO(e.burkov): Store the filename from which the rule was parsed. - translator map[string]string -} - -// MatchRequest processes the request rewriting hostnames and addresses read -// from the operating system's hosts files. res is nil for any request having -// not an A/AAAA or PTR type, see man 5 hosts. -// -// It's safe for concurrent use. -func (rm *requestMatcher) MatchRequest( - req *urlfilter.DNSRequest, -) (res *urlfilter.DNSResult, ok bool) { - switch req.DNSType { - case dns.TypeA, dns.TypeAAAA, dns.TypePTR: - log.Debug( - "%s: handling %s request for %s", - hostsContainerPrefix, - dns.Type(req.DNSType), - req.Hostname, - ) - - rm.stateLock.RLock() - defer rm.stateLock.RUnlock() - - return rm.engine.MatchRequest(req) - default: - return nil, false +// MatchAddr returns the records for the IP address. +func (hc *HostsContainer) MatchAddr(ip netip.Addr) (recs []*hostsfile.Record) { + cur := hc.current.Load() + if cur == nil { + return nil } + + return cur.addrs[ip] } -// Translate returns the source hosts-syntax rule for the generated dnsrewrite -// rule or an empty string if the last doesn't exist. The returned rules are in -// a processed format like: -// -// ip host1 host2 ... -func (rm *requestMatcher) Translate(rule string) (hostRule string) { - rm.stateLock.RLock() - defer rm.stateLock.RUnlock() +// MatchName returns the records for the hostname. +func (hc *HostsContainer) MatchName(name string) (recs []*hostsfile.Record) { + cur := hc.current.Load() + if cur != nil { + recs = cur.names[name] + } - return rm.translator[rule] -} - -// resetEng updates container's engine and the translation map. -func (rm *requestMatcher) resetEng(rulesStrg *filterlist.RuleStorage, tr map[string]string) { - rm.stateLock.Lock() - defer rm.stateLock.Unlock() - - rm.rulesStrg = rulesStrg - rm.engine = urlfilter.NewDNSEngine(rm.rulesStrg) - - rm.translator = tr + return recs } // hostsContainerPrefix is a prefix for logging and wrapping errors in // HostsContainer's methods. const hostsContainerPrefix = "hosts container" +// Hosts is a map of IP addresses to the records, as it primarily stored in the +// [HostsContainer]. It should not be accessed for writing since it may be read +// concurrently, users should clone it before modifying. +// +// The order of records for each address is preserved from original files, but +// the order of the addresses, being a map key, is not. +// +// TODO(e.burkov): Probably, this should be a sorted slice of records. +type Hosts map[netip.Addr][]*hostsfile.Record + // HostsContainer stores the relevant hosts database provided by the OS and // processes both A/AAAA and PTR DNS requests for those. -// -// TODO(e.burkov): Improve API and move to golibs. type HostsContainer struct { - // requestMatcher matches the requests and translates the rules. It's - // embedded to implement MatchRequest and Translate for *HostsContainer. - // - // TODO(a.garipov, e.burkov): Consider fully merging into HostsContainer. - requestMatcher - // done is the channel to sign closing the container. done chan struct{} // updates is the channel for receiving updated hosts. - updates chan HostsRecords + updates chan Hosts - // last is the set of hosts that was cached within last detected change. - last HostsRecords + // current is the last set of hosts parsed. + current atomic.Pointer[hostsIndex] // fsys is the working file system to read hosts files from. fsys fs.FS @@ -127,30 +77,6 @@ type HostsContainer struct { // patterns stores specified paths in the fs.Glob-compatible form. patterns []string - - // listID is the identifier for the list of generated rules. - listID int -} - -// HostsRecords is a mapping of an IP address to its hosts data. -type HostsRecords map[netip.Addr]*HostsRecord - -// HostsRecord represents a single hosts file record. -type HostsRecord struct { - Aliases *stringutil.Set - Canonical string -} - -// Equal returns true if all fields of rec are equal to field in other or they -// both are nil. -func (rec *HostsRecord) Equal(other *HostsRecord) (ok bool) { - if rec == nil { - return other == nil - } else if other == nil { - return false - } - - return rec.Canonical == other.Canonical && rec.Aliases.Equal(other.Aliases) } // ErrNoHostsPaths is returned when there are no valid paths to watch passed to @@ -162,7 +88,6 @@ const ErrNoHostsPaths errors.Error = "no valid paths to hosts files provided" // shouldn't be empty and each of paths should locate either a file or a // directory in fsys. fsys and w must be non-nil. func NewHostsContainer( - listID int, fsys fs.FS, w aghos.FSWatcher, paths ...string, @@ -182,12 +107,8 @@ func NewHostsContainer( } hc = &HostsContainer{ - requestMatcher: requestMatcher{ - stateLock: &sync.RWMutex{}, - }, - listID: listID, done: make(chan struct{}, 1), - updates: make(chan HostsRecords, 1), + updates: make(chan Hosts, 1), fsys: fsys, watcher: w, patterns: patterns, @@ -233,7 +154,7 @@ func (hc *HostsContainer) Close() (err error) { } // Upd returns the channel into which the updates are sent. -func (hc *HostsContainer) Upd() (updates <-chan HostsRecords) { +func (hc *HostsContainer) Upd() (updates <-chan Hosts) { return hc.updates } @@ -288,198 +209,68 @@ func (hc *HostsContainer) handleEvents() { } } -// hostsParser is a helper type to parse rules from the operating system's hosts -// file. It exists for only a single refreshing session. -type hostsParser struct { - // rulesBuilder builds the resulting rules list content. - rulesBuilder *strings.Builder - - // translations maps generated rules into actual hosts file lines. - translations map[string]string - - // table stores only the unique IP-hostname pairs. It's also sent to the - // updates channel afterwards. - table HostsRecords -} - -// newHostsParser creates a new *hostsParser with buffers of size taken from the -// previous parse. -func (hc *HostsContainer) newHostsParser() (hp *hostsParser) { - return &hostsParser{ - rulesBuilder: &strings.Builder{}, - translations: map[string]string{}, - table: make(HostsRecords, len(hc.last)), - } -} - -// parseFile is a aghos.FileWalker for parsing the files with hosts syntax. It -// never signs to stop walking and never returns any additional patterns. -// -// See man hosts(5). -func (hp *hostsParser) parseFile(r io.Reader) (patterns []string, cont bool, err error) { - s := bufio.NewScanner(r) - for s.Scan() { - ip, hosts := hp.parseLine(s.Text()) - if ip == (netip.Addr{}) || len(hosts) == 0 { - continue - } - - hp.addRecord(ip, hosts) - } - - return nil, true, s.Err() -} - -// parseLine parses the line having the hosts syntax ignoring invalid ones. -func (hp *hostsParser) parseLine(line string) (ip netip.Addr, hosts []string) { - fields := strings.Fields(line) - if len(fields) < 2 { - return netip.Addr{}, nil - } - - ip, err := netip.ParseAddr(fields[0]) - if err != nil { - return netip.Addr{}, nil - } - - for _, f := range fields[1:] { - hashIdx := strings.IndexByte(f, '#') - if hashIdx == 0 { - // The rest of the fields are a part of the comment so return. - break - } else if hashIdx > 0 { - // Only a part of the field is a comment. - f = f[:hashIdx] - } - - // Make sure that invalid hosts aren't turned into rules. - // - // See https://github.com/AdguardTeam/AdGuardHome/issues/3946. - // - // TODO(e.burkov): Investigate if hosts may contain DNS-SD domains. - err = netutil.ValidateHostname(f) - if err != nil { - log.Error("%s: host %q is invalid, ignoring", hostsContainerPrefix, f) - - continue - } - - hosts = append(hosts, f) - } - - return ip, hosts -} - -// addRecord puts the record for the IP address to the rules builder if needed. -// The first host is considered to be the canonical name for the IP address. -// hosts must have at least one name. -func (hp *hostsParser) addRecord(ip netip.Addr, hosts []string) { - line := strings.Join(append([]string{ip.String()}, hosts...), " ") - - rec, ok := hp.table[ip] - if !ok { - rec = &HostsRecord{ - Aliases: stringutil.NewSet(), - } - - rec.Canonical, hosts = hosts[0], hosts[1:] - hp.addRules(ip, rec.Canonical, line) - hp.table[ip] = rec - } - - for _, host := range hosts { - if rec.Canonical == host || rec.Aliases.Has(host) { - continue - } - - rec.Aliases.Add(host) - - hp.addRules(ip, host, line) - } -} - -// addRules adds rules and rule translations for the line. -func (hp *hostsParser) addRules(ip netip.Addr, host, line string) { - rule, rulePtr := hp.writeRules(host, ip) - hp.translations[rule], hp.translations[rulePtr] = line, line - - log.Debug("%s: added ip-host pair %q-%q", hostsContainerPrefix, ip, host) -} - -// writeRules writes the actual rule for the qtype and the PTR for the host-ip -// pair into internal builders. -func (hp *hostsParser) writeRules(host string, ip netip.Addr) (rule, rulePtr string) { - // TODO(a.garipov): Add a netip.Addr version to netutil. - arpa, err := netutil.IPToReversedAddr(ip.AsSlice()) - if err != nil { - return "", "" - } - - const ( - nl = "\n" - - rwSuccess = "^$dnsrewrite=NOERROR;" - rwSuccessPTR = "^$dnsrewrite=NOERROR;PTR;" - - modLen = len(rules.MaskPipe) + len(rwSuccess) + len(";") - modLenPTR = len(rules.MaskPipe) + len(rwSuccessPTR) - ) - - var qtype string - // The validation of the IP address has been performed earlier so it is - // guaranteed to be either an IPv4 or an IPv6. - if ip.Is4() { - qtype = "A" - } else { - qtype = "AAAA" - } - - ipStr := ip.String() - fqdn := dns.Fqdn(host) - - ruleBuilder := &strings.Builder{} - ruleBuilder.Grow(modLen + len(host) + len(qtype) + len(ipStr)) - stringutil.WriteToBuilder(ruleBuilder, rules.MaskPipe, host, rwSuccess, qtype, ";", ipStr) - rule = ruleBuilder.String() - - ruleBuilder.Reset() - - ruleBuilder.Grow(modLenPTR + len(arpa) + len(fqdn)) - stringutil.WriteToBuilder(ruleBuilder, rules.MaskPipe, arpa, rwSuccessPTR, fqdn) - - rulePtr = ruleBuilder.String() - - hp.rulesBuilder.Grow(len(rule) + len(rulePtr) + 2*len(nl)) - stringutil.WriteToBuilder(hp.rulesBuilder, rule, nl, rulePtr, nl) - - return rule, rulePtr -} - // sendUpd tries to send the parsed data to the ch. -func (hp *hostsParser) sendUpd(ch chan HostsRecords) { +func (hc *HostsContainer) sendUpd(recs Hosts) { log.Debug("%s: sending upd", hostsContainerPrefix) - upd := hp.table + ch := hc.updates select { - case ch <- upd: + case ch <- recs: // Updates are delivered. Go on. case <-ch: - ch <- upd + ch <- recs log.Debug("%s: replaced the last update", hostsContainerPrefix) - case ch <- upd: + case ch <- recs: // The previous update was just read and the next one pushed. Go on. default: log.Error("%s: the updates channel is broken", hostsContainerPrefix) } } -// newStrg creates a new rules storage from parsed data. -func (hp *hostsParser) newStrg(id int) (s *filterlist.RuleStorage, err error) { - return filterlist.NewRuleStorage([]filterlist.RuleList{&filterlist.StringRuleList{ - ID: id, - RulesText: hp.rulesBuilder.String(), - IgnoreCosmetic: true, - }}) +// hostsIndex is a [hostsfile.Set] to enumerate all the records. +type hostsIndex struct { + // addrs maps IP addresses to the records. + addrs Hosts + + // names maps hostnames to the records. + names map[string][]*hostsfile.Record +} + +// walk is a file walking function for hostsIndex. +func (idx *hostsIndex) walk(r io.Reader) (patterns []string, cont bool, err error) { + return nil, true, hostsfile.Parse(idx, r, nil) +} + +// type check +var _ hostsfile.Set = (*hostsIndex)(nil) + +// Add puts the record for the IP address to the rules builder if needed. +// The first host is considered to be the canonical name for the IP address. +// hosts must have at least one name. +func (idx *hostsIndex) Add(rec *hostsfile.Record) { + idx.addrs[rec.Addr] = append(idx.addrs[rec.Addr], rec) + for _, name := range rec.Names { + idx.names[name] = append(idx.names[name], rec) + } +} + +// equalRecs is an equality function for [*hostsfile.Record]. +func equalRecs(a, b *hostsfile.Record) (ok bool) { + return a.Addr == b.Addr && a.Source == b.Source && slices.Equal(a.Names, b.Names) +} + +// equalRecSlices is an equality function for slices of [*hostsfile.Record]. +func equalRecSlices(a, b []*hostsfile.Record) (ok bool) { return slices.EqualFunc(a, b, equalRecs) } + +// Equal returns true if indexes are equal. +func (idx *hostsIndex) Equal(other *hostsIndex) (ok bool) { + if idx == nil { + return other == nil + } else if other == nil { + return false + } + + return maps.EqualFunc(idx.addrs, other.addrs, equalRecSlices) } // refresh gets the data from specified files and propagates the updates if @@ -489,27 +280,31 @@ func (hp *hostsParser) newStrg(id int) (s *filterlist.RuleStorage, err error) { func (hc *HostsContainer) refresh() (err error) { log.Debug("%s: refreshing", hostsContainerPrefix) - hp := hc.newHostsParser() - if _, err = aghos.FileWalker(hp.parseFile).Walk(hc.fsys, hc.patterns...); err != nil { - return fmt.Errorf("refreshing : %w", err) + var addrLen, nameLen int + last := hc.current.Load() + if last != nil { + addrLen, nameLen = len(last.addrs), len(last.names) + } + idx := &hostsIndex{ + addrs: make(Hosts, addrLen), + names: make(map[string][]*hostsfile.Record, nameLen), } - // hc.last is nil on the first refresh, so let that one through. - if hc.last != nil && maps.EqualFunc(hp.table, hc.last, (*HostsRecord).Equal) { - log.Debug("%s: no changes detected", hostsContainerPrefix) + _, err = aghos.FileWalker(idx.walk).Walk(hc.fsys, hc.patterns...) - return nil - } - defer hp.sendUpd(hc.updates) - - hc.last = maps.Clone(hp.table) - - var rulesStrg *filterlist.RuleStorage - if rulesStrg, err = hp.newStrg(hc.listID); err != nil { - return fmt.Errorf("initializing rules storage: %w", err) + if err != nil { + if len(idx.addrs) == 0 { + return fmt.Errorf("refreshing : %w", err) + } else { + log.Debug("%s: refreshing: %s", hostsContainerPrefix, err) + } } - hc.resetEng(rulesStrg, hp.translations) + // TODO(e.burkov): Serialize updates using time. + if !last.Equal(idx) { + hc.current.Store(idx) + hc.sendUpd(idx.addrs) + } return nil } diff --git a/internal/aghnet/hostscontainer_internal_test.go b/internal/aghnet/hostscontainer_internal_test.go index e3855f39..321c3dec 100644 --- a/internal/aghnet/hostscontainer_internal_test.go +++ b/internal/aghnet/hostscontainer_internal_test.go @@ -2,13 +2,11 @@ package aghnet import ( "io/fs" - "net/netip" "path" "testing" "testing/fstest" "github.com/AdguardTeam/golibs/errors" - "github.com/AdguardTeam/golibs/netutil" "github.com/AdguardTeam/golibs/testutil/fakefs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -76,69 +74,3 @@ func TestHostsContainer_PathsToPatterns(t *testing.T) { assert.ErrorIs(t, err, errStat) }) } - -func TestUniqueRules_ParseLine(t *testing.T) { - ip := netutil.IPv4Localhost() - ipStr := ip.String() - - testCases := []struct { - name string - line string - wantIP netip.Addr - wantHosts []string - }{{ - name: "simple", - line: ipStr + ` hostname`, - wantIP: ip, - wantHosts: []string{"hostname"}, - }, { - name: "aliases", - line: ipStr + ` hostname alias`, - wantIP: ip, - wantHosts: []string{"hostname", "alias"}, - }, { - name: "invalid_line", - line: ipStr, - wantIP: netip.Addr{}, - wantHosts: nil, - }, { - name: "invalid_line_hostname", - line: ipStr + ` # hostname`, - wantIP: ip, - wantHosts: nil, - }, { - name: "commented_aliases", - line: ipStr + ` hostname # alias`, - wantIP: ip, - wantHosts: []string{"hostname"}, - }, { - name: "whole_comment", - line: `# ` + ipStr + ` hostname`, - wantIP: netip.Addr{}, - wantHosts: nil, - }, { - name: "partial_comment", - line: ipStr + ` host#name`, - wantIP: ip, - wantHosts: []string{"host"}, - }, { - name: "empty", - line: ``, - wantIP: netip.Addr{}, - wantHosts: nil, - }, { - name: "bad_hosts", - line: ipStr + ` bad..host bad._tld empty.tld. ok.host`, - wantIP: ip, - wantHosts: []string{"ok.host"}, - }} - - for _, tc := range testCases { - hp := hostsParser{} - t.Run(tc.name, func(t *testing.T) { - got, hosts := hp.parseLine(tc.line) - assert.Equal(t, tc.wantIP, got) - assert.Equal(t, tc.wantHosts, hosts) - }) - } -} diff --git a/internal/aghnet/hostscontainer_test.go b/internal/aghnet/hostscontainer_test.go index 00c2aeed..6079ab72 100644 --- a/internal/aghnet/hostscontainer_test.go +++ b/internal/aghnet/hostscontainer_test.go @@ -1,9 +1,8 @@ package aghnet_test import ( - "net" + "net/netip" "path" - "strings" "sync/atomic" "testing" "testing/fstest" @@ -13,18 +12,144 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/AdGuardHome/internal/aghtest" "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/hostsfile" "github.com/AdguardTeam/golibs/netutil" - "github.com/AdguardTeam/golibs/stringutil" "github.com/AdguardTeam/golibs/testutil" - "github.com/AdguardTeam/urlfilter" - "github.com/AdguardTeam/urlfilter/rules" - "github.com/miekg/dns" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// nl is a newline character. const nl = "\n" +// Variables mirroring the etc_hosts file from testdata. +var ( + addr1000 = netip.MustParseAddr("1.0.0.0") + addr1001 = netip.MustParseAddr("1.0.0.1") + addr1002 = netip.MustParseAddr("1.0.0.2") + addr1003 = netip.MustParseAddr("1.0.0.3") + addr1004 = netip.MustParseAddr("1.0.0.4") + addr1357 = netip.MustParseAddr("1.3.5.7") + addr4216 = netip.MustParseAddr("4.2.1.6") + addr7531 = netip.MustParseAddr("7.5.3.1") + + addr0 = netip.MustParseAddr("::") + addr1 = netip.MustParseAddr("::1") + addr2 = netip.MustParseAddr("::2") + addr3 = netip.MustParseAddr("::3") + addr4 = netip.MustParseAddr("::4") + addr42 = netip.MustParseAddr("::42") + addr13 = netip.MustParseAddr("::13") + addr31 = netip.MustParseAddr("::31") + + testHosts = map[netip.Addr][]*hostsfile.Record{ + addr1000: {{ + Addr: addr1000, + Source: "./testdata/etc_hosts", + Names: []string{"hello", "hello.world"}, + }, { + Addr: addr1000, + Source: "./testdata/etc_hosts", + Names: []string{"hello.world.again"}, + }, { + Addr: addr1000, + Source: "./testdata/etc_hosts", + Names: []string{"hello.world"}, + }}, + addr1001: {{ + Addr: addr1001, + Source: "./testdata/etc_hosts", + Names: []string{"simplehost"}, + }, { + Addr: addr1001, + Source: "./testdata/etc_hosts", + Names: []string{"simplehost"}, + }}, + addr1002: {{ + Addr: addr1002, + Source: "./testdata/etc_hosts", + Names: []string{"a.whole", "lot.of", "aliases", "for.testing"}, + }}, + addr1003: {{ + Addr: addr1003, + Source: "./testdata/etc_hosts", + Names: []string{"*"}, + }}, + addr1004: {{ + Addr: addr1004, + Source: "./testdata/etc_hosts", + Names: []string{"*.com"}, + }}, + addr1357: {{ + Addr: addr1357, + Source: "./testdata/etc_hosts", + Names: []string{"domain4", "domain4.alias"}, + }}, + addr7531: {{ + Addr: addr7531, + Source: "./testdata/etc_hosts", + Names: []string{"domain4.alias", "domain4"}, + }}, + addr4216: {{ + Addr: addr4216, + Source: "./testdata/etc_hosts", + Names: []string{"domain", "domain.alias"}, + }}, + addr0: {{ + Addr: addr0, + Source: "./testdata/etc_hosts", + Names: []string{"hello", "hello.world"}, + }, { + Addr: addr0, + Source: "./testdata/etc_hosts", + Names: []string{"hello.world.again"}, + }, { + Addr: addr0, + Source: "./testdata/etc_hosts", + Names: []string{"hello.world"}, + }}, + addr1: {{ + Addr: addr1, + Source: "./testdata/etc_hosts", + Names: []string{"simplehost"}, + }, { + Addr: addr1, + Source: "./testdata/etc_hosts", + Names: []string{"simplehost"}, + }}, + addr2: {{ + Addr: addr2, + Source: "./testdata/etc_hosts", + Names: []string{"a.whole", "lot.of", "aliases", "for.testing"}, + }}, + addr3: {{ + Addr: addr3, + Source: "./testdata/etc_hosts", + Names: []string{"*"}, + }}, + addr4: {{ + Addr: addr4, + Source: "./testdata/etc_hosts", + Names: []string{"*.com"}, + }}, + addr42: {{ + Addr: addr42, + Source: "./testdata/etc_hosts", + Names: []string{"domain.alias", "domain"}, + }}, + addr13: {{ + Addr: addr13, + Source: "./testdata/etc_hosts", + Names: []string{"domain6", "domain6.alias"}, + }}, + addr31: {{ + Addr: addr31, + Source: "./testdata/etc_hosts", + Names: []string{"domain6.alias", "domain6"}, + }}, + } +) + func TestNewHostsContainer(t *testing.T) { const dirname = "dir" const filename = "file1" @@ -73,7 +198,7 @@ func TestNewHostsContainer(t *testing.T) { return eventsCh } - hc, err := aghnet.NewHostsContainer(0, testFS, &aghtest.FSWatcher{ + hc, err := aghnet.NewHostsContainer(testFS, &aghtest.FSWatcher{ OnEvents: onEvents, OnAdd: onAdd, OnClose: func() (err error) { return nil }, @@ -99,7 +224,7 @@ func TestNewHostsContainer(t *testing.T) { t.Run("nil_fs", func(t *testing.T) { require.Panics(t, func() { - _, _ = aghnet.NewHostsContainer(0, nil, &aghtest.FSWatcher{ + _, _ = aghnet.NewHostsContainer(nil, &aghtest.FSWatcher{ // Those shouldn't panic. OnEvents: func() (e <-chan struct{}) { return nil }, OnAdd: func(name string) (err error) { return nil }, @@ -110,7 +235,7 @@ func TestNewHostsContainer(t *testing.T) { t.Run("nil_watcher", func(t *testing.T) { require.Panics(t, func() { - _, _ = aghnet.NewHostsContainer(0, testFS, nil, p) + _, _ = aghnet.NewHostsContainer(testFS, nil, p) }) }) @@ -123,7 +248,7 @@ func TestNewHostsContainer(t *testing.T) { OnClose: func() (err error) { return nil }, } - hc, err := aghnet.NewHostsContainer(0, testFS, errWatcher, p) + hc, err := aghnet.NewHostsContainer(testFS, errWatcher, p) require.ErrorIs(t, err, errOnAdd) assert.Nil(t, hc) @@ -136,6 +261,9 @@ func TestHostsContainer_refresh(t *testing.T) { ip := netutil.IPv4Localhost() ipStr := ip.String() + anotherIPStr := "1.2.3.4" + anotherIP := netip.MustParseAddr(anotherIPStr) + testFS := fstest.MapFS{"dir/file1": &fstest.MapFile{Data: []byte(ipStr + ` hostname` + nl)}} // event is a convenient alias for an empty struct{} to emit test events. @@ -154,40 +282,44 @@ func TestHostsContainer_refresh(t *testing.T) { OnClose: func() (err error) { return nil }, } - hc, err := aghnet.NewHostsContainer(0, testFS, w, "dir") + hc, err := aghnet.NewHostsContainer(testFS, w, "dir") require.NoError(t, err) testutil.CleanupAndRequireSuccess(t, hc.Close) - checkRefresh := func(t *testing.T, want *aghnet.HostsRecord) { + checkRefresh := func(t *testing.T, want aghnet.Hosts) { t.Helper() upd, ok := aghchan.MustReceive(hc.Upd(), 1*time.Second) require.True(t, ok) - require.NotNil(t, upd) - assert.Len(t, upd, 1) - - rec, ok := upd[ip] - require.True(t, ok) - require.NotNil(t, rec) - - assert.Truef(t, rec.Equal(want), "%+v != %+v", rec, want) + assert.Equal(t, want, upd) } t.Run("initial_refresh", func(t *testing.T) { - checkRefresh(t, &aghnet.HostsRecord{ - Aliases: stringutil.NewSet(), - Canonical: "hostname", + checkRefresh(t, aghnet.Hosts{ + ip: {{ + Addr: ip, + Source: "file1", + Names: []string{"hostname"}, + }}, }) }) t.Run("second_refresh", func(t *testing.T) { - testFS["dir/file2"] = &fstest.MapFile{Data: []byte(ipStr + ` alias` + nl)} + testFS["dir/file2"] = &fstest.MapFile{Data: []byte(anotherIPStr + ` alias` + nl)} eventsCh <- event{} - checkRefresh(t, &aghnet.HostsRecord{ - Aliases: stringutil.NewSet("alias"), - Canonical: "hostname", + checkRefresh(t, aghnet.Hosts{ + ip: {{ + Addr: ip, + Source: "file1", + Names: []string{"hostname"}, + }}, + anotherIP: {{ + Addr: anotherIP, + Source: "file2", + Names: []string{"alias"}, + }}, }) }) @@ -198,12 +330,9 @@ func TestHostsContainer_refresh(t *testing.T) { // Require the changes are written. require.Eventually(t, func() bool { - res, ok := hc.MatchRequest(&urlfilter.DNSRequest{ - Hostname: "hostname", - DNSType: dns.TypeA, - }) + ips := hc.MatchName("hostname") - return !ok && res.DNSRewrites() == nil + return len(ips) == 0 }, 5*time.Second, time.Second/2) // Make a change again. @@ -212,285 +341,117 @@ func TestHostsContainer_refresh(t *testing.T) { // Require the changes are written. require.Eventually(t, func() bool { - res, ok := hc.MatchRequest(&urlfilter.DNSRequest{ - Hostname: "hostname", - DNSType: dns.TypeA, - }) + ips := hc.MatchName("hostname") - return !ok && res.DNSRewrites() != nil + return len(ips) > 0 }, 5*time.Second, time.Second/2) assert.Len(t, hc.Upd(), 1) }) } -func TestHostsContainer_Translate(t *testing.T) { +func TestHostsContainer_MatchName(t *testing.T) { + require.NoError(t, fstest.TestFS(testdata, "etc_hosts")) + stubWatcher := aghtest.FSWatcher{ OnEvents: func() (e <-chan struct{}) { return nil }, OnAdd: func(name string) (err error) { return nil }, OnClose: func() (err error) { return nil }, } - require.NoError(t, fstest.TestFS(testdata, "etc_hosts")) - - hc, err := aghnet.NewHostsContainer(0, testdata, &stubWatcher, "etc_hosts") - require.NoError(t, err) - testutil.CleanupAndRequireSuccess(t, hc.Close) - testCases := []struct { - name string - rule string - wantTrans []string - }{{ - name: "simplehost", - rule: "|simplehost^$dnsrewrite=NOERROR;A;1.0.0.1", - wantTrans: []string{"1.0.0.1", "simplehost"}, - }, { - name: "hello", - rule: "|hello^$dnsrewrite=NOERROR;A;1.0.0.0", - wantTrans: []string{"1.0.0.0", "hello", "hello.world"}, - }, { - name: "hello-alias", - rule: "|hello.world.again^$dnsrewrite=NOERROR;A;1.0.0.0", - wantTrans: []string{"1.0.0.0", "hello.world.again"}, - }, { - name: "simplehost_v6", - rule: "|simplehost^$dnsrewrite=NOERROR;AAAA;::1", - wantTrans: []string{"::1", "simplehost"}, - }, { - name: "hello_v6", - rule: "|hello^$dnsrewrite=NOERROR;AAAA;::", - wantTrans: []string{"::", "hello", "hello.world"}, - }, { - name: "hello_v6-alias", - rule: "|hello.world.again^$dnsrewrite=NOERROR;AAAA;::", - wantTrans: []string{"::", "hello.world.again"}, - }, { - name: "simplehost_ptr", - rule: "|1.0.0.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;simplehost.", - wantTrans: []string{"1.0.0.1", "simplehost"}, - }, { - name: "hello_ptr", - rule: "|0.0.0.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;hello.", - wantTrans: []string{"1.0.0.0", "hello", "hello.world"}, - }, { - name: "hello_ptr-alias", - rule: "|0.0.0.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;hello.world.again.", - wantTrans: []string{"1.0.0.0", "hello.world.again"}, - }, { - name: "simplehost_ptr_v6", - rule: "|1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" + - "^$dnsrewrite=NOERROR;PTR;simplehost.", - wantTrans: []string{"::1", "simplehost"}, - }, { - name: "hello_ptr_v6", - rule: "|0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" + - "^$dnsrewrite=NOERROR;PTR;hello.", - wantTrans: []string{"::", "hello", "hello.world"}, - }, { - name: "hello_ptr_v6-alias", - rule: "|0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" + - "^$dnsrewrite=NOERROR;PTR;hello.world.again.", - wantTrans: []string{"::", "hello.world.again"}, - }} - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - got := stringutil.NewSet(strings.Fields(hc.Translate(tc.rule))...) - assert.True(t, stringutil.NewSet(tc.wantTrans...).Equal(got)) - }) - } -} - -func TestHostsContainer(t *testing.T) { - const listID = 1234 - - require.NoError(t, fstest.TestFS(testdata, "etc_hosts")) - - testCases := []struct { - req *urlfilter.DNSRequest + req string name string - want []*rules.DNSRewrite + want []*hostsfile.Record }{{ - req: &urlfilter.DNSRequest{ - Hostname: "simplehost", - DNSType: dns.TypeA, - }, + req: "simplehost", name: "simple", - want: []*rules.DNSRewrite{{ - RCode: dns.RcodeSuccess, - Value: net.IPv4(1, 0, 0, 1), - RRType: dns.TypeA, - }, { - RCode: dns.RcodeSuccess, - Value: net.ParseIP("::1"), - RRType: dns.TypeAAAA, - }}, + want: append(testHosts[addr1001], testHosts[addr1]...), }, { - req: &urlfilter.DNSRequest{ - Hostname: "hello.world", - DNSType: dns.TypeA, - }, + req: "hello.world", name: "hello_alias", - want: []*rules.DNSRewrite{{ - RCode: dns.RcodeSuccess, - Value: net.IPv4(1, 0, 0, 0), - RRType: dns.TypeA, - }, { - RCode: dns.RcodeSuccess, - Value: net.ParseIP("::"), - RRType: dns.TypeAAAA, - }}, - }, { - req: &urlfilter.DNSRequest{ - Hostname: "hello.world.again", - DNSType: dns.TypeA, + want: []*hostsfile.Record{ + testHosts[addr1000][0], + testHosts[addr1000][2], + testHosts[addr0][0], + testHosts[addr0][2], }, + }, { + req: "hello.world.again", name: "other_line_alias", - want: []*rules.DNSRewrite{{ - RCode: dns.RcodeSuccess, - Value: net.IPv4(1, 0, 0, 0), - RRType: dns.TypeA, - }, { - RCode: dns.RcodeSuccess, - Value: net.ParseIP("::"), - RRType: dns.TypeAAAA, - }}, - }, { - req: &urlfilter.DNSRequest{ - Hostname: "say.hello", - DNSType: dns.TypeA, + want: []*hostsfile.Record{ + testHosts[addr1000][1], + testHosts[addr0][1], }, + }, { + req: "say.hello", name: "hello_subdomain", - want: []*rules.DNSRewrite{}, - }, { - req: &urlfilter.DNSRequest{ - Hostname: "say.hello.world", - DNSType: dns.TypeA, - }, - name: "hello_alias_subdomain", - want: []*rules.DNSRewrite{}, - }, { - req: &urlfilter.DNSRequest{ - Hostname: "for.testing", - DNSType: dns.TypeA, - }, - name: "lots_of_aliases", - want: []*rules.DNSRewrite{{ - RCode: dns.RcodeSuccess, - RRType: dns.TypeA, - Value: net.IPv4(1, 0, 0, 2), - }, { - RCode: dns.RcodeSuccess, - RRType: dns.TypeAAAA, - Value: net.ParseIP("::2"), - }}, - }, { - req: &urlfilter.DNSRequest{ - Hostname: "1.0.0.1.in-addr.arpa", - DNSType: dns.TypePTR, - }, - name: "reverse", - want: []*rules.DNSRewrite{{ - RCode: dns.RcodeSuccess, - RRType: dns.TypePTR, - Value: "simplehost.", - }}, - }, { - req: &urlfilter.DNSRequest{ - Hostname: "nonexistent.example", - DNSType: dns.TypeA, - }, - name: "non-existing", - want: []*rules.DNSRewrite{}, - }, { - req: &urlfilter.DNSRequest{ - Hostname: "1.0.0.1.in-addr.arpa", - DNSType: dns.TypeSRV, - }, - name: "bad_type", want: nil, }, { - req: &urlfilter.DNSRequest{ - Hostname: "domain", - DNSType: dns.TypeA, - }, + req: "say.hello.world", + name: "hello_alias_subdomain", + want: nil, + }, { + req: "for.testing", + name: "lots_of_aliases", + want: append(testHosts[addr1002], testHosts[addr2]...), + }, { + req: "nonexistent.example", + name: "non-existing", + want: nil, + }, { + req: "domain", name: "issue_4216_4_6", - want: []*rules.DNSRewrite{{ - RCode: dns.RcodeSuccess, - RRType: dns.TypeA, - Value: net.IPv4(4, 2, 1, 6), - }, { - RCode: dns.RcodeSuccess, - RRType: dns.TypeAAAA, - Value: net.ParseIP("::42"), - }}, + want: append(testHosts[addr4216], testHosts[addr42]...), }, { - req: &urlfilter.DNSRequest{ - Hostname: "domain4", - DNSType: dns.TypeA, - }, + req: "domain4", name: "issue_4216_4", - want: []*rules.DNSRewrite{{ - RCode: dns.RcodeSuccess, - RRType: dns.TypeA, - Value: net.IPv4(7, 5, 3, 1), - }, { - RCode: dns.RcodeSuccess, - RRType: dns.TypeA, - Value: net.IPv4(1, 3, 5, 7), - }}, + want: append(testHosts[addr1357], testHosts[addr7531]...), }, { - req: &urlfilter.DNSRequest{ - Hostname: "domain6", - DNSType: dns.TypeAAAA, - }, + req: "domain6", name: "issue_4216_6", - want: []*rules.DNSRewrite{{ - RCode: dns.RcodeSuccess, - RRType: dns.TypeAAAA, - Value: net.ParseIP("::13"), - }, { - RCode: dns.RcodeSuccess, - RRType: dns.TypeAAAA, - Value: net.ParseIP("::31"), - }}, + want: append(testHosts[addr13], testHosts[addr31]...), }} + hc, err := aghnet.NewHostsContainer(testdata, &stubWatcher, "etc_hosts") + require.NoError(t, err) + testutil.CleanupAndRequireSuccess(t, hc.Close) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + recs := hc.MatchName(tc.req) + assert.Equal(t, tc.want, recs) + }) + } +} + +func TestHostsContainer_MatchAddr(t *testing.T) { + require.NoError(t, fstest.TestFS(testdata, "etc_hosts")) + stubWatcher := aghtest.FSWatcher{ OnEvents: func() (e <-chan struct{}) { return nil }, OnAdd: func(name string) (err error) { return nil }, OnClose: func() (err error) { return nil }, } - hc, err := aghnet.NewHostsContainer(listID, testdata, &stubWatcher, "etc_hosts") + hc, err := aghnet.NewHostsContainer(testdata, &stubWatcher, "etc_hosts") require.NoError(t, err) testutil.CleanupAndRequireSuccess(t, hc.Close) + testCases := []struct { + req netip.Addr + name string + want []*hostsfile.Record + }{{ + req: netip.AddrFrom4([4]byte{1, 0, 0, 1}), + name: "reverse", + want: testHosts[addr1001], + }} + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - res, ok := hc.MatchRequest(tc.req) - require.False(t, ok) - - if tc.want == nil { - assert.Nil(t, res) - - return - } - - require.NotNil(t, res) - - rewrites := res.DNSRewrites() - require.Len(t, rewrites, len(tc.want)) - - for i, rewrite := range rewrites { - require.Equal(t, listID, rewrite.FilterListID) - - rw := rewrite.DNSRewrite - require.NotNil(t, rw) - - assert.Equal(t, tc.want[i], rw) - } + recs := hc.MatchAddr(tc.req) + assert.Equal(t, tc.want, recs) }) } } diff --git a/internal/dnsforward/dnsforward_test.go b/internal/dnsforward/dnsforward_test.go index f41b936f..acc8a6d1 100644 --- a/internal/dnsforward/dnsforward_test.go +++ b/internal/dnsforward/dnsforward_test.go @@ -1187,7 +1187,7 @@ func TestPTRResponseFromHosts(t *testing.T) { } var eventsCalledCounter uint32 - hc, err := aghnet.NewHostsContainer(0, testFS, &aghtest.FSWatcher{ + hc, err := aghnet.NewHostsContainer(testFS, &aghtest.FSWatcher{ OnEvents: func() (e <-chan struct{}) { assert.Equal(t, uint32(1), atomic.AddUint32(&eventsCalledCounter, 1)) diff --git a/internal/dnsforward/http.go b/internal/dnsforward/http.go index 55a87df5..24695b9f 100644 --- a/internal/dnsforward/http.go +++ b/internal/dnsforward/http.go @@ -665,9 +665,11 @@ func (s *Server) parseUpstreamLine( // dnsFilter can be nil during application update. if s.dnsFilter != nil && s.dnsFilter.EtcHosts != nil { - resolved := s.resolveUpstreamHost(extractUpstreamHost(upstreamAddr)) - sortNetIPAddrs(resolved, opts.PreferIPv6) - opts.ServerIPAddrs = resolved + recs := s.dnsFilter.EtcHosts.MatchName(extractUpstreamHost(upstreamAddr)) + for _, rec := range recs { + opts.ServerIPAddrs = append(opts.ServerIPAddrs, rec.Addr.AsSlice()) + } + sortNetIPAddrs(opts.ServerIPAddrs, opts.PreferIPv6) } u, err = upstream.AddressToUpstream(upstreamAddr, opts) if err != nil { diff --git a/internal/dnsforward/http_test.go b/internal/dnsforward/http_test.go index fe82618f..434bad5d 100644 --- a/internal/dnsforward/http_test.go +++ b/internal/dnsforward/http_test.go @@ -479,7 +479,6 @@ func TestServer_HandleTestUpstreamDNS(t *testing.T) { }).String() hc, err := aghnet.NewHostsContainer( - filtering.SysHostsListID, fstest.MapFS{ hostsFileName: &fstest.MapFile{ Data: []byte(hostsListener.Addr().String() + " " + upstreamHost), diff --git a/internal/dnsforward/upstreams.go b/internal/dnsforward/upstreams.go index 004d5e40..c08ab057 100644 --- a/internal/dnsforward/upstreams.go +++ b/internal/dnsforward/upstreams.go @@ -14,8 +14,6 @@ import ( "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/netutil" "github.com/AdguardTeam/golibs/stringutil" - "github.com/AdguardTeam/urlfilter" - "github.com/miekg/dns" "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) @@ -159,17 +157,20 @@ func (s *Server) resolveUpstreamsWithHosts( withIPs, ok := resolved[host] if !ok { - ips := s.resolveUpstreamHost(host) - if len(ips) == 0 { + recs := s.dnsFilter.EtcHosts.MatchName(host) + if len(recs) == 0 { resolved[host] = nil return nil } - sortNetIPAddrs(ips, opts.PreferIPv6) - withIPs = opts.Clone() - withIPs.ServerIPAddrs = ips + withIPs.ServerIPAddrs = make([]net.IP, 0, len(recs)) + for _, rec := range recs { + withIPs.ServerIPAddrs = append(withIPs.ServerIPAddrs, rec.Addr.AsSlice()) + } + + sortNetIPAddrs(withIPs.ServerIPAddrs, opts.PreferIPv6) resolved[host] = withIPs } else if withIPs == nil { continue @@ -217,33 +218,6 @@ func extractUpstreamHost(addr string) (host string) { return host } -// resolveUpstreamHost returns the version of ups with IP addresses from the -// system hosts file placed into its options. -func (s *Server) resolveUpstreamHost(host string) (addrs []net.IP) { - req := &urlfilter.DNSRequest{ - Hostname: host, - DNSType: dns.TypeA, - } - aRes, _ := s.dnsFilter.EtcHosts.MatchRequest(req) - - req.DNSType = dns.TypeAAAA - aaaaRes, _ := s.dnsFilter.EtcHosts.MatchRequest(req) - - var ips []net.IP - for _, rw := range append(aRes.DNSRewrites(), aaaaRes.DNSRewrites()...) { - dr := rw.DNSRewrite - if dr == nil || dr.Value == nil { - continue - } - - if ip, ok := dr.Value.(net.IP); ok { - ips = append(ips, ip) - } - } - - return ips -} - // sortNetIPAddrs sorts addrs in accordance with the protocol preferences. // Invalid addresses are sorted near the end. // diff --git a/internal/filtering/dnsrewrite.go b/internal/filtering/dnsrewrite.go index 58a8dd7b..edfc8b91 100644 --- a/internal/filtering/dnsrewrite.go +++ b/internal/filtering/dnsrewrite.go @@ -1,6 +1,10 @@ package filtering import ( + "net" + + "github.com/AdguardTeam/golibs/hostsfile" + "github.com/AdguardTeam/urlfilter" "github.com/AdguardTeam/urlfilter/rules" "github.com/miekg/dns" ) @@ -73,3 +77,59 @@ func (d *DNSFilter) processDNSRewrites(dnsr []*rules.NetworkRule) (res Result) { Reason: RewrittenRule, } } + +// processDNSResultRewrites returns an empty Result if there are no dnsrewrite +// rules in dnsres. Otherwise, it returns the processed Result. +func (d *DNSFilter) processDNSResultRewrites( + dnsres *urlfilter.DNSResult, + host string, +) (dnsRWRes Result) { + dnsr := dnsres.DNSRewrites() + if len(dnsr) == 0 { + return Result{} + } + + res := d.processDNSRewrites(dnsr) + if res.Reason == RewrittenRule && res.CanonName == host { + // A rewrite of a host to itself. Go on and try matching other things. + return Result{} + } + + return res +} + +// appendRewriteResultFromHost appends the rewrite result from rec to vals and +// resRules. +func appendRewriteResultFromHost( + vals []rules.RRValue, + resRules []*ResultRule, + rec *hostsfile.Record, + qtype uint16, +) (updatedVals []rules.RRValue, updatedRules []*ResultRule) { + switch qtype { + case dns.TypeA: + if !rec.Addr.Is4() { + return vals, resRules + } + + vals = append(vals, net.IP(rec.Addr.AsSlice())) + case dns.TypeAAAA: + if !rec.Addr.Is6() { + return vals, resRules + } + + vals = append(vals, net.IP(rec.Addr.AsSlice())) + case dns.TypePTR: + for _, name := range rec.Names { + vals = append(vals, name) + } + } + + recText, _ := rec.MarshalText() + resRules = append(resRules, &ResultRule{ + FilterListID: SysHostsListID, + Text: string(recText), + }) + + return vals, resRules +} diff --git a/internal/filtering/dnsrewrite_test.go b/internal/filtering/dnsrewrite_test.go index c75ea2b9..884c40ad 100644 --- a/internal/filtering/dnsrewrite_test.go +++ b/internal/filtering/dnsrewrite_test.go @@ -1,10 +1,17 @@ package filtering import ( + "fmt" "net" + "net/netip" "path" "testing" + "testing/fstest" + "github.com/AdguardTeam/AdGuardHome/internal/aghnet" + "github.com/AdguardTeam/AdGuardHome/internal/aghtest" + "github.com/AdguardTeam/golibs/testutil" + "github.com/AdguardTeam/urlfilter/rules" "github.com/miekg/dns" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -202,3 +209,154 @@ func TestDNSFilter_CheckHostRules_dnsrewrite(t *testing.T) { assert.Equal(t, "new-ptr-with-dot.", ptr) }) } + +func TestDNSFilter_CheckHost_hostsContainer(t *testing.T) { + addrv4 := netip.MustParseAddr("1.2.3.4") + addrv6 := netip.MustParseAddr("::1") + addrMapped := netip.MustParseAddr("::ffff:1.2.3.4") + + data := fmt.Sprintf( + ""+ + "%s v4.host.example\n"+ + "%s v6.host.example\n"+ + "%s mapped.host.example\n", + addrv4, + addrv6, + addrMapped, + ) + + files := fstest.MapFS{ + "hosts": &fstest.MapFile{ + Data: []byte(data), + }, + } + watcher := &aghtest.FSWatcher{ + OnEvents: func() (e <-chan struct{}) { return nil }, + OnAdd: func(name string) (err error) { return nil }, + OnClose: func() (err error) { return nil }, + } + hc, err := aghnet.NewHostsContainer(files, watcher, "hosts") + require.NoError(t, err) + testutil.CleanupAndRequireSuccess(t, hc.Close) + + f, _ := newForTest(t, &Config{EtcHosts: hc}, nil) + setts := &Settings{ + FilteringEnabled: true, + } + + testCases := []struct { + name string + host string + wantRules []*ResultRule + wantResps []rules.RRValue + dtyp uint16 + }{{ + name: "v4", + host: "v4.host.example", + dtyp: dns.TypeA, + wantRules: []*ResultRule{{ + Text: "1.2.3.4 v4.host.example", + FilterListID: SysHostsListID, + }}, + wantResps: []rules.RRValue{net.IP(addrv4.AsSlice())}, + }, { + name: "v6", + host: "v6.host.example", + dtyp: dns.TypeAAAA, + wantRules: []*ResultRule{{ + Text: "::1 v6.host.example", + FilterListID: SysHostsListID, + }}, + wantResps: []rules.RRValue{net.IP(addrv6.AsSlice())}, + }, { + name: "mapped", + host: "mapped.host.example", + dtyp: dns.TypeAAAA, + wantRules: []*ResultRule{{ + Text: "::ffff:1.2.3.4 mapped.host.example", + FilterListID: SysHostsListID, + }}, + wantResps: []rules.RRValue{net.IP(addrMapped.AsSlice())}, + }, { + name: "ptr", + host: "4.3.2.1.in-addr.arpa", + dtyp: dns.TypePTR, + wantRules: []*ResultRule{{ + Text: "1.2.3.4 v4.host.example", + FilterListID: SysHostsListID, + }}, + wantResps: []rules.RRValue{"v4.host.example"}, + }, { + name: "ptr-mapped", + host: "4.0.3.0.2.0.1.0.f.f.f.f.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa", + dtyp: dns.TypePTR, + wantRules: []*ResultRule{{ + Text: "::ffff:1.2.3.4 mapped.host.example", + FilterListID: SysHostsListID, + }}, + wantResps: []rules.RRValue{"mapped.host.example"}, + }, { + name: "not_found_v4", + host: "non.existent.example", + dtyp: dns.TypeA, + wantRules: nil, + wantResps: nil, + }, { + name: "not_found_v6", + host: "non.existent.example", + dtyp: dns.TypeAAAA, + wantRules: nil, + wantResps: nil, + }, { + name: "not_found_ptr", + host: "4.3.2.2.in-addr.arpa", + dtyp: dns.TypePTR, + wantRules: nil, + wantResps: nil, + }, { + name: "v4_mismatch", + host: "v4.host.example", + dtyp: dns.TypeAAAA, + wantRules: nil, + wantResps: nil, + }, { + name: "v6_mismatch", + host: "v6.host.example", + dtyp: dns.TypeA, + wantRules: nil, + wantResps: nil, + }, { + name: "wrong_ptr", + host: "4.3.2.1.ip6.arpa", + dtyp: dns.TypePTR, + wantRules: nil, + wantResps: nil, + }, { + name: "unsupported_type", + host: "v4.host.example", + dtyp: dns.TypeCNAME, + wantRules: nil, + wantResps: nil, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var res Result + res, err = f.CheckHost(tc.host, tc.dtyp, setts) + require.NoError(t, err) + + if len(tc.wantRules) == 0 { + assert.Empty(t, res.Rules) + assert.Nil(t, res.DNSRewriteResult) + + return + } + + require.NotNil(t, res.DNSRewriteResult) + require.Contains(t, res.DNSRewriteResult.Response, tc.dtyp) + + assert.Equal(t, tc.wantResps, res.DNSRewriteResult.Response[tc.dtyp]) + assert.Equal(t, tc.wantRules, res.Rules) + }) + } +} diff --git a/internal/filtering/filtering.go b/internal/filtering/filtering.go index b6249cee..eb630f0e 100644 --- a/internal/filtering/filtering.go +++ b/internal/filtering/filtering.go @@ -7,6 +7,7 @@ import ( "io/fs" "net" "net/http" + "net/netip" "os" "path/filepath" "runtime" @@ -19,8 +20,10 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/hostsfile" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/mathutil" + "github.com/AdguardTeam/golibs/netutil" "github.com/AdguardTeam/golibs/stringutil" "github.com/AdguardTeam/urlfilter" "github.com/AdguardTeam/urlfilter/filterlist" @@ -326,16 +329,6 @@ func (d *DNSFilter) WriteDiskConfig(c *Config) { c.UserRules = slices.Clone(d.UserRules) } -// cloneRewrites returns a deep copy of entries. -func cloneRewrites(entries []*LegacyRewrite) (clone []*LegacyRewrite) { - clone = make([]*LegacyRewrite, len(entries)) - for i, rw := range entries { - clone[i] = rw.clone() - } - - return clone -} - // setFilters sets new filters, synchronously or asynchronously. When filters // are set asynchronously, the old filters continue working until the new // filters are ready. @@ -503,31 +496,50 @@ func (d *DNSFilter) matchSysHosts( qtype uint16, setts *Settings, ) (res Result, err error) { + // TODO(e.burkov): Where else is this checked? if !setts.FilteringEnabled || d.EtcHosts == nil { return res, nil } - dnsres, _ := d.EtcHosts.MatchRequest(&urlfilter.DNSRequest{ - Hostname: host, - SortedClientTags: setts.ClientTags, - // TODO(e.burkov): Wait for urlfilter update to pass netip.Addr. - ClientIP: setts.ClientIP.String(), - ClientName: setts.ClientName, - DNSType: qtype, - }) - if dnsres == nil { - return res, nil + var recs []*hostsfile.Record + switch qtype { + case dns.TypeA, dns.TypeAAAA: + recs = d.EtcHosts.MatchName(host) + case dns.TypePTR: + var ip net.IP + ip, err = netutil.IPFromReversedAddr(host) + if err != nil { + log.Debug("filtering: failed to parse PTR record %q: %s", host, err) + + return res, nil + } + + addr, _ := netip.AddrFromSlice(ip) + recs = d.EtcHosts.MatchAddr(addr) + default: + log.Debug("filtering: unsupported query type %s", dns.Type(qtype)) } - dnsr := dnsres.DNSRewrites() - if len(dnsr) == 0 { - return res, nil + var vals []rules.RRValue + var resRules []*ResultRule + resRulesLen := 0 + for _, rec := range recs { + vals, resRules = appendRewriteResultFromHost(vals, resRules, rec, qtype) + if len(resRules) > resRulesLen { + resRulesLen = len(resRules) + log.Debug("filtering: matched %s in %q", host, rec.Source) + } } - res = d.processDNSRewrites(dnsr) - res.Reason = RewrittenAutoHosts - for _, r := range res.Rules { - r.Text = stringutil.Coalesce(d.EtcHosts.Translate(r.Text), r.Text) + if len(vals) > 0 { + res.DNSRewriteResult = &DNSRewriteResult{ + Response: DNSRewriteResultResponse{ + qtype: vals, + }, + RCode: dns.RcodeSuccess, + } + res.Rules = resRules + res.Reason = RewrittenRule } return res, nil @@ -594,25 +606,6 @@ func (d *DNSFilter) processRewrites(host string, qtype uint16) (res Result) { return res } -// setRewriteResult sets the Reason or IPList of res if necessary. res must not -// be nil. -func setRewriteResult(res *Result, host string, rewrites []*LegacyRewrite, qtype uint16) { - for _, rw := range rewrites { - if rw.Type == qtype && (qtype == dns.TypeA || qtype == dns.TypeAAAA) { - if rw.IP == nil { - // "A"/"AAAA" exception: allow getting from upstream. - res.Reason = NotFilteredNotFound - - return - } - - res.IPList = append(res.IPList, rw.IP) - - log.Debug("rewrite: a/aaaa for %s is %s", host, rw.IP) - } - } -} - // matchBlockedServicesRules checks the host against the blocked services rules // in settings, if any. The err is always nil, it is only there to make this // a valid hostChecker function. @@ -895,26 +888,6 @@ func (d *DNSFilter) matchHost( return res, nil } -// processDNSResultRewrites returns an empty Result if there are no dnsrewrite -// rules in dnsres. Otherwise, it returns the processed Result. -func (d *DNSFilter) processDNSResultRewrites( - dnsres *urlfilter.DNSResult, - host string, -) (dnsRWRes Result) { - dnsr := dnsres.DNSRewrites() - if len(dnsr) == 0 { - return Result{} - } - - res := d.processDNSRewrites(dnsr) - if res.Reason == RewrittenRule && res.CanonName == host { - // A rewrite of a host to itself. Go on and try matching other things. - return Result{} - } - - return res -} - // makeResult returns a properly constructed Result. func makeResult(matchedRules []rules.Rule, reason Reason) (res Result) { resRules := make([]*ResultRule, len(matchedRules)) @@ -987,7 +960,6 @@ func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) { if d.BlockedServices != nil { err = d.BlockedServices.Validate() - if err != nil { return nil, fmt.Errorf("filtering: %w", err) } diff --git a/internal/filtering/filtering_test.go b/internal/filtering/filtering_test.go index e7b55d6f..9159f7be 100644 --- a/internal/filtering/filtering_test.go +++ b/internal/filtering/filtering_test.go @@ -91,7 +91,7 @@ func (d *DNSFilter) checkMatchEmpty(t *testing.T, hostname string, setts *Settin assert.Falsef(t, res.IsFiltered, "host %q", hostname) } -func TestEtcHostsMatching(t *testing.T) { +func TestDNSFilter_CheckHost_hostRules(t *testing.T) { addr := "216.239.38.120" addr6 := "::1" text := fmt.Sprintf(` %s google.com www.google.com # enforce google's safesearch diff --git a/internal/filtering/rewrites.go b/internal/filtering/rewrites.go index eb3a4363..3d0dd64f 100644 --- a/internal/filtering/rewrites.go +++ b/internal/filtering/rewrites.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/mathutil" "github.com/miekg/dns" "golang.org/x/exp/slices" @@ -200,3 +201,32 @@ func findRewrites( return rewrites, matched } + +// setRewriteResult sets the Reason or IPList of res if necessary. res must not +// be nil. +func setRewriteResult(res *Result, host string, rewrites []*LegacyRewrite, qtype uint16) { + for _, rw := range rewrites { + if rw.Type == qtype && (qtype == dns.TypeA || qtype == dns.TypeAAAA) { + if rw.IP == nil { + // "A"/"AAAA" exception: allow getting from upstream. + res.Reason = NotFilteredNotFound + + return + } + + res.IPList = append(res.IPList, rw.IP) + + log.Debug("rewrite: a/aaaa for %s is %s", host, rw.IP) + } + } +} + +// cloneRewrites returns a deep copy of entries. +func cloneRewrites(entries []*LegacyRewrite) (clone []*LegacyRewrite) { + clone = make([]*LegacyRewrite, len(entries)) + for i, rw := range entries { + clone[i] = rw.clone() + } + + return clone +} diff --git a/internal/home/clients.go b/internal/home/clients.go index b62d87c7..d11f46a9 100644 --- a/internal/home/clients.go +++ b/internal/home/clients.go @@ -855,15 +855,19 @@ func (clients *clientsContainer) rmHostsBySrc(src clientSource) { // addFromHostsFile fills the client-hostname pairing index from the system's // hosts files. -func (clients *clientsContainer) addFromHostsFile(hosts aghnet.HostsRecords) { +func (clients *clientsContainer) addFromHostsFile(hosts aghnet.Hosts) { clients.lock.Lock() defer clients.lock.Unlock() clients.rmHostsBySrc(ClientSourceHostsFile) n := 0 - for ip, rec := range hosts { - clients.addHostLocked(ip, rec.Canonical, ClientSourceHostsFile) + for addr, rec := range hosts { + // Only the first name of the first record is considered a canonical + // hostname for the IP address. + // + // TODO(e.burkov): Consider using all the names from all the records. + clients.addHostLocked(addr, rec[0].Names[0], ClientSourceHostsFile) n++ } diff --git a/internal/home/home.go b/internal/home/home.go index a844961c..19bb0ca4 100644 --- a/internal/home/home.go +++ b/internal/home/home.go @@ -233,7 +233,6 @@ func setupHostsContainer() (err error) { } Context.etcHosts, err = aghnet.NewHostsContainer( - filtering.SysHostsListID, aghos.RootDirFS(), hostsWatcher, aghnet.DefaultHostsPaths()...,