mirrors
/
AdGuardHome
mirror of https://github.com/AdguardTeam/AdGuardHome.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pull request 1964: AG-23599 use hostsfile 

Merge in DNS/adguard-home from AG-23599-use-hostsfile to master

Squashed commit of the following:

commit 4766e67a9d5faa4bc89a2a935d187ce4829f7214
Merge: 38369360b 762e5be97
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Aug 22 16:33:54 2023 +0300

    Merge branch 'master' into AG-23599-use-hostsfile

commit 38369360b7d0e5c9ec373c5a06bac8792ca9cd69
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Aug 21 18:09:15 2023 +0300

    filtering: imp tests

commit 1c4d4a9f9639f048173e1c949f39f9ecb6ed0347
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Aug 21 14:00:10 2023 +0300

    filtering: imp cognit, cyclo

commit c50c33d7240c2812a715759fabf140e02184b729
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Aug 21 12:57:31 2023 +0300

    filtering: imp code

commit 92203b16719a717a2946c0401e166b1b38ddb7bc
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Aug 18 17:39:11 2023 +0300

    all: imp code, docs

commit 523e8cd50f9136feede657385b7274fa6ba64131
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Aug 17 15:14:02 2023 +0300

    all: fix ipv6

commit 6ce4537132615cbdc34a0b1f326fedd2b63c355d
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Aug 17 14:17:27 2023 +0300

    all: rm urlfilter from hosts

commit d6666e851680c7e586325ea5970e0356ab919074
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Aug 16 15:09:52 2023 +0300

    WIP

commit 4a2732960558bef6636d3c428bad4c7c830016ca
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Aug 16 14:47:13 2023 +0300

    all: use hostsfile
This commit is contained in:
Eugene Burkov 2023-08-22 16:45:11 +03:00
parent 762e5be97a
commit 4b4036fa6a
14 changed files with 639 additions and 753 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

407
internal/aghnet/hostscontainer.go
View File

@ -1,25 +1,19 @@
package aghnet package aghnet
import ( import (
"bufio"
"fmt" "fmt"
"io" "io"
"io/fs" "io/fs"
"net/netip" "net/netip"
"path" "path"
"strings" "sync/atomic"
"sync"
"github.com/AdguardTeam/AdGuardHome/internal/aghos" "github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/urlfilter"
"github.com/AdguardTeam/urlfilter/filterlist"
"github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"golang.org/x/exp/slices"
) )
// DefaultHostsPaths returns the slice of paths default for the operating system // DefaultHostsPaths returns the slice of paths default for the operating system
@ -29,95 +23,51 @@ func DefaultHostsPaths() (paths []string) {
return defaultHostsPaths() return defaultHostsPaths()
} }
// requestMatcher combines the logic for matching requests and translating the // MatchAddr returns the records for the IP address.
// appropriate rules. func (hc *HostsContainer) MatchAddr(ip netip.Addr) (recs []*hostsfile.Record) {
type requestMatcher struct { cur := hc.current.Load()
// stateLock protects all the fields of requestMatcher. if cur == nil {
stateLock *sync.RWMutex return nil
// rulesStrg stores the rules obtained from the hosts' file.
rulesStrg *filterlist.RuleStorage
// engine serves rulesStrg.
engine *urlfilter.DNSEngine
// translator maps generated $dnsrewrite rules into hosts-syntax rules.
//
// TODO(e.burkov): Store the filename from which the rule was parsed.
translator map[string]string
}
// MatchRequest processes the request rewriting hostnames and addresses read
// from the operating system's hosts files. res is nil for any request having
// not an A/AAAA or PTR type, see man 5 hosts.
//
// It's safe for concurrent use.
func (rm *requestMatcher) MatchRequest(
req *urlfilter.DNSRequest,
) (res *urlfilter.DNSResult, ok bool) {
switch req.DNSType {
case dns.TypeA, dns.TypeAAAA, dns.TypePTR:
log.Debug(
"%s: handling %s request for %s",
hostsContainerPrefix,
dns.Type(req.DNSType),
req.Hostname,
)
rm.stateLock.RLock()
defer rm.stateLock.RUnlock()
return rm.engine.MatchRequest(req)
default:
return nil, false
} }
return cur.addrs[ip]
} }
// Translate returns the source hosts-syntax rule for the generated dnsrewrite // MatchName returns the records for the hostname.
// rule or an empty string if the last doesn't exist. The returned rules are in func (hc *HostsContainer) MatchName(name string) (recs []*hostsfile.Record) {
// a processed format like: cur := hc.current.Load()
// if cur != nil {
// ip host1 host2 ... recs = cur.names[name]
func (rm *requestMatcher) Translate(rule string) (hostRule string) { }
rm.stateLock.RLock()
defer rm.stateLock.RUnlock()
return rm.translator[rule] return recs
}
// resetEng updates container's engine and the translation map.
func (rm *requestMatcher) resetEng(rulesStrg *filterlist.RuleStorage, tr map[string]string) {
rm.stateLock.Lock()
defer rm.stateLock.Unlock()
rm.rulesStrg = rulesStrg
rm.engine = urlfilter.NewDNSEngine(rm.rulesStrg)
rm.translator = tr
} }
// hostsContainerPrefix is a prefix for logging and wrapping errors in // hostsContainerPrefix is a prefix for logging and wrapping errors in
// HostsContainer's methods. // HostsContainer's methods.
const hostsContainerPrefix = "hosts container" const hostsContainerPrefix = "hosts container"
// Hosts is a map of IP addresses to the records, as it primarily stored in the
// [HostsContainer]. It should not be accessed for writing since it may be read
// concurrently, users should clone it before modifying.
//
// The order of records for each address is preserved from original files, but
// the order of the addresses, being a map key, is not.
//
// TODO(e.burkov): Probably, this should be a sorted slice of records.
type Hosts map[netip.Addr][]*hostsfile.Record
// HostsContainer stores the relevant hosts database provided by the OS and // HostsContainer stores the relevant hosts database provided by the OS and
// processes both A/AAAA and PTR DNS requests for those. // processes both A/AAAA and PTR DNS requests for those.
//
// TODO(e.burkov): Improve API and move to golibs.
type HostsContainer struct { type HostsContainer struct {
// requestMatcher matches the requests and translates the rules. It's
// embedded to implement MatchRequest and Translate for *HostsContainer.
//
// TODO(a.garipov, e.burkov): Consider fully merging into HostsContainer.
requestMatcher
// done is the channel to sign closing the container. // done is the channel to sign closing the container.
done chan struct{} done chan struct{}
// updates is the channel for receiving updated hosts. // updates is the channel for receiving updated hosts.
updates chan HostsRecords updates chan Hosts
// last is the set of hosts that was cached within last detected change. // current is the last set of hosts parsed.
last HostsRecords current atomic.Pointer[hostsIndex]
// fsys is the working file system to read hosts files from. // fsys is the working file system to read hosts files from.
fsys fs.FS fsys fs.FS
@ -127,30 +77,6 @@ type HostsContainer struct {
// patterns stores specified paths in the fs.Glob-compatible form. // patterns stores specified paths in the fs.Glob-compatible form.
patterns []string patterns []string
// listID is the identifier for the list of generated rules.
listID int
}
// HostsRecords is a mapping of an IP address to its hosts data.
type HostsRecords map[netip.Addr]*HostsRecord
// HostsRecord represents a single hosts file record.
type HostsRecord struct {
Aliases *stringutil.Set
Canonical string
}
// Equal returns true if all fields of rec are equal to field in other or they
// both are nil.
func (rec *HostsRecord) Equal(other *HostsRecord) (ok bool) {
if rec == nil {
return other == nil
} else if other == nil {
return false
}
return rec.Canonical == other.Canonical && rec.Aliases.Equal(other.Aliases)
} }
// ErrNoHostsPaths is returned when there are no valid paths to watch passed to // ErrNoHostsPaths is returned when there are no valid paths to watch passed to
@ -162,7 +88,6 @@ const ErrNoHostsPaths errors.Error = "no valid paths to hosts files provided"
// shouldn't be empty and each of paths should locate either a file or a // shouldn't be empty and each of paths should locate either a file or a
// directory in fsys. fsys and w must be non-nil. // directory in fsys. fsys and w must be non-nil.
func NewHostsContainer( func NewHostsContainer(
listID int,
fsys fs.FS, fsys fs.FS,
w aghos.FSWatcher, w aghos.FSWatcher,
paths ...string, paths ...string,
@ -182,12 +107,8 @@ func NewHostsContainer(
} }
hc = &HostsContainer{ hc = &HostsContainer{
requestMatcher: requestMatcher{
stateLock: &sync.RWMutex{},
},
listID: listID,
done: make(chan struct{}, 1), done: make(chan struct{}, 1),
updates: make(chan HostsRecords, 1), updates: make(chan Hosts, 1),
fsys: fsys, fsys: fsys,
watcher: w, watcher: w,
patterns: patterns, patterns: patterns,
@ -233,7 +154,7 @@ func (hc *HostsContainer) Close() (err error) {
} }
// Upd returns the channel into which the updates are sent. // Upd returns the channel into which the updates are sent.
func (hc *HostsContainer) Upd() (updates <-chan HostsRecords) { func (hc *HostsContainer) Upd() (updates <-chan Hosts) {
return hc.updates return hc.updates
} }
@ -288,198 +209,68 @@ func (hc *HostsContainer) handleEvents() {
} }
} }
// hostsParser is a helper type to parse rules from the operating system's hosts
// file. It exists for only a single refreshing session.
type hostsParser struct {
// rulesBuilder builds the resulting rules list content.
rulesBuilder *strings.Builder
// translations maps generated rules into actual hosts file lines.
translations map[string]string
// table stores only the unique IP-hostname pairs. It's also sent to the
// updates channel afterwards.
table HostsRecords
}
// newHostsParser creates a new *hostsParser with buffers of size taken from the
// previous parse.
func (hc *HostsContainer) newHostsParser() (hp *hostsParser) {
return &hostsParser{
rulesBuilder: &strings.Builder{},
translations: map[string]string{},
table: make(HostsRecords, len(hc.last)),
}
}
// parseFile is a aghos.FileWalker for parsing the files with hosts syntax. It
// never signs to stop walking and never returns any additional patterns.
//
// See man hosts(5).
func (hp *hostsParser) parseFile(r io.Reader) (patterns []string, cont bool, err error) {
s := bufio.NewScanner(r)
for s.Scan() {
ip, hosts := hp.parseLine(s.Text())
if ip == (netip.Addr{}) || len(hosts) == 0 {
continue
}
hp.addRecord(ip, hosts)
}
return nil, true, s.Err()
}
// parseLine parses the line having the hosts syntax ignoring invalid ones.
func (hp *hostsParser) parseLine(line string) (ip netip.Addr, hosts []string) {
fields := strings.Fields(line)
if len(fields) < 2 {
return netip.Addr{}, nil
}
ip, err := netip.ParseAddr(fields[0])
if err != nil {
return netip.Addr{}, nil
}
for _, f := range fields[1:] {
hashIdx := strings.IndexByte(f, '#')
if hashIdx == 0 {
// The rest of the fields are a part of the comment so return.
break
} else if hashIdx > 0 {
// Only a part of the field is a comment.
f = f[:hashIdx]
}
// Make sure that invalid hosts aren't turned into rules.
//
// See https://github.com/AdguardTeam/AdGuardHome/issues/3946.
//
// TODO(e.burkov): Investigate if hosts may contain DNS-SD domains.
err = netutil.ValidateHostname(f)
if err != nil {
log.Error("%s: host %q is invalid, ignoring", hostsContainerPrefix, f)
continue
}
hosts = append(hosts, f)
}
return ip, hosts
}
// addRecord puts the record for the IP address to the rules builder if needed.
// The first host is considered to be the canonical name for the IP address.
// hosts must have at least one name.
func (hp *hostsParser) addRecord(ip netip.Addr, hosts []string) {
line := strings.Join(append([]string{ip.String()}, hosts...), " ")
rec, ok := hp.table[ip]
if !ok {
rec = &HostsRecord{
Aliases: stringutil.NewSet(),
}
rec.Canonical, hosts = hosts[0], hosts[1:]
hp.addRules(ip, rec.Canonical, line)
hp.table[ip] = rec
}
for _, host := range hosts {
if rec.Canonical == host || rec.Aliases.Has(host) {
continue
}
rec.Aliases.Add(host)
hp.addRules(ip, host, line)
}
}
// addRules adds rules and rule translations for the line.
func (hp *hostsParser) addRules(ip netip.Addr, host, line string) {
rule, rulePtr := hp.writeRules(host, ip)
hp.translations[rule], hp.translations[rulePtr] = line, line
log.Debug("%s: added ip-host pair %q-%q", hostsContainerPrefix, ip, host)
}
// writeRules writes the actual rule for the qtype and the PTR for the host-ip
// pair into internal builders.
func (hp *hostsParser) writeRules(host string, ip netip.Addr) (rule, rulePtr string) {
// TODO(a.garipov): Add a netip.Addr version to netutil.
arpa, err := netutil.IPToReversedAddr(ip.AsSlice())
if err != nil {
return "", ""
}
const (
nl = "\n"
rwSuccess = "^$dnsrewrite=NOERROR;"
rwSuccessPTR = "^$dnsrewrite=NOERROR;PTR;"
modLen = len(rules.MaskPipe) + len(rwSuccess) + len(";")
modLenPTR = len(rules.MaskPipe) + len(rwSuccessPTR)
)
var qtype string
// The validation of the IP address has been performed earlier so it is
// guaranteed to be either an IPv4 or an IPv6.
if ip.Is4() {
qtype = "A"
} else {
qtype = "AAAA"
}
ipStr := ip.String()
fqdn := dns.Fqdn(host)
ruleBuilder := &strings.Builder{}
ruleBuilder.Grow(modLen + len(host) + len(qtype) + len(ipStr))
stringutil.WriteToBuilder(ruleBuilder, rules.MaskPipe, host, rwSuccess, qtype, ";", ipStr)
rule = ruleBuilder.String()
ruleBuilder.Reset()
ruleBuilder.Grow(modLenPTR + len(arpa) + len(fqdn))
stringutil.WriteToBuilder(ruleBuilder, rules.MaskPipe, arpa, rwSuccessPTR, fqdn)
rulePtr = ruleBuilder.String()
hp.rulesBuilder.Grow(len(rule) + len(rulePtr) + 2*len(nl))
stringutil.WriteToBuilder(hp.rulesBuilder, rule, nl, rulePtr, nl)
return rule, rulePtr
}
// sendUpd tries to send the parsed data to the ch. // sendUpd tries to send the parsed data to the ch.
func (hp *hostsParser) sendUpd(ch chan HostsRecords) { func (hc *HostsContainer) sendUpd(recs Hosts) {
log.Debug("%s: sending upd", hostsContainerPrefix) log.Debug("%s: sending upd", hostsContainerPrefix)
upd := hp.table ch := hc.updates
select { select {
case ch <- upd: case ch <- recs:
// Updates are delivered. Go on. // Updates are delivered. Go on.
case <-ch: case <-ch:
ch <- upd ch <- recs
log.Debug("%s: replaced the last update", hostsContainerPrefix) log.Debug("%s: replaced the last update", hostsContainerPrefix)
case ch <- upd: case ch <- recs:
// The previous update was just read and the next one pushed. Go on. // The previous update was just read and the next one pushed. Go on.
default: default:
log.Error("%s: the updates channel is broken", hostsContainerPrefix) log.Error("%s: the updates channel is broken", hostsContainerPrefix)
} }
} }
// newStrg creates a new rules storage from parsed data. // hostsIndex is a [hostsfile.Set] to enumerate all the records.
func (hp *hostsParser) newStrg(id int) (s *filterlist.RuleStorage, err error) { type hostsIndex struct {
return filterlist.NewRuleStorage([]filterlist.RuleList{&filterlist.StringRuleList{ // addrs maps IP addresses to the records.
ID: id, addrs Hosts
RulesText: hp.rulesBuilder.String(),
IgnoreCosmetic: true, // names maps hostnames to the records.
}}) names map[string][]*hostsfile.Record
}
// walk is a file walking function for hostsIndex.
func (idx *hostsIndex) walk(r io.Reader) (patterns []string, cont bool, err error) {
return nil, true, hostsfile.Parse(idx, r, nil)
}
// type check
var _ hostsfile.Set = (*hostsIndex)(nil)
// Add puts the record for the IP address to the rules builder if needed.
// The first host is considered to be the canonical name for the IP address.
// hosts must have at least one name.
func (idx *hostsIndex) Add(rec *hostsfile.Record) {
idx.addrs[rec.Addr] = append(idx.addrs[rec.Addr], rec)
for _, name := range rec.Names {
idx.names[name] = append(idx.names[name], rec)
}
}
// equalRecs is an equality function for [*hostsfile.Record].
func equalRecs(a, b *hostsfile.Record) (ok bool) {
return a.Addr == b.Addr && a.Source == b.Source && slices.Equal(a.Names, b.Names)
}
// equalRecSlices is an equality function for slices of [*hostsfile.Record].
func equalRecSlices(a, b []*hostsfile.Record) (ok bool) { return slices.EqualFunc(a, b, equalRecs) }
// Equal returns true if indexes are equal.
func (idx *hostsIndex) Equal(other *hostsIndex) (ok bool) {
if idx == nil {
return other == nil
} else if other == nil {
return false
}
return maps.EqualFunc(idx.addrs, other.addrs, equalRecSlices)
} }
// refresh gets the data from specified files and propagates the updates if // refresh gets the data from specified files and propagates the updates if
@ -489,27 +280,31 @@ func (hp *hostsParser) newStrg(id int) (s *filterlist.RuleStorage, err error) {
func (hc *HostsContainer) refresh() (err error) { func (hc *HostsContainer) refresh() (err error) {
log.Debug("%s: refreshing", hostsContainerPrefix) log.Debug("%s: refreshing", hostsContainerPrefix)
hp := hc.newHostsParser() var addrLen, nameLen int
if _, err = aghos.FileWalker(hp.parseFile).Walk(hc.fsys, hc.patterns...); err != nil { last := hc.current.Load()
return fmt.Errorf("refreshing : %w", err) if last != nil {
addrLen, nameLen = len(last.addrs), len(last.names)
}
idx := &hostsIndex{
addrs: make(Hosts, addrLen),
names: make(map[string][]*hostsfile.Record, nameLen),
} }
// hc.last is nil on the first refresh, so let that one through. _, err = aghos.FileWalker(idx.walk).Walk(hc.fsys, hc.patterns...)
if hc.last != nil && maps.EqualFunc(hp.table, hc.last, (*HostsRecord).Equal) {
log.Debug("%s: no changes detected", hostsContainerPrefix)
return nil if err != nil {
} if len(idx.addrs) == 0 {
defer hp.sendUpd(hc.updates) return fmt.Errorf("refreshing : %w", err)
} else {
hc.last = maps.Clone(hp.table) log.Debug("%s: refreshing: %s", hostsContainerPrefix, err)
}
var rulesStrg *filterlist.RuleStorage
if rulesStrg, err = hp.newStrg(hc.listID); err != nil {
return fmt.Errorf("initializing rules storage: %w", err)
} }
hc.resetEng(rulesStrg, hp.translations) // TODO(e.burkov): Serialize updates using time.
if !last.Equal(idx) {
hc.current.Store(idx)
hc.sendUpd(idx.addrs)
}
return nil return nil
} }

68
internal/aghnet/hostscontainer_internal_test.go
View File

@ -2,13 +2,11 @@ package aghnet
import ( import (
"io/fs" "io/fs"
"net/netip"
"path" "path"
"testing" "testing"
"testing/fstest" "testing/fstest"
"github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/testutil/fakefs" "github.com/AdguardTeam/golibs/testutil/fakefs"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -76,69 +74,3 @@ func TestHostsContainer_PathsToPatterns(t *testing.T) {
assert.ErrorIs(t, err, errStat) assert.ErrorIs(t, err, errStat)
}) })
} }
func TestUniqueRules_ParseLine(t *testing.T) {
ip := netutil.IPv4Localhost()
ipStr := ip.String()
testCases := []struct {
name string
line string
wantIP netip.Addr
wantHosts []string
}{{
name: "simple",
line: ipStr + ` hostname`,
wantIP: ip,
wantHosts: []string{"hostname"},
}, {
name: "aliases",
line: ipStr + ` hostname alias`,
wantIP: ip,
wantHosts: []string{"hostname", "alias"},
}, {
name: "invalid_line",
line: ipStr,
wantIP: netip.Addr{},
wantHosts: nil,
}, {
name: "invalid_line_hostname",
line: ipStr + ` # hostname`,
wantIP: ip,
wantHosts: nil,
}, {
name: "commented_aliases",
line: ipStr + ` hostname # alias`,
wantIP: ip,
wantHosts: []string{"hostname"},
}, {
name: "whole_comment",
line: `# ` + ipStr + ` hostname`,
wantIP: netip.Addr{},
wantHosts: nil,
}, {
name: "partial_comment",
line: ipStr + ` host#name`,
wantIP: ip,
wantHosts: []string{"host"},
}, {
name: "empty",
line: ``,
wantIP: netip.Addr{},
wantHosts: nil,
}, {
name: "bad_hosts",
line: ipStr + ` bad..host bad._tld empty.tld. ok.host`,
wantIP: ip,
wantHosts: []string{"ok.host"},
}}
for _, tc := range testCases {
hp := hostsParser{}
t.Run(tc.name, func(t *testing.T) {
got, hosts := hp.parseLine(tc.line)
assert.Equal(t, tc.wantIP, got)
assert.Equal(t, tc.wantHosts, hosts)
})
}
}

497
internal/aghnet/hostscontainer_test.go
View File

@ -1,9 +1,8 @@
package aghnet_test package aghnet_test
import ( import (
"net" "net/netip"
"path" "path"
"strings"
"sync/atomic" "sync/atomic"
"testing" "testing"
"testing/fstest" "testing/fstest"
@ -13,18 +12,144 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest" "github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/golibs/netutil" "github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/testutil" "github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/urlfilter"
"github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
// nl is a newline character.
const nl = "\n" const nl = "\n"
// Variables mirroring the etc_hosts file from testdata.
var (
addr1000 = netip.MustParseAddr("1.0.0.0")
addr1001 = netip.MustParseAddr("1.0.0.1")
addr1002 = netip.MustParseAddr("1.0.0.2")
addr1003 = netip.MustParseAddr("1.0.0.3")
addr1004 = netip.MustParseAddr("1.0.0.4")
addr1357 = netip.MustParseAddr("1.3.5.7")
addr4216 = netip.MustParseAddr("4.2.1.6")
addr7531 = netip.MustParseAddr("7.5.3.1")
addr0 = netip.MustParseAddr("::")
addr1 = netip.MustParseAddr("::1")
addr2 = netip.MustParseAddr("::2")
addr3 = netip.MustParseAddr("::3")
addr4 = netip.MustParseAddr("::4")
addr42 = netip.MustParseAddr("::42")
addr13 = netip.MustParseAddr("::13")
addr31 = netip.MustParseAddr("::31")
testHosts = map[netip.Addr][]*hostsfile.Record{
addr1000: {{
Addr: addr1000,
Source: "./testdata/etc_hosts",
Names: []string{"hello", "hello.world"},
}, {
Addr: addr1000,
Source: "./testdata/etc_hosts",
Names: []string{"hello.world.again"},
}, {
Addr: addr1000,
Source: "./testdata/etc_hosts",
Names: []string{"hello.world"},
}},
addr1001: {{
Addr: addr1001,
Source: "./testdata/etc_hosts",
Names: []string{"simplehost"},
}, {
Addr: addr1001,
Source: "./testdata/etc_hosts",
Names: []string{"simplehost"},
}},
addr1002: {{
Addr: addr1002,
Source: "./testdata/etc_hosts",
Names: []string{"a.whole", "lot.of", "aliases", "for.testing"},
}},
addr1003: {{
Addr: addr1003,
Source: "./testdata/etc_hosts",
Names: []string{"*"},
}},
addr1004: {{
Addr: addr1004,
Source: "./testdata/etc_hosts",
Names: []string{"*.com"},
}},
addr1357: {{
Addr: addr1357,
Source: "./testdata/etc_hosts",
Names: []string{"domain4", "domain4.alias"},
}},
addr7531: {{
Addr: addr7531,
Source: "./testdata/etc_hosts",
Names: []string{"domain4.alias", "domain4"},
}},
addr4216: {{
Addr: addr4216,
Source: "./testdata/etc_hosts",
Names: []string{"domain", "domain.alias"},
}},
addr0: {{
Addr: addr0,
Source: "./testdata/etc_hosts",
Names: []string{"hello", "hello.world"},
}, {
Addr: addr0,
Source: "./testdata/etc_hosts",
Names: []string{"hello.world.again"},
}, {
Addr: addr0,
Source: "./testdata/etc_hosts",
Names: []string{"hello.world"},
}},
addr1: {{
Addr: addr1,
Source: "./testdata/etc_hosts",
Names: []string{"simplehost"},
}, {
Addr: addr1,
Source: "./testdata/etc_hosts",
Names: []string{"simplehost"},
}},
addr2: {{
Addr: addr2,
Source: "./testdata/etc_hosts",
Names: []string{"a.whole", "lot.of", "aliases", "for.testing"},
}},
addr3: {{
Addr: addr3,
Source: "./testdata/etc_hosts",
Names: []string{"*"},
}},
addr4: {{
Addr: addr4,
Source: "./testdata/etc_hosts",
Names: []string{"*.com"},
}},
addr42: {{
Addr: addr42,
Source: "./testdata/etc_hosts",
Names: []string{"domain.alias", "domain"},
}},
addr13: {{
Addr: addr13,
Source: "./testdata/etc_hosts",
Names: []string{"domain6", "domain6.alias"},
}},
addr31: {{
Addr: addr31,
Source: "./testdata/etc_hosts",
Names: []string{"domain6.alias", "domain6"},
}},
}
)
func TestNewHostsContainer(t *testing.T) { func TestNewHostsContainer(t *testing.T) {
const dirname = "dir" const dirname = "dir"
const filename = "file1" const filename = "file1"
@ -73,7 +198,7 @@ func TestNewHostsContainer(t *testing.T) {
return eventsCh return eventsCh
} }
hc, err := aghnet.NewHostsContainer(0, testFS, &aghtest.FSWatcher{ hc, err := aghnet.NewHostsContainer(testFS, &aghtest.FSWatcher{
OnEvents: onEvents, OnEvents: onEvents,
OnAdd: onAdd, OnAdd: onAdd,
OnClose: func() (err error) { return nil }, OnClose: func() (err error) { return nil },
@ -99,7 +224,7 @@ func TestNewHostsContainer(t *testing.T) {
t.Run("nil_fs", func(t *testing.T) { t.Run("nil_fs", func(t *testing.T) {
require.Panics(t, func() { require.Panics(t, func() {
_, _ = aghnet.NewHostsContainer(0, nil, &aghtest.FSWatcher{ _, _ = aghnet.NewHostsContainer(nil, &aghtest.FSWatcher{
// Those shouldn't panic. // Those shouldn't panic.
OnEvents: func() (e <-chan struct{}) { return nil }, OnEvents: func() (e <-chan struct{}) { return nil },
OnAdd: func(name string) (err error) { return nil }, OnAdd: func(name string) (err error) { return nil },
@ -110,7 +235,7 @@ func TestNewHostsContainer(t *testing.T) {
t.Run("nil_watcher", func(t *testing.T) { t.Run("nil_watcher", func(t *testing.T) {
require.Panics(t, func() { require.Panics(t, func() {
_, _ = aghnet.NewHostsContainer(0, testFS, nil, p) _, _ = aghnet.NewHostsContainer(testFS, nil, p)
}) })
}) })
@ -123,7 +248,7 @@ func TestNewHostsContainer(t *testing.T) {
OnClose: func() (err error) { return nil }, OnClose: func() (err error) { return nil },
} }
hc, err := aghnet.NewHostsContainer(0, testFS, errWatcher, p) hc, err := aghnet.NewHostsContainer(testFS, errWatcher, p)
require.ErrorIs(t, err, errOnAdd) require.ErrorIs(t, err, errOnAdd)
assert.Nil(t, hc) assert.Nil(t, hc)
@ -136,6 +261,9 @@ func TestHostsContainer_refresh(t *testing.T) {
ip := netutil.IPv4Localhost() ip := netutil.IPv4Localhost()
ipStr := ip.String() ipStr := ip.String()
anotherIPStr := "1.2.3.4"
anotherIP := netip.MustParseAddr(anotherIPStr)
testFS := fstest.MapFS{"dir/file1": &fstest.MapFile{Data: []byte(ipStr + ` hostname` + nl)}} testFS := fstest.MapFS{"dir/file1": &fstest.MapFile{Data: []byte(ipStr + ` hostname` + nl)}}
// event is a convenient alias for an empty struct{} to emit test events. // event is a convenient alias for an empty struct{} to emit test events.
@ -154,40 +282,44 @@ func TestHostsContainer_refresh(t *testing.T) {
OnClose: func() (err error) { return nil }, OnClose: func() (err error) { return nil },
} }
hc, err := aghnet.NewHostsContainer(0, testFS, w, "dir") hc, err := aghnet.NewHostsContainer(testFS, w, "dir")
require.NoError(t, err) require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close) testutil.CleanupAndRequireSuccess(t, hc.Close)
checkRefresh := func(t *testing.T, want *aghnet.HostsRecord) { checkRefresh := func(t *testing.T, want aghnet.Hosts) {
t.Helper() t.Helper()
upd, ok := aghchan.MustReceive(hc.Upd(), 1*time.Second) upd, ok := aghchan.MustReceive(hc.Upd(), 1*time.Second)
require.True(t, ok) require.True(t, ok)
require.NotNil(t, upd)
assert.Len(t, upd, 1) assert.Equal(t, want, upd)
rec, ok := upd[ip]
require.True(t, ok)
require.NotNil(t, rec)
assert.Truef(t, rec.Equal(want), "%+v != %+v", rec, want)
} }
t.Run("initial_refresh", func(t *testing.T) { t.Run("initial_refresh", func(t *testing.T) {
checkRefresh(t, &aghnet.HostsRecord{ checkRefresh(t, aghnet.Hosts{
Aliases: stringutil.NewSet(), ip: {{
Canonical: "hostname", Addr: ip,
Source: "file1",
Names: []string{"hostname"},
}},
}) })
}) })
t.Run("second_refresh", func(t *testing.T) { t.Run("second_refresh", func(t *testing.T) {
testFS["dir/file2"] = &fstest.MapFile{Data: []byte(ipStr + ` alias` + nl)} testFS["dir/file2"] = &fstest.MapFile{Data: []byte(anotherIPStr + ` alias` + nl)}
eventsCh <- event{} eventsCh <- event{}
checkRefresh(t, &aghnet.HostsRecord{ checkRefresh(t, aghnet.Hosts{
Aliases: stringutil.NewSet("alias"), ip: {{
Canonical: "hostname", Addr: ip,
Source: "file1",
Names: []string{"hostname"},
}},
anotherIP: {{
Addr: anotherIP,
Source: "file2",
Names: []string{"alias"},
}},
}) })
}) })
@ -198,12 +330,9 @@ func TestHostsContainer_refresh(t *testing.T) {
// Require the changes are written. // Require the changes are written.
require.Eventually(t, func() bool { require.Eventually(t, func() bool {
res, ok := hc.MatchRequest(&urlfilter.DNSRequest{ ips := hc.MatchName("hostname")
Hostname: "hostname",
DNSType: dns.TypeA,
})
return !ok && res.DNSRewrites() == nil return len(ips) == 0
}, 5*time.Second, time.Second/2) }, 5*time.Second, time.Second/2)
// Make a change again. // Make a change again.
@ -212,285 +341,117 @@ func TestHostsContainer_refresh(t *testing.T) {
// Require the changes are written. // Require the changes are written.
require.Eventually(t, func() bool { require.Eventually(t, func() bool {
res, ok := hc.MatchRequest(&urlfilter.DNSRequest{ ips := hc.MatchName("hostname")
Hostname: "hostname",
DNSType: dns.TypeA,
})
return !ok && res.DNSRewrites() != nil return len(ips) > 0
}, 5*time.Second, time.Second/2) }, 5*time.Second, time.Second/2)
assert.Len(t, hc.Upd(), 1) assert.Len(t, hc.Upd(), 1)
}) })
} }
func TestHostsContainer_Translate(t *testing.T) { func TestHostsContainer_MatchName(t *testing.T) {
require.NoError(t, fstest.TestFS(testdata, "etc_hosts"))
stubWatcher := aghtest.FSWatcher{ stubWatcher := aghtest.FSWatcher{
OnEvents: func() (e <-chan struct{}) { return nil }, OnEvents: func() (e <-chan struct{}) { return nil },
OnAdd: func(name string) (err error) { return nil }, OnAdd: func(name string) (err error) { return nil },
OnClose: func() (err error) { return nil }, OnClose: func() (err error) { return nil },
} }
require.NoError(t, fstest.TestFS(testdata, "etc_hosts"))
hc, err := aghnet.NewHostsContainer(0, testdata, &stubWatcher, "etc_hosts")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close)
testCases := []struct { testCases := []struct {
name string req string
rule string
wantTrans []string
}{{
name: "simplehost",
rule: "|simplehost^$dnsrewrite=NOERROR;A;1.0.0.1",
wantTrans: []string{"1.0.0.1", "simplehost"},
}, {
name: "hello",
rule: "|hello^$dnsrewrite=NOERROR;A;1.0.0.0",
wantTrans: []string{"1.0.0.0", "hello", "hello.world"},
}, {
name: "hello-alias",
rule: "|hello.world.again^$dnsrewrite=NOERROR;A;1.0.0.0",
wantTrans: []string{"1.0.0.0", "hello.world.again"},
}, {
name: "simplehost_v6",
rule: "|simplehost^$dnsrewrite=NOERROR;AAAA;::1",
wantTrans: []string{"::1", "simplehost"},
}, {
name: "hello_v6",
rule: "|hello^$dnsrewrite=NOERROR;AAAA;::",
wantTrans: []string{"::", "hello", "hello.world"},
}, {
name: "hello_v6-alias",
rule: "|hello.world.again^$dnsrewrite=NOERROR;AAAA;::",
wantTrans: []string{"::", "hello.world.again"},
}, {
name: "simplehost_ptr",
rule: "|1.0.0.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;simplehost.",
wantTrans: []string{"1.0.0.1", "simplehost"},
}, {
name: "hello_ptr",
rule: "|0.0.0.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;hello.",
wantTrans: []string{"1.0.0.0", "hello", "hello.world"},
}, {
name: "hello_ptr-alias",
rule: "|0.0.0.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;hello.world.again.",
wantTrans: []string{"1.0.0.0", "hello.world.again"},
}, {
name: "simplehost_ptr_v6",
rule: "|1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" +
"^$dnsrewrite=NOERROR;PTR;simplehost.",
wantTrans: []string{"::1", "simplehost"},
}, {
name: "hello_ptr_v6",
rule: "|0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" +
"^$dnsrewrite=NOERROR;PTR;hello.",
wantTrans: []string{"::", "hello", "hello.world"},
}, {
name: "hello_ptr_v6-alias",
rule: "|0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" +
"^$dnsrewrite=NOERROR;PTR;hello.world.again.",
wantTrans: []string{"::", "hello.world.again"},
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
got := stringutil.NewSet(strings.Fields(hc.Translate(tc.rule))...)
assert.True(t, stringutil.NewSet(tc.wantTrans...).Equal(got))
})
}
}
func TestHostsContainer(t *testing.T) {
const listID = 1234
require.NoError(t, fstest.TestFS(testdata, "etc_hosts"))
testCases := []struct {
req *urlfilter.DNSRequest
name string name string
want []*rules.DNSRewrite want []*hostsfile.Record
}{{ }{{
req: &urlfilter.DNSRequest{ req: "simplehost",
Hostname: "simplehost",
DNSType: dns.TypeA,
},
name: "simple", name: "simple",
want: []*rules.DNSRewrite{{ want: append(testHosts[addr1001], testHosts[addr1]...),
RCode: dns.RcodeSuccess,
Value: net.IPv4(1, 0, 0, 1),
RRType: dns.TypeA,
}, {
RCode: dns.RcodeSuccess,
Value: net.ParseIP("::1"),
RRType: dns.TypeAAAA,
}},
}, { }, {
req: &urlfilter.DNSRequest{ req: "hello.world",
Hostname: "hello.world",
DNSType: dns.TypeA,
},
name: "hello_alias", name: "hello_alias",
want: []*rules.DNSRewrite{{ want: []*hostsfile.Record{
RCode: dns.RcodeSuccess, testHosts[addr1000][0],
Value: net.IPv4(1, 0, 0, 0), testHosts[addr1000][2],
RRType: dns.TypeA, testHosts[addr0][0],
}, { testHosts[addr0][2],
RCode: dns.RcodeSuccess,
Value: net.ParseIP("::"),
RRType: dns.TypeAAAA,
}},
}, {
req: &urlfilter.DNSRequest{
Hostname: "hello.world.again",
DNSType: dns.TypeA,
}, },
}, {
req: "hello.world.again",
name: "other_line_alias", name: "other_line_alias",
want: []*rules.DNSRewrite{{ want: []*hostsfile.Record{
RCode: dns.RcodeSuccess, testHosts[addr1000][1],
Value: net.IPv4(1, 0, 0, 0), testHosts[addr0][1],
RRType: dns.TypeA,
}, {
RCode: dns.RcodeSuccess,
Value: net.ParseIP("::"),
RRType: dns.TypeAAAA,
}},
}, {
req: &urlfilter.DNSRequest{
Hostname: "say.hello",
DNSType: dns.TypeA,
}, },
}, {
req: "say.hello",
name: "hello_subdomain", name: "hello_subdomain",
want: []*rules.DNSRewrite{},
}, {
req: &urlfilter.DNSRequest{
Hostname: "say.hello.world",
DNSType: dns.TypeA,
},
name: "hello_alias_subdomain",
want: []*rules.DNSRewrite{},
}, {
req: &urlfilter.DNSRequest{
Hostname: "for.testing",
DNSType: dns.TypeA,
},
name: "lots_of_aliases",
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
RRType: dns.TypeA,
Value: net.IPv4(1, 0, 0, 2),
}, {
RCode: dns.RcodeSuccess,
RRType: dns.TypeAAAA,
Value: net.ParseIP("::2"),
}},
}, {
req: &urlfilter.DNSRequest{
Hostname: "1.0.0.1.in-addr.arpa",
DNSType: dns.TypePTR,
},
name: "reverse",
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
RRType: dns.TypePTR,
Value: "simplehost.",
}},
}, {
req: &urlfilter.DNSRequest{
Hostname: "nonexistent.example",
DNSType: dns.TypeA,
},
name: "non-existing",
want: []*rules.DNSRewrite{},
}, {
req: &urlfilter.DNSRequest{
Hostname: "1.0.0.1.in-addr.arpa",
DNSType: dns.TypeSRV,
},
name: "bad_type",
want: nil, want: nil,
}, { }, {
req: &urlfilter.DNSRequest{ req: "say.hello.world",
Hostname: "domain", name: "hello_alias_subdomain",
DNSType: dns.TypeA, want: nil,
}, }, {
req: "for.testing",
name: "lots_of_aliases",
want: append(testHosts[addr1002], testHosts[addr2]...),
}, {
req: "nonexistent.example",
name: "non-existing",
want: nil,
}, {
req: "domain",
name: "issue_4216_4_6", name: "issue_4216_4_6",
want: []*rules.DNSRewrite{{ want: append(testHosts[addr4216], testHosts[addr42]...),
RCode: dns.RcodeSuccess,
RRType: dns.TypeA,
Value: net.IPv4(4, 2, 1, 6),
}, {
RCode: dns.RcodeSuccess,
RRType: dns.TypeAAAA,
Value: net.ParseIP("::42"),
}},
}, { }, {
req: &urlfilter.DNSRequest{ req: "domain4",
Hostname: "domain4",
DNSType: dns.TypeA,
},
name: "issue_4216_4", name: "issue_4216_4",
want: []*rules.DNSRewrite{{ want: append(testHosts[addr1357], testHosts[addr7531]...),
RCode: dns.RcodeSuccess,
RRType: dns.TypeA,
Value: net.IPv4(7, 5, 3, 1),
}, {
RCode: dns.RcodeSuccess,
RRType: dns.TypeA,
Value: net.IPv4(1, 3, 5, 7),
}},
}, { }, {
req: &urlfilter.DNSRequest{ req: "domain6",
Hostname: "domain6",
DNSType: dns.TypeAAAA,
},
name: "issue_4216_6", name: "issue_4216_6",
want: []*rules.DNSRewrite{{ want: append(testHosts[addr13], testHosts[addr31]...),
RCode: dns.RcodeSuccess,
RRType: dns.TypeAAAA,
Value: net.ParseIP("::13"),
}, {
RCode: dns.RcodeSuccess,
RRType: dns.TypeAAAA,
Value: net.ParseIP("::31"),
}},
}} }}
hc, err := aghnet.NewHostsContainer(testdata, &stubWatcher, "etc_hosts")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
recs := hc.MatchName(tc.req)
assert.Equal(t, tc.want, recs)
})
}
}
func TestHostsContainer_MatchAddr(t *testing.T) {
require.NoError(t, fstest.TestFS(testdata, "etc_hosts"))
stubWatcher := aghtest.FSWatcher{ stubWatcher := aghtest.FSWatcher{
OnEvents: func() (e <-chan struct{}) { return nil }, OnEvents: func() (e <-chan struct{}) { return nil },
OnAdd: func(name string) (err error) { return nil }, OnAdd: func(name string) (err error) { return nil },
OnClose: func() (err error) { return nil }, OnClose: func() (err error) { return nil },
} }
hc, err := aghnet.NewHostsContainer(listID, testdata, &stubWatcher, "etc_hosts") hc, err := aghnet.NewHostsContainer(testdata, &stubWatcher, "etc_hosts")
require.NoError(t, err) require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close) testutil.CleanupAndRequireSuccess(t, hc.Close)
testCases := []struct {
req netip.Addr
name string
want []*hostsfile.Record
}{{
req: netip.AddrFrom4([4]byte{1, 0, 0, 1}),
name: "reverse",
want: testHosts[addr1001],
}}
for _, tc := range testCases { for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
res, ok := hc.MatchRequest(tc.req) recs := hc.MatchAddr(tc.req)
require.False(t, ok) assert.Equal(t, tc.want, recs)
if tc.want == nil {
assert.Nil(t, res)
return
}
require.NotNil(t, res)
rewrites := res.DNSRewrites()
require.Len(t, rewrites, len(tc.want))
for i, rewrite := range rewrites {
require.Equal(t, listID, rewrite.FilterListID)
rw := rewrite.DNSRewrite
require.NotNil(t, rw)
assert.Equal(t, tc.want[i], rw)
}
}) })
} }
} }

2
internal/dnsforward/dnsforward_test.go
View File

@ -1187,7 +1187,7 @@ func TestPTRResponseFromHosts(t *testing.T) {
} }
var eventsCalledCounter uint32 var eventsCalledCounter uint32
hc, err := aghnet.NewHostsContainer(0, testFS, &aghtest.FSWatcher{ hc, err := aghnet.NewHostsContainer(testFS, &aghtest.FSWatcher{
OnEvents: func() (e <-chan struct{}) { OnEvents: func() (e <-chan struct{}) {
assert.Equal(t, uint32(1), atomic.AddUint32(&eventsCalledCounter, 1)) assert.Equal(t, uint32(1), atomic.AddUint32(&eventsCalledCounter, 1))

8
internal/dnsforward/http.go
View File

@ -665,9 +665,11 @@ func (s *Server) parseUpstreamLine(
// dnsFilter can be nil during application update. // dnsFilter can be nil during application update.
if s.dnsFilter != nil && s.dnsFilter.EtcHosts != nil { if s.dnsFilter != nil && s.dnsFilter.EtcHosts != nil {
resolved := s.resolveUpstreamHost(extractUpstreamHost(upstreamAddr)) recs := s.dnsFilter.EtcHosts.MatchName(extractUpstreamHost(upstreamAddr))
sortNetIPAddrs(resolved, opts.PreferIPv6) for _, rec := range recs {
opts.ServerIPAddrs = resolved opts.ServerIPAddrs = append(opts.ServerIPAddrs, rec.Addr.AsSlice())
}
sortNetIPAddrs(opts.ServerIPAddrs, opts.PreferIPv6)
} }
u, err = upstream.AddressToUpstream(upstreamAddr, opts) u, err = upstream.AddressToUpstream(upstreamAddr, opts)
if err != nil { if err != nil {

1
internal/dnsforward/http_test.go
View File

@ -479,7 +479,6 @@ func TestServer_HandleTestUpstreamDNS(t *testing.T) {
}).String() }).String()
hc, err := aghnet.NewHostsContainer( hc, err := aghnet.NewHostsContainer(
filtering.SysHostsListID,
fstest.MapFS{ fstest.MapFS{
hostsFileName: &fstest.MapFile{ hostsFileName: &fstest.MapFile{
Data: []byte(hostsListener.Addr().String() + " " + upstreamHost), Data: []byte(hostsListener.Addr().String() + " " + upstreamHost),

42
internal/dnsforward/upstreams.go
View File

@ -14,8 +14,6 @@ import (
"github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil" "github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil" "github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/urlfilter"
"github.com/miekg/dns"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
) )
@ -159,17 +157,20 @@ func (s *Server) resolveUpstreamsWithHosts(
withIPs, ok := resolved[host] withIPs, ok := resolved[host]
if !ok { if !ok {
ips := s.resolveUpstreamHost(host) recs := s.dnsFilter.EtcHosts.MatchName(host)
if len(ips) == 0 { if len(recs) == 0 {
resolved[host] = nil resolved[host] = nil
return nil return nil
} }
sortNetIPAddrs(ips, opts.PreferIPv6)
withIPs = opts.Clone() withIPs = opts.Clone()
withIPs.ServerIPAddrs = ips withIPs.ServerIPAddrs = make([]net.IP, 0, len(recs))
for _, rec := range recs {
withIPs.ServerIPAddrs = append(withIPs.ServerIPAddrs, rec.Addr.AsSlice())
}
sortNetIPAddrs(withIPs.ServerIPAddrs, opts.PreferIPv6)
resolved[host] = withIPs resolved[host] = withIPs
} else if withIPs == nil { } else if withIPs == nil {
continue continue
@ -217,33 +218,6 @@ func extractUpstreamHost(addr string) (host string) {
return host return host
} }
// resolveUpstreamHost returns the version of ups with IP addresses from the
// system hosts file placed into its options.
func (s *Server) resolveUpstreamHost(host string) (addrs []net.IP) {
req := &urlfilter.DNSRequest{
Hostname: host,
DNSType: dns.TypeA,
}
aRes, _ := s.dnsFilter.EtcHosts.MatchRequest(req)
req.DNSType = dns.TypeAAAA
aaaaRes, _ := s.dnsFilter.EtcHosts.MatchRequest(req)
var ips []net.IP
for _, rw := range append(aRes.DNSRewrites(), aaaaRes.DNSRewrites()...) {
dr := rw.DNSRewrite
if dr == nil || dr.Value == nil {
continue
}
if ip, ok := dr.Value.(net.IP); ok {
ips = append(ips, ip)
}
}
return ips
}
// sortNetIPAddrs sorts addrs in accordance with the protocol preferences. // sortNetIPAddrs sorts addrs in accordance with the protocol preferences.
// Invalid addresses are sorted near the end. // Invalid addresses are sorted near the end.
// //

60
internal/filtering/dnsrewrite.go
View File

@ -1,6 +1,10 @@
package filtering package filtering
import ( import (
"net"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/urlfilter"
"github.com/AdguardTeam/urlfilter/rules" "github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns" "github.com/miekg/dns"
) )
@ -73,3 +77,59 @@ func (d *DNSFilter) processDNSRewrites(dnsr []*rules.NetworkRule) (res Result) {
Reason: RewrittenRule, Reason: RewrittenRule,
} }
} }
// processDNSResultRewrites returns an empty Result if there are no dnsrewrite
// rules in dnsres. Otherwise, it returns the processed Result.
func (d *DNSFilter) processDNSResultRewrites(
dnsres *urlfilter.DNSResult,
host string,
) (dnsRWRes Result) {
dnsr := dnsres.DNSRewrites()
if len(dnsr) == 0 {
return Result{}
}
res := d.processDNSRewrites(dnsr)
if res.Reason == RewrittenRule && res.CanonName == host {
// A rewrite of a host to itself. Go on and try matching other things.
return Result{}
}
return res
}
// appendRewriteResultFromHost appends the rewrite result from rec to vals and
// resRules.
func appendRewriteResultFromHost(
vals []rules.RRValue,
resRules []*ResultRule,
rec *hostsfile.Record,
qtype uint16,
) (updatedVals []rules.RRValue, updatedRules []*ResultRule) {
switch qtype {
case dns.TypeA:
if !rec.Addr.Is4() {
return vals, resRules
}
vals = append(vals, net.IP(rec.Addr.AsSlice()))
case dns.TypeAAAA:
if !rec.Addr.Is6() {
return vals, resRules
}
vals = append(vals, net.IP(rec.Addr.AsSlice()))
case dns.TypePTR:
for _, name := range rec.Names {
vals = append(vals, name)
}
}
recText, _ := rec.MarshalText()
resRules = append(resRules, &ResultRule{
FilterListID: SysHostsListID,
Text: string(recText),
})
return vals, resRules
}

158
internal/filtering/dnsrewrite_test.go
View File

@ -1,10 +1,17 @@
package filtering package filtering
import ( import (
"fmt"
"net" "net"
"net/netip"
"path" "path"
"testing" "testing"
"testing/fstest"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns" "github.com/miekg/dns"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -202,3 +209,154 @@ func TestDNSFilter_CheckHostRules_dnsrewrite(t *testing.T) {
assert.Equal(t, "new-ptr-with-dot.", ptr) assert.Equal(t, "new-ptr-with-dot.", ptr)
}) })
} }
func TestDNSFilter_CheckHost_hostsContainer(t *testing.T) {
addrv4 := netip.MustParseAddr("1.2.3.4")
addrv6 := netip.MustParseAddr("::1")
addrMapped := netip.MustParseAddr("::ffff:1.2.3.4")
data := fmt.Sprintf(
""+
"%s v4.host.example\n"+
"%s v6.host.example\n"+
"%s mapped.host.example\n",
addrv4,
addrv6,
addrMapped,
)
files := fstest.MapFS{
"hosts": &fstest.MapFile{
Data: []byte(data),
},
}
watcher := &aghtest.FSWatcher{
OnEvents: func() (e <-chan struct{}) { return nil },
OnAdd: func(name string) (err error) { return nil },
OnClose: func() (err error) { return nil },
}
hc, err := aghnet.NewHostsContainer(files, watcher, "hosts")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close)
f, _ := newForTest(t, &Config{EtcHosts: hc}, nil)
setts := &Settings{
FilteringEnabled: true,
}
testCases := []struct {
name string
host string
wantRules []*ResultRule
wantResps []rules.RRValue
dtyp uint16
}{{
name: "v4",
host: "v4.host.example",
dtyp: dns.TypeA,
wantRules: []*ResultRule{{
Text: "1.2.3.4 v4.host.example",
FilterListID: SysHostsListID,
}},
wantResps: []rules.RRValue{net.IP(addrv4.AsSlice())},
}, {
name: "v6",
host: "v6.host.example",
dtyp: dns.TypeAAAA,
wantRules: []*ResultRule{{
Text: "::1 v6.host.example",
FilterListID: SysHostsListID,
}},
wantResps: []rules.RRValue{net.IP(addrv6.AsSlice())},
}, {
name: "mapped",
host: "mapped.host.example",
dtyp: dns.TypeAAAA,
wantRules: []*ResultRule{{
Text: "::ffff:1.2.3.4 mapped.host.example",
FilterListID: SysHostsListID,
}},
wantResps: []rules.RRValue{net.IP(addrMapped.AsSlice())},
}, {
name: "ptr",
host: "4.3.2.1.in-addr.arpa",
dtyp: dns.TypePTR,
wantRules: []*ResultRule{{
Text: "1.2.3.4 v4.host.example",
FilterListID: SysHostsListID,
}},
wantResps: []rules.RRValue{"v4.host.example"},
}, {
name: "ptr-mapped",
host: "4.0.3.0.2.0.1.0.f.f.f.f.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa",
dtyp: dns.TypePTR,
wantRules: []*ResultRule{{
Text: "::ffff:1.2.3.4 mapped.host.example",
FilterListID: SysHostsListID,
}},
wantResps: []rules.RRValue{"mapped.host.example"},
}, {
name: "not_found_v4",
host: "non.existent.example",
dtyp: dns.TypeA,
wantRules: nil,
wantResps: nil,
}, {
name: "not_found_v6",
host: "non.existent.example",
dtyp: dns.TypeAAAA,
wantRules: nil,
wantResps: nil,
}, {
name: "not_found_ptr",
host: "4.3.2.2.in-addr.arpa",
dtyp: dns.TypePTR,
wantRules: nil,
wantResps: nil,
}, {
name: "v4_mismatch",
host: "v4.host.example",
dtyp: dns.TypeAAAA,
wantRules: nil,
wantResps: nil,
}, {
name: "v6_mismatch",
host: "v6.host.example",
dtyp: dns.TypeA,
wantRules: nil,
wantResps: nil,
}, {
name: "wrong_ptr",
host: "4.3.2.1.ip6.arpa",
dtyp: dns.TypePTR,
wantRules: nil,
wantResps: nil,
}, {
name: "unsupported_type",
host: "v4.host.example",
dtyp: dns.TypeCNAME,
wantRules: nil,
wantResps: nil,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
var res Result
res, err = f.CheckHost(tc.host, tc.dtyp, setts)
require.NoError(t, err)
if len(tc.wantRules) == 0 {
assert.Empty(t, res.Rules)
assert.Nil(t, res.DNSRewriteResult)
return
}
require.NotNil(t, res.DNSRewriteResult)
require.Contains(t, res.DNSRewriteResult.Response, tc.dtyp)
assert.Equal(t, tc.wantResps, res.DNSRewriteResult.Response[tc.dtyp])
assert.Equal(t, tc.wantRules, res.Rules)
})
}
}

106
internal/filtering/filtering.go
View File

@ -7,6 +7,7 @@ import (
"io/fs" "io/fs"
"net" "net"
"net/http" "net/http"
"net/netip"
"os" "os"
"path/filepath" "path/filepath"
"runtime" "runtime"
@ -19,8 +20,10 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
"github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/mathutil" "github.com/AdguardTeam/golibs/mathutil"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil" "github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/urlfilter" "github.com/AdguardTeam/urlfilter"
"github.com/AdguardTeam/urlfilter/filterlist" "github.com/AdguardTeam/urlfilter/filterlist"
@ -326,16 +329,6 @@ func (d *DNSFilter) WriteDiskConfig(c *Config) {
c.UserRules = slices.Clone(d.UserRules) c.UserRules = slices.Clone(d.UserRules)
} }
// cloneRewrites returns a deep copy of entries.
func cloneRewrites(entries []*LegacyRewrite) (clone []*LegacyRewrite) {
clone = make([]*LegacyRewrite, len(entries))
for i, rw := range entries {
clone[i] = rw.clone()
}
return clone
}
// setFilters sets new filters, synchronously or asynchronously. When filters // setFilters sets new filters, synchronously or asynchronously. When filters
// are set asynchronously, the old filters continue working until the new // are set asynchronously, the old filters continue working until the new
// filters are ready. // filters are ready.
@ -503,31 +496,50 @@ func (d *DNSFilter) matchSysHosts(
qtype uint16, qtype uint16,
setts *Settings, setts *Settings,
) (res Result, err error) { ) (res Result, err error) {
// TODO(e.burkov): Where else is this checked?
if !setts.FilteringEnabled || d.EtcHosts == nil { if !setts.FilteringEnabled || d.EtcHosts == nil {
return res, nil return res, nil
} }
dnsres, _ := d.EtcHosts.MatchRequest(&urlfilter.DNSRequest{ var recs []*hostsfile.Record
Hostname: host, switch qtype {
SortedClientTags: setts.ClientTags, case dns.TypeA, dns.TypeAAAA:
// TODO(e.burkov): Wait for urlfilter update to pass netip.Addr. recs = d.EtcHosts.MatchName(host)
ClientIP: setts.ClientIP.String(), case dns.TypePTR:
ClientName: setts.ClientName, var ip net.IP
DNSType: qtype, ip, err = netutil.IPFromReversedAddr(host)
}) if err != nil {
if dnsres == nil { log.Debug("filtering: failed to parse PTR record %q: %s", host, err)
return res, nil
return res, nil
}
addr, _ := netip.AddrFromSlice(ip)
recs = d.EtcHosts.MatchAddr(addr)
default:
log.Debug("filtering: unsupported query type %s", dns.Type(qtype))
} }
dnsr := dnsres.DNSRewrites() var vals []rules.RRValue
if len(dnsr) == 0 { var resRules []*ResultRule
return res, nil resRulesLen := 0
for _, rec := range recs {
vals, resRules = appendRewriteResultFromHost(vals, resRules, rec, qtype)
if len(resRules) > resRulesLen {
resRulesLen = len(resRules)
log.Debug("filtering: matched %s in %q", host, rec.Source)
}
} }
res = d.processDNSRewrites(dnsr) if len(vals) > 0 {
res.Reason = RewrittenAutoHosts res.DNSRewriteResult = &DNSRewriteResult{
for _, r := range res.Rules { Response: DNSRewriteResultResponse{
r.Text = stringutil.Coalesce(d.EtcHosts.Translate(r.Text), r.Text) qtype: vals,
},
RCode: dns.RcodeSuccess,
}
res.Rules = resRules
res.Reason = RewrittenRule
} }
return res, nil return res, nil
@ -594,25 +606,6 @@ func (d *DNSFilter) processRewrites(host string, qtype uint16) (res Result) {
return res return res
} }
// setRewriteResult sets the Reason or IPList of res if necessary. res must not
// be nil.
func setRewriteResult(res *Result, host string, rewrites []*LegacyRewrite, qtype uint16) {
for _, rw := range rewrites {
if rw.Type == qtype && (qtype == dns.TypeA || qtype == dns.TypeAAAA) {
if rw.IP == nil {
// "A"/"AAAA" exception: allow getting from upstream.
res.Reason = NotFilteredNotFound
return
}
res.IPList = append(res.IPList, rw.IP)
log.Debug("rewrite: a/aaaa for %s is %s", host, rw.IP)
}
}
}
// matchBlockedServicesRules checks the host against the blocked services rules // matchBlockedServicesRules checks the host against the blocked services rules
// in settings, if any. The err is always nil, it is only there to make this // in settings, if any. The err is always nil, it is only there to make this
// a valid hostChecker function. // a valid hostChecker function.
@ -895,26 +888,6 @@ func (d *DNSFilter) matchHost(
return res, nil return res, nil
} }
// processDNSResultRewrites returns an empty Result if there are no dnsrewrite
// rules in dnsres. Otherwise, it returns the processed Result.
func (d *DNSFilter) processDNSResultRewrites(
dnsres *urlfilter.DNSResult,
host string,
) (dnsRWRes Result) {
dnsr := dnsres.DNSRewrites()
if len(dnsr) == 0 {
return Result{}
}
res := d.processDNSRewrites(dnsr)
if res.Reason == RewrittenRule && res.CanonName == host {
// A rewrite of a host to itself. Go on and try matching other things.
return Result{}
}
return res
}
// makeResult returns a properly constructed Result. // makeResult returns a properly constructed Result.
func makeResult(matchedRules []rules.Rule, reason Reason) (res Result) { func makeResult(matchedRules []rules.Rule, reason Reason) (res Result) {
resRules := make([]*ResultRule, len(matchedRules)) resRules := make([]*ResultRule, len(matchedRules))
@ -987,7 +960,6 @@ func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) {
if d.BlockedServices != nil { if d.BlockedServices != nil {
err = d.BlockedServices.Validate() err = d.BlockedServices.Validate()
if err != nil { if err != nil {
return nil, fmt.Errorf("filtering: %w", err) return nil, fmt.Errorf("filtering: %w", err)
} }

2
internal/filtering/filtering_test.go
View File

@ -91,7 +91,7 @@ func (d *DNSFilter) checkMatchEmpty(t *testing.T, hostname string, setts *Settin
assert.Falsef(t, res.IsFiltered, "host %q", hostname) assert.Falsef(t, res.IsFiltered, "host %q", hostname)
} }
func TestEtcHostsMatching(t *testing.T) { func TestDNSFilter_CheckHost_hostRules(t *testing.T) {
addr := "216.239.38.120" addr := "216.239.38.120"
addr6 := "::1" addr6 := "::1"
text := fmt.Sprintf(` %s google.com www.google.com # enforce google's safesearch text := fmt.Sprintf(` %s google.com www.google.com # enforce google's safesearch

30
internal/filtering/rewrites.go
View File

@ -6,6 +6,7 @@ import (
"strings" "strings"
"github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/mathutil" "github.com/AdguardTeam/golibs/mathutil"
"github.com/miekg/dns" "github.com/miekg/dns"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
@ -200,3 +201,32 @@ func findRewrites(
return rewrites, matched return rewrites, matched
} }
// setRewriteResult sets the Reason or IPList of res if necessary. res must not
// be nil.
func setRewriteResult(res *Result, host string, rewrites []*LegacyRewrite, qtype uint16) {
for _, rw := range rewrites {
if rw.Type == qtype && (qtype == dns.TypeA || qtype == dns.TypeAAAA) {
if rw.IP == nil {
// "A"/"AAAA" exception: allow getting from upstream.
res.Reason = NotFilteredNotFound
return
}
res.IPList = append(res.IPList, rw.IP)
log.Debug("rewrite: a/aaaa for %s is %s", host, rw.IP)
}
}
}
// cloneRewrites returns a deep copy of entries.
func cloneRewrites(entries []*LegacyRewrite) (clone []*LegacyRewrite) {
clone = make([]*LegacyRewrite, len(entries))
for i, rw := range entries {
clone[i] = rw.clone()
}
return clone
}

10
internal/home/clients.go
View File

@ -855,15 +855,19 @@ func (clients *clientsContainer) rmHostsBySrc(src clientSource) {
// addFromHostsFile fills the client-hostname pairing index from the system's // addFromHostsFile fills the client-hostname pairing index from the system's
// hosts files. // hosts files.
func (clients *clientsContainer) addFromHostsFile(hosts aghnet.HostsRecords) { func (clients *clientsContainer) addFromHostsFile(hosts aghnet.Hosts) {
clients.lock.Lock() clients.lock.Lock()
defer clients.lock.Unlock() defer clients.lock.Unlock()
clients.rmHostsBySrc(ClientSourceHostsFile) clients.rmHostsBySrc(ClientSourceHostsFile)
n := 0 n := 0
for ip, rec := range hosts { for addr, rec := range hosts {
clients.addHostLocked(ip, rec.Canonical, ClientSourceHostsFile) // Only the first name of the first record is considered a canonical
// hostname for the IP address.
//
// TODO(e.burkov): Consider using all the names from all the records.
clients.addHostLocked(addr, rec[0].Names[0], ClientSourceHostsFile)
n++ n++
} }

1
internal/home/home.go
View File

@ -233,7 +233,6 @@ func setupHostsContainer() (err error) {
} }
Context.etcHosts, err = aghnet.NewHostsContainer( Context.etcHosts, err = aghnet.NewHostsContainer(
filtering.SysHostsListID,
aghos.RootDirFS(), aghos.RootDirFS(),
hostsWatcher, hostsWatcher,
aghnet.DefaultHostsPaths()..., aghnet.DefaultHostsPaths()...,