Pull request: all: use canonical names for hosts file runtime clients

Updates #4683.

Squashed commit of the following:

commit daa8fdaee574d4ac2171f6b13c5ce3f3fedd9801
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Jul 7 19:13:29 2022 +0300

    all: use canonical names for hosts file runtime clients
This commit is contained in:
Ainar Garipov 2022-07-07 19:33:32 +03:00
parent 14d8f58592
commit 3505ce8739
5 changed files with 164 additions and 110 deletions

View File

@ -23,7 +23,7 @@ and this project adheres to
### Added ### Added
- Support for Discovery of Designated Resolvers (DDR) according to the [RFC - Support for Discovery of Designated Resolvers (DDR) according to the [RFC
draft][ddr-draft-06] ([#4463]). draft][ddr-draft] ([#4463]).
- `windows/arm64` support ([#3057]). - `windows/arm64` support ([#3057]).
### Deprecated ### Deprecated
@ -32,6 +32,7 @@ and this project adheres to
### Fixed ### Fixed
- Inconsistent names of runtime clients from hosts files ([#4683]).
- PTR requests for addresses leased by DHCP will now be resolved into hostnames - PTR requests for addresses leased by DHCP will now be resolved into hostnames
under `dhcp.local_domain_name` ([#4699]). under `dhcp.local_domain_name` ([#4699]).
- Broken service installation on OpenWrt ([#4677]). - Broken service installation on OpenWrt ([#4677]).
@ -39,9 +40,10 @@ and this project adheres to
[#2993]: https://github.com/AdguardTeam/AdGuardHome/issues/2993 [#2993]: https://github.com/AdguardTeam/AdGuardHome/issues/2993
[#3057]: https://github.com/AdguardTeam/AdGuardHome/issues/3057 [#3057]: https://github.com/AdguardTeam/AdGuardHome/issues/3057
[#4677]: https://github.com/AdguardTeam/AdGuardHome/issues/4677 [#4677]: https://github.com/AdguardTeam/AdGuardHome/issues/4677
[#4683]: https://github.com/AdguardTeam/AdGuardHome/issues/4683
[#4699]: https://github.com/AdguardTeam/AdGuardHome/issues/4699 [#4699]: https://github.com/AdguardTeam/AdGuardHome/issues/4699
[ddr-draft-06]: https://www.ietf.org/archive/id/draft-ietf-add-ddr-06.html [ddr-draft]: https://datatracker.ietf.org/doc/html/draft-ietf-add-ddr-08

View File

@ -198,7 +198,7 @@ func (hc *HostsContainer) Close() (err error) {
} }
// Upd returns the channel into which the updates are sent. The receivable // Upd returns the channel into which the updates are sent. The receivable
// map's values are guaranteed to be of type of *stringutil.Set. // map's values are guaranteed to be of type of *HostsRecord.
func (hc *HostsContainer) Upd() (updates <-chan *netutil.IPMap) { func (hc *HostsContainer) Upd() (updates <-chan *netutil.IPMap) {
return hc.updates return hc.updates
} }
@ -290,7 +290,7 @@ func (hp *hostsParser) parseFile(r io.Reader) (patterns []string, cont bool, err
continue continue
} }
hp.addPairs(ip, hosts) hp.addRecord(ip, hosts)
} }
return nil, true, s.Err() return nil, true, s.Err()
@ -335,39 +335,66 @@ func (hp *hostsParser) parseLine(line string) (ip net.IP, hosts []string) {
return ip, hosts return ip, hosts
} }
// addPair puts the pair of ip and host to the rules builder if needed. For // HostsRecord represents a single hosts file record.
// each ip the first member of hosts will become the main one. type HostsRecord struct {
func (hp *hostsParser) addPairs(ip net.IP, hosts []string) { Aliases *stringutil.Set
Canonical string
}
// Equal returns true if all fields of rec are equal to field in other or they
// both are nil.
func (rec *HostsRecord) Equal(other *HostsRecord) (ok bool) {
if rec == nil {
return other == nil
}
return rec.Canonical == other.Canonical && rec.Aliases.Equal(other.Aliases)
}
// addRecord puts the record for the IP address to the rules builder if needed.
// The first host is considered to be the canonical name for the IP address.
// hosts must have at least one name.
func (hp *hostsParser) addRecord(ip net.IP, hosts []string) {
line := strings.Join(append([]string{ip.String()}, hosts...), " ")
var rec *HostsRecord
v, ok := hp.table.Get(ip) v, ok := hp.table.Get(ip)
if !ok { if !ok {
// This ip is added at the first time. rec = &HostsRecord{
v = stringutil.NewSet() Aliases: stringutil.NewSet(),
hp.table.Set(ip, v) }
rec.Canonical, hosts = hosts[0], hosts[1:]
hp.addRules(ip, rec.Canonical, line)
hp.table.Set(ip, rec)
} else {
rec, ok = v.(*HostsRecord)
if !ok {
log.Error("%s: adding pairs: unexpected type %T", hostsContainerPref, v)
return
}
} }
var set *stringutil.Set for _, host := range hosts {
set, ok = v.(*stringutil.Set) if rec.Canonical == host || rec.Aliases.Has(host) {
if !ok {
log.Debug("%s: adding pairs: unexpected value type %T", hostsContainerPref, v)
return
}
processed := strings.Join(append([]string{ip.String()}, hosts...), " ")
for _, h := range hosts {
if set.Has(h) {
continue continue
} }
set.Add(h) rec.Aliases.Add(host)
rule, rulePtr := hp.writeRules(h, ip) hp.addRules(ip, host, line)
hp.translations[rule], hp.translations[rulePtr] = processed, processed
log.Debug("%s: added ip-host pair %q-%q", hostsContainerPref, ip, h)
} }
} }
// addRules adds rules and rule translations for the line.
func (hp *hostsParser) addRules(ip net.IP, host, line string) {
rule, rulePtr := hp.writeRules(host, ip)
hp.translations[rule], hp.translations[rulePtr] = line, line
log.Debug("%s: added ip-host pair %q-%q", hostsContainerPref, ip, host)
}
// writeRules writes the actual rule for the qtype and the PTR for the host-ip // writeRules writes the actual rule for the qtype and the PTR for the host-ip
// pair into internal builders. // pair into internal builders.
func (hp *hostsParser) writeRules(host string, ip net.IP) (rule, rulePtr string) { func (hp *hostsParser) writeRules(host string, ip net.IP) (rule, rulePtr string) {
@ -417,6 +444,7 @@ func (hp *hostsParser) writeRules(host string, ip net.IP) (rule, rulePtr string)
} }
// equalSet returns true if the internal hosts table just parsed equals target. // equalSet returns true if the internal hosts table just parsed equals target.
// target's values must be of type *HostsRecord.
func (hp *hostsParser) equalSet(target *netutil.IPMap) (ok bool) { func (hp *hostsParser) equalSet(target *netutil.IPMap) (ok bool) {
if target == nil { if target == nil {
// hp.table shouldn't appear nil since it's initialized on each refresh. // hp.table shouldn't appear nil since it's initialized on each refresh.
@ -427,22 +455,35 @@ func (hp *hostsParser) equalSet(target *netutil.IPMap) (ok bool) {
return false return false
} }
hp.table.Range(func(ip net.IP, b interface{}) (cont bool) { hp.table.Range(func(ip net.IP, recVal interface{}) (cont bool) {
// ok is set to true if the target doesn't contain ip or if the var targetVal interface{}
// appropriate hosts set isn't equal to the checked one. targetVal, ok = target.Get(ip)
if a, hasIP := target.Get(ip); !hasIP { if !ok {
ok = true return false
} else if hosts, aok := a.(*stringutil.Set); aok {
ok = !hosts.Equal(b.(*stringutil.Set))
} }
// Continue only if maps has no discrepancies. var rec *HostsRecord
return !ok rec, ok = recVal.(*HostsRecord)
if !ok {
log.Error("%s: comparing: unexpected type %T", hostsContainerPref, recVal)
return false
}
var targetRec *HostsRecord
targetRec, ok = targetVal.(*HostsRecord)
if !ok {
log.Error("%s: comparing: target: unexpected type %T", hostsContainerPref, targetVal)
return false
}
ok = rec.Equal(targetRec)
return ok
}) })
// Return true if every value from the IP map has no discrepancies with the return ok
// appropriate one from the target.
return !ok
} }
// sendUpd tries to send the parsed data to the ch. // sendUpd tries to send the parsed data to the ch.

View File

@ -12,6 +12,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghtest" "github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil" "github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/testutil" "github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/urlfilter" "github.com/AdguardTeam/urlfilter"
@ -159,31 +160,47 @@ func TestHostsContainer_refresh(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close) testutil.CleanupAndRequireSuccess(t, hc.Close)
checkRefresh := func(t *testing.T, wantHosts *stringutil.Set) { checkRefresh := func(t *testing.T, want *HostsRecord) {
upd, ok := <-hc.Upd() t.Helper()
require.True(t, ok)
require.NotNil(t, upd) var ok bool
var upd *netutil.IPMap
select {
case upd, ok = <-hc.Upd():
require.True(t, ok)
require.NotNil(t, upd)
case <-time.After(1 * time.Second):
t.Fatal("did not receive after 1s")
}
assert.Equal(t, 1, upd.Len()) assert.Equal(t, 1, upd.Len())
v, ok := upd.Get(ip) v, ok := upd.Get(ip)
require.True(t, ok) require.True(t, ok)
var set *stringutil.Set require.IsType(t, (*HostsRecord)(nil), v)
set, ok = v.(*stringutil.Set)
require.True(t, ok)
assert.True(t, set.Equal(wantHosts)) rec, _ := v.(*HostsRecord)
require.NotNil(t, rec)
assert.Truef(t, rec.Equal(want), "%+v != %+v", rec, want)
} }
t.Run("initial_refresh", func(t *testing.T) { t.Run("initial_refresh", func(t *testing.T) {
checkRefresh(t, stringutil.NewSet("hostname")) checkRefresh(t, &HostsRecord{
Aliases: stringutil.NewSet(),
Canonical: "hostname",
})
}) })
t.Run("second_refresh", func(t *testing.T) { t.Run("second_refresh", func(t *testing.T) {
testFS["dir/file2"] = &fstest.MapFile{Data: []byte(ipStr + ` alias` + nl)} testFS["dir/file2"] = &fstest.MapFile{Data: []byte(ipStr + ` alias` + nl)}
eventsCh <- event{} eventsCh <- event{}
checkRefresh(t, stringutil.NewSet("hostname", "alias"))
checkRefresh(t, &HostsRecord{
Aliases: stringutil.NewSet("alias"),
Canonical: "hostname",
})
}) })
t.Run("double_refresh", func(t *testing.T) { t.Run("double_refresh", func(t *testing.T) {
@ -363,10 +380,15 @@ func TestHostsContainer(t *testing.T) {
require.NoError(t, fstest.TestFS(testdata, "etc_hosts")) require.NoError(t, fstest.TestFS(testdata, "etc_hosts"))
testCases := []struct { testCases := []struct {
want []*rules.DNSRewrite
name string
req *urlfilter.DNSRequest req *urlfilter.DNSRequest
name string
want []*rules.DNSRewrite
}{{ }{{
req: &urlfilter.DNSRequest{
Hostname: "simplehost",
DNSType: dns.TypeA,
},
name: "simple",
want: []*rules.DNSRewrite{{ want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess, RCode: dns.RcodeSuccess,
Value: net.IPv4(1, 0, 0, 1), Value: net.IPv4(1, 0, 0, 1),
@ -376,27 +398,12 @@ func TestHostsContainer(t *testing.T) {
Value: net.ParseIP("::1"), Value: net.ParseIP("::1"),
RRType: dns.TypeAAAA, RRType: dns.TypeAAAA,
}}, }},
name: "simple",
req: &urlfilter.DNSRequest{
Hostname: "simplehost",
DNSType: dns.TypeA,
},
}, { }, {
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
Value: net.IPv4(1, 0, 0, 0),
RRType: dns.TypeA,
}, {
RCode: dns.RcodeSuccess,
Value: net.ParseIP("::"),
RRType: dns.TypeAAAA,
}},
name: "hello_alias",
req: &urlfilter.DNSRequest{ req: &urlfilter.DNSRequest{
Hostname: "hello.world", Hostname: "hello.world",
DNSType: dns.TypeA, DNSType: dns.TypeA,
}, },
}, { name: "hello_alias",
want: []*rules.DNSRewrite{{ want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess, RCode: dns.RcodeSuccess,
Value: net.IPv4(1, 0, 0, 0), Value: net.IPv4(1, 0, 0, 0),
@ -406,26 +413,41 @@ func TestHostsContainer(t *testing.T) {
Value: net.ParseIP("::"), Value: net.ParseIP("::"),
RRType: dns.TypeAAAA, RRType: dns.TypeAAAA,
}}, }},
name: "other_line_alias", }, {
req: &urlfilter.DNSRequest{ req: &urlfilter.DNSRequest{
Hostname: "hello.world.again", Hostname: "hello.world.again",
DNSType: dns.TypeA, DNSType: dns.TypeA,
}, },
name: "other_line_alias",
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
Value: net.IPv4(1, 0, 0, 0),
RRType: dns.TypeA,
}, {
RCode: dns.RcodeSuccess,
Value: net.ParseIP("::"),
RRType: dns.TypeAAAA,
}},
}, { }, {
want: []*rules.DNSRewrite{},
name: "hello_subdomain",
req: &urlfilter.DNSRequest{ req: &urlfilter.DNSRequest{
Hostname: "say.hello", Hostname: "say.hello",
DNSType: dns.TypeA, DNSType: dns.TypeA,
}, },
}, { name: "hello_subdomain",
want: []*rules.DNSRewrite{}, want: []*rules.DNSRewrite{},
name: "hello_alias_subdomain", }, {
req: &urlfilter.DNSRequest{ req: &urlfilter.DNSRequest{
Hostname: "say.hello.world", Hostname: "say.hello.world",
DNSType: dns.TypeA, DNSType: dns.TypeA,
}, },
name: "hello_alias_subdomain",
want: []*rules.DNSRewrite{},
}, { }, {
req: &urlfilter.DNSRequest{
Hostname: "for.testing",
DNSType: dns.TypeA,
},
name: "lots_of_aliases",
want: []*rules.DNSRewrite{{ want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess, RCode: dns.RcodeSuccess,
RRType: dns.TypeA, RRType: dns.TypeA,
@ -435,37 +457,37 @@ func TestHostsContainer(t *testing.T) {
RRType: dns.TypeAAAA, RRType: dns.TypeAAAA,
Value: net.ParseIP("::2"), Value: net.ParseIP("::2"),
}}, }},
name: "lots_of_aliases",
req: &urlfilter.DNSRequest{
Hostname: "for.testing",
DNSType: dns.TypeA,
},
}, { }, {
req: &urlfilter.DNSRequest{
Hostname: "1.0.0.1.in-addr.arpa",
DNSType: dns.TypePTR,
},
name: "reverse",
want: []*rules.DNSRewrite{{ want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess, RCode: dns.RcodeSuccess,
RRType: dns.TypePTR, RRType: dns.TypePTR,
Value: "simplehost.", Value: "simplehost.",
}}, }},
name: "reverse",
req: &urlfilter.DNSRequest{
Hostname: "1.0.0.1.in-addr.arpa",
DNSType: dns.TypePTR,
},
}, { }, {
want: []*rules.DNSRewrite{},
name: "non-existing",
req: &urlfilter.DNSRequest{ req: &urlfilter.DNSRequest{
Hostname: "nonexisting", Hostname: "nonexisting",
DNSType: dns.TypeA, DNSType: dns.TypeA,
}, },
name: "non-existing",
want: []*rules.DNSRewrite{},
}, { }, {
want: nil,
name: "bad_type",
req: &urlfilter.DNSRequest{ req: &urlfilter.DNSRequest{
Hostname: "1.0.0.1.in-addr.arpa", Hostname: "1.0.0.1.in-addr.arpa",
DNSType: dns.TypeSRV, DNSType: dns.TypeSRV,
}, },
name: "bad_type",
want: nil,
}, { }, {
req: &urlfilter.DNSRequest{
Hostname: "domain",
DNSType: dns.TypeA,
},
name: "issue_4216_4_6",
want: []*rules.DNSRewrite{{ want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess, RCode: dns.RcodeSuccess,
RRType: dns.TypeA, RRType: dns.TypeA,
@ -475,12 +497,12 @@ func TestHostsContainer(t *testing.T) {
RRType: dns.TypeAAAA, RRType: dns.TypeAAAA,
Value: net.ParseIP("::42"), Value: net.ParseIP("::42"),
}}, }},
name: "issue_4216_4_6", }, {
req: &urlfilter.DNSRequest{ req: &urlfilter.DNSRequest{
Hostname: "domain", Hostname: "domain4",
DNSType: dns.TypeA, DNSType: dns.TypeA,
}, },
}, { name: "issue_4216_4",
want: []*rules.DNSRewrite{{ want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess, RCode: dns.RcodeSuccess,
RRType: dns.TypeA, RRType: dns.TypeA,
@ -490,12 +512,12 @@ func TestHostsContainer(t *testing.T) {
RRType: dns.TypeA, RRType: dns.TypeA,
Value: net.IPv4(1, 3, 5, 7), Value: net.IPv4(1, 3, 5, 7),
}}, }},
name: "issue_4216_4",
req: &urlfilter.DNSRequest{
Hostname: "domain4",
DNSType: dns.TypeA,
},
}, { }, {
req: &urlfilter.DNSRequest{
Hostname: "domain6",
DNSType: dns.TypeAAAA,
},
name: "issue_4216_6",
want: []*rules.DNSRewrite{{ want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess, RCode: dns.RcodeSuccess,
RRType: dns.TypeAAAA, RRType: dns.TypeAAAA,
@ -505,11 +527,6 @@ func TestHostsContainer(t *testing.T) {
RRType: dns.TypeAAAA, RRType: dns.TypeAAAA,
Value: net.ParseIP("::31"), Value: net.ParseIP("::31"),
}}, }},
name: "issue_4216_6",
req: &urlfilter.DNSRequest{
Hostname: "domain6",
DNSType: dns.TypeAAAA,
},
}} }}
stubWatcher := aghtest.FSWatcher{ stubWatcher := aghtest.FSWatcher{

View File

@ -19,7 +19,7 @@ type SystemResolvers interface {
} }
// NewSystemResolvers returns a SystemResolvers with the cache refresh rate // NewSystemResolvers returns a SystemResolvers with the cache refresh rate
// defined by refreshIvl. It disables auto-resfreshing if refreshIvl is 0. If // defined by refreshIvl. It disables auto-refreshing if refreshIvl is 0. If
// nil is passed for hostGenFunc, the default generator will be used. // nil is passed for hostGenFunc, the default generator will be used.
func NewSystemResolvers( func NewSystemResolvers(
hostGenFunc HostGenFunc, hostGenFunc HostGenFunc,

View File

@ -743,8 +743,7 @@ func (clients *clientsContainer) AddHost(ip net.IP, host string, src clientSourc
// addHostLocked adds a new IP-hostname pairing. For internal use only. // addHostLocked adds a new IP-hostname pairing. For internal use only.
func (clients *clientsContainer) addHostLocked(ip net.IP, host string, src clientSource) (ok bool) { func (clients *clientsContainer) addHostLocked(ip net.IP, host string, src clientSource) (ok bool) {
var rc *RuntimeClient rc, ok := clients.findRuntimeClientLocked(ip)
rc, ok = clients.findRuntimeClientLocked(ip)
if ok { if ok {
if rc.Source > src { if rc.Source > src {
return false return false
@ -799,25 +798,20 @@ func (clients *clientsContainer) addFromHostsFile(hosts *netutil.IPMap) {
n := 0 n := 0
hosts.Range(func(ip net.IP, v interface{}) (cont bool) { hosts.Range(func(ip net.IP, v interface{}) (cont bool) {
hosts, ok := v.(*stringutil.Set) rec, ok := v.(*aghnet.HostsRecord)
if !ok { if !ok {
log.Error("dns: bad type %T in ipToRC for %s", v, ip) log.Error("dns: bad type %T in ipToRC for %s", v, ip)
return true return true
} }
hosts.Range(func(name string) (cont bool) { clients.addHostLocked(ip, rec.Canonical, ClientSourceHostsFile)
if clients.addHostLocked(ip, name, ClientSourceHostsFile) { n++
n++
}
return true
})
return true return true
}) })
log.Debug("clients: added %d client aliases from system hosts-file", n) log.Debug("clients: added %d client aliases from system hosts file", n)
} }
// addFromSystemARP adds the IP-hostname pairings from the output of the arp -a // addFromSystemARP adds the IP-hostname pairings from the output of the arp -a