Pull request: 3846 hosts querylog
Merge in DNS/adguard-home from 3846-hosts-querylog to master
Updates #3846.
Squashed commit of the following:
commit 722e96628b1ccca1a5b5a716b8bcb1da2aefcc3b
Merge: a20ad71e ed868fa4
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date: Tue Nov 23 17:52:08 2021 +0300
Merge branch 'master' into 3846-hosts-querylog
commit a20ad71e723dbfa3483c3bdf9e4c8fd15c8b0e3c
Author: Ildar Kamalov <ik@adguard.com>
Date: Tue Nov 23 17:28:12 2021 +0300
client: fix variable name
commit 7013bff05d6cff75c6c25a38d614db8b4b2f0b87
Author: Ildar Kamalov <ik@adguard.com>
Date: Tue Nov 23 17:03:26 2021 +0300
client: fix missing import
commit 8e4a0fb047b4d39ab44a285f59420573d7ba5eec
Author: Ildar Kamalov <ik@adguard.com>
Date: Tue Nov 23 16:56:50 2021 +0300
client: handle system host filter id
commit abbbf662d2f3ea3f5d3569a9c45418e356adbf3c
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date: Mon Nov 22 13:54:52 2021 +0300
all: imp code
commit c2df63e46e75f84f70a610d18deccbeee672ebda
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date: Mon Nov 22 12:50:51 2021 +0300
querylog: rm unused test data
commit 8a1d47d266254fd4aedd4c61c7ea9e48168ea375
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date: Mon Nov 22 02:52:50 2021 +0300
aghnet: final imps
commit ade3acb4bebc8bdd755e56f314cdf19bc9375557
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date: Fri Nov 19 15:48:40 2021 +0300
all: add hosts container rule list support
This commit is contained in:
Eugene Burkov
parent
ed868fa46a
commit
51f11d2f8e
|
|
@ -201,6 +201,7 @@
|
|||
"form_error_url_or_path_format": "Invalid URL or absolute path of the list",
|
||||
"custom_filter_rules": "Custom filtering rules",
|
||||
"custom_filter_rules_hint": "Enter one rule on a line. You can use either adblock rules or hosts files syntax.",
|
||||
"system_host_files": "System hosts files",
|
||||
"examples_title": "Examples",
|
||||
"example_meaning_filter_block": "block access to the example.org domain and all its subdomains",
|
||||
"example_meaning_filter_whitelist": "unblock access to the example.org domain and all its subdomains",
|
||||
|
|
|
|||
|
|
@ -529,6 +529,7 @@ export const DETAILED_DATE_FORMAT_OPTIONS = {
|
|||
};
|
||||
|
||||
export const CUSTOM_FILTERING_RULES_ID = 0;
|
||||
export const SYSTEM_HOSTS_FILTER_ID = -1;
|
||||
|
||||
export const BLOCK_ACTIONS = {
|
||||
BLOCK: 'block',
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import {
|
|||
STANDARD_DNS_PORT,
|
||||
STANDARD_HTTPS_PORT,
|
||||
STANDARD_WEB_PORT,
|
||||
SYSTEM_HOSTS_FILTER_ID,
|
||||
} from './constants';
|
||||
|
||||
/**
|
||||
|
|
@ -791,9 +792,12 @@ export const getFilterName = (
|
|||
return i18n.t(customFilterTranslationKey);
|
||||
}
|
||||
|
||||
if (filterId === SYSTEM_HOSTS_FILTER_ID) {
|
||||
return i18n.t('system_host_files');
|
||||
}
|
||||
|
||||
const matchIdPredicate = (filter) => filter.id === filterId;
|
||||
const filter = filters.find(matchIdPredicate) || whitelistFilters.find(matchIdPredicate);
|
||||
|
||||
return resolveFilterName(filter);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ func (lr *limitedReader) Read(p []byte) (n int, err error) {
|
|||
}
|
||||
|
||||
if int64(len(p)) > lr.n {
|
||||
p = p[0:lr.n]
|
||||
p = p[:lr.n]
|
||||
}
|
||||
|
||||
n, err = lr.r.Read(p)
|
||||
|
|
|
|||
|
|
@ -28,6 +28,69 @@ func DefaultHostsPaths() (paths []string) {
|
|||
return defaultHostsPaths()
|
||||
}
|
||||
|
||||
// requestMatcher combines the logic for matching requests and translating the
|
||||
// appropriate rules.
|
||||
type requestMatcher struct {
|
||||
// stateLock protects all the fields of requestMatcher.
|
||||
stateLock *sync.RWMutex
|
||||
|
||||
// rulesStrg stores the rules obtained from the hosts' file.
|
||||
rulesStrg *filterlist.RuleStorage
|
||||
// engine serves rulesStrg.
|
||||
engine *urlfilter.DNSEngine
|
||||
|
||||
// translator maps generated $dnsrewrite rules into hosts-syntax rules.
|
||||
//
|
||||
// TODO(e.burkov): Store the filename from which the rule was parsed.
|
||||
translator map[string]string
|
||||
}
|
||||
|
||||
// MatchRequest processes the request rewriting hostnames and addresses read
|
||||
// from the operating system's hosts files.
|
||||
//
|
||||
// res is nil for any request having not an A/AAAA or PTR type. Results
|
||||
// containing CNAME information may be queried again with the same question type
|
||||
// and the returned CNAME for Host field of request. Results are guaranteed to
|
||||
// be direct, i.e. any returned CNAME resolves into actual address like an alias
|
||||
// in hosts does, see man hosts (5).
|
||||
//
|
||||
// It's safe for concurrent use.
|
||||
func (rm *requestMatcher) MatchRequest(
|
||||
req urlfilter.DNSRequest,
|
||||
) (res *urlfilter.DNSResult, ok bool) {
|
||||
switch req.DNSType {
|
||||
case dns.TypeA, dns.TypeAAAA, dns.TypePTR:
|
||||
log.Debug("%s: handling the request", hostsContainerPref)
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
|
||||
rm.stateLock.RLock()
|
||||
defer rm.stateLock.RUnlock()
|
||||
|
||||
return rm.engine.MatchRequest(req)
|
||||
}
|
||||
|
||||
// Translate returns the source hosts-syntax rule for the generated dnsrewrite
|
||||
// rule or an empty string if the last doesn't exist.
|
||||
func (rm *requestMatcher) Translate(rule string) (hostRule string) {
|
||||
rm.stateLock.RLock()
|
||||
defer rm.stateLock.RUnlock()
|
||||
|
||||
return rm.translator[rule]
|
||||
}
|
||||
|
||||
// resetEng updates container's engine and the translation map.
|
||||
func (rm *requestMatcher) resetEng(rulesStrg *filterlist.RuleStorage, tr map[string]string) {
|
||||
rm.stateLock.Lock()
|
||||
defer rm.stateLock.Unlock()
|
||||
|
||||
rm.rulesStrg = rulesStrg
|
||||
rm.engine = urlfilter.NewDNSEngine(rm.rulesStrg)
|
||||
|
||||
rm.translator = tr
|
||||
}
|
||||
|
||||
// hostsContainerPref is a prefix for logging and wrapping errors in
|
||||
// HostsContainer's methods.
|
||||
const hostsContainerPref = "hosts container"
|
||||
|
|
@ -35,13 +98,9 @@ const hostsContainerPref = "hosts container"
|
|||
// HostsContainer stores the relevant hosts database provided by the OS and
|
||||
// processes both A/AAAA and PTR DNS requests for those.
|
||||
type HostsContainer struct {
|
||||
// engLock protects rulesStrg and engine.
|
||||
engLock *sync.RWMutex
|
||||
|
||||
// rulesStrg stores the rules obtained from the hosts' file.
|
||||
rulesStrg *filterlist.RuleStorage
|
||||
// engine serves rulesStrg.
|
||||
engine *urlfilter.DNSEngine
|
||||
// requestMatcher matches the requests and translates the rules. It's
|
||||
// embedded to implement MatchRequest and Translate for *HostsContainer.
|
||||
requestMatcher
|
||||
|
||||
// done is the channel to sign closing the container.
|
||||
done chan struct{}
|
||||
|
|
@ -87,7 +146,9 @@ func NewHostsContainer(
|
|||
}
|
||||
|
||||
hc = &HostsContainer{
|
||||
engLock: &sync.RWMutex{},
|
||||
requestMatcher: requestMatcher{
|
||||
stateLock: &sync.RWMutex{},
|
||||
},
|
||||
done: make(chan struct{}, 1),
|
||||
updates: make(chan *netutil.IPMap, 1),
|
||||
fsys: fsys,
|
||||
|
|
@ -117,25 +178,6 @@ func NewHostsContainer(
|
|||
return hc, nil
|
||||
}
|
||||
|
||||
// MatchRequest is the request processing method to resolve hostnames and
|
||||
// addresses from the operating system's hosts files. res is nil for any
|
||||
// request having not an A/AAAA or PTR type. It's safe for concurrent use.
|
||||
func (hc *HostsContainer) MatchRequest(
|
||||
req urlfilter.DNSRequest,
|
||||
) (res *urlfilter.DNSResult, ok bool) {
|
||||
switch req.DNSType {
|
||||
case dns.TypeA, dns.TypeAAAA, dns.TypePTR:
|
||||
log.Debug("%s: handling the request", hostsContainerPref)
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
|
||||
hc.engLock.RLock()
|
||||
defer hc.engLock.RUnlock()
|
||||
|
||||
return hc.engine.MatchRequest(req)
|
||||
}
|
||||
|
||||
// Close implements the io.Closer interface for *HostsContainer. Close must
|
||||
// only be called once. The returned err is always nil.
|
||||
func (hc *HostsContainer) Close() (err error) {
|
||||
|
|
@ -203,10 +245,17 @@ func (hc *HostsContainer) handleEvents() {
|
|||
}
|
||||
|
||||
// hostsParser is a helper type to parse rules from the operating system's hosts
|
||||
// file.
|
||||
// file. It exists for only a single refreshing session.
|
||||
type hostsParser struct {
|
||||
// rules builds the resulting rules list content.
|
||||
rules *strings.Builder
|
||||
// rulesBuilder builds the resulting rulesBuilder list content.
|
||||
rulesBuilder *strings.Builder
|
||||
|
||||
// translations maps generated $dnsrewrite rules to the hosts-translations
|
||||
// rules.
|
||||
translations map[string]string
|
||||
|
||||
// cnameSet prevents duplicating cname rules.
|
||||
cnameSet *stringutil.Set
|
||||
|
||||
// table stores only the unique IP-hostname pairs. It's also sent to the
|
||||
// updates channel afterwards.
|
||||
|
|
@ -215,8 +264,11 @@ type hostsParser struct {
|
|||
|
||||
func (hc *HostsContainer) newHostsParser() (hp *hostsParser) {
|
||||
return &hostsParser{
|
||||
rules: &strings.Builder{},
|
||||
table: netutil.NewIPMap(hc.last.Len()),
|
||||
rulesBuilder: &strings.Builder{},
|
||||
// For A/AAAA and PTRs.
|
||||
translations: make(map[string]string, hc.last.Len()*2),
|
||||
cnameSet: stringutil.NewSet(),
|
||||
table: netutil.NewIPMap(hc.last.Len()),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -234,9 +286,7 @@ func (hp *hostsParser) parseFile(
|
|||
continue
|
||||
}
|
||||
|
||||
for _, host := range hosts {
|
||||
hp.addPair(ip, host)
|
||||
}
|
||||
hp.addPairs(ip, hosts)
|
||||
}
|
||||
|
||||
return nil, true, s.Err()
|
||||
|
|
@ -244,7 +294,6 @@ func (hp *hostsParser) parseFile(
|
|||
|
||||
// parseLine parses the line having the hosts syntax ignoring invalid ones.
|
||||
func (hp *hostsParser) parseLine(line string) (ip net.IP, hosts []string) {
|
||||
line = strings.TrimSpace(line)
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
return nil, nil
|
||||
|
|
@ -274,74 +323,142 @@ loop:
|
|||
return ip, hosts
|
||||
}
|
||||
|
||||
// add returns true if the pair of ip and host wasn't added to the hp before.
|
||||
func (hp *hostsParser) add(ip net.IP, host string) (added bool) {
|
||||
// Simple types of hosts in hosts database. Zero value isn't used to be able
|
||||
// quizzaciously emulate nil with 0.
|
||||
const (
|
||||
_ = iota
|
||||
hostAlias
|
||||
hostMain
|
||||
)
|
||||
|
||||
// add tries to add the ip-host pair. It returns:
|
||||
//
|
||||
// hostAlias if the host is not the first one added for the ip.
|
||||
// hostMain if the host is the first one added for the ip.
|
||||
// 0 if the ip-host pair has already been added.
|
||||
//
|
||||
func (hp *hostsParser) add(ip net.IP, host string) (hostType int) {
|
||||
v, ok := hp.table.Get(ip)
|
||||
hosts, _ := v.(*stringutil.Set)
|
||||
switch {
|
||||
switch hosts, _ := v.(*stringutil.Set); {
|
||||
case ok && hosts.Has(host):
|
||||
return false
|
||||
return 0
|
||||
case hosts == nil:
|
||||
hosts = stringutil.NewSet(host)
|
||||
hp.table.Set(ip, hosts)
|
||||
|
||||
return hostMain
|
||||
default:
|
||||
hosts.Add(host)
|
||||
}
|
||||
|
||||
return true
|
||||
return hostAlias
|
||||
}
|
||||
}
|
||||
|
||||
// addPair puts the pair of ip and host to the rules builder if needed.
|
||||
func (hp *hostsParser) addPair(ip net.IP, host string) {
|
||||
// addPair puts the pair of ip and host to the rules builder if needed. For
|
||||
// each ip the first member of hosts will become the main one.
|
||||
func (hp *hostsParser) addPairs(ip net.IP, hosts []string) {
|
||||
// Put the rule in a preproccesed format like:
|
||||
//
|
||||
// ip host1 host2 ...
|
||||
//
|
||||
hostsLine := strings.Join(append([]string{ip.String()}, hosts...), " ")
|
||||
var mainHost string
|
||||
for _, host := range hosts {
|
||||
switch hp.add(ip, host) {
|
||||
case 0:
|
||||
continue
|
||||
case hostMain:
|
||||
mainHost = host
|
||||
added, addedPtr := hp.writeMainHostRule(host, ip)
|
||||
hp.translations[added], hp.translations[addedPtr] = hostsLine, hostsLine
|
||||
case hostAlias:
|
||||
pair := fmt.Sprint(host, " ", mainHost)
|
||||
if hp.cnameSet.Has(pair) {
|
||||
continue
|
||||
}
|
||||
// Since the hostAlias couldn't be returned from add before the
|
||||
// hostMain the mainHost shouldn't appear empty.
|
||||
hp.writeAliasHostRule(host, mainHost)
|
||||
hp.cnameSet.Add(pair)
|
||||
}
|
||||
|
||||
log.Debug("%s: added ip-host pair %q-%q", hostsContainerPref, ip, host)
|
||||
}
|
||||
}
|
||||
|
||||
// writeAliasHostRule writes the CNAME rule for the alias-host pair into
|
||||
// internal builders.
|
||||
func (hp *hostsParser) writeAliasHostRule(alias, host string) {
|
||||
const (
|
||||
nl = "\n"
|
||||
sc = ";"
|
||||
|
||||
rwSuccess = rules.MaskSeparator + "$dnsrewrite=NOERROR" + sc + "CNAME" + sc
|
||||
constLen = len(rules.MaskStartURL) + len(rwSuccess) + len(nl)
|
||||
)
|
||||
|
||||
hp.rulesBuilder.Grow(constLen + len(host) + len(alias))
|
||||
stringutil.WriteToBuilder(hp.rulesBuilder, rules.MaskStartURL, alias, rwSuccess, host, nl)
|
||||
}
|
||||
|
||||
// writeMainHostRule writes the actual rule for the qtype and the PTR for the
|
||||
// host-ip pair into internal builders.
|
||||
func (hp *hostsParser) writeMainHostRule(host string, ip net.IP) (added, addedPtr string) {
|
||||
arpa, err := netutil.IPToReversedAddr(ip)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if !hp.add(ip, host) {
|
||||
return
|
||||
}
|
||||
|
||||
qtype := "AAAA"
|
||||
if ip.To4() != nil {
|
||||
// Assume the validation of the IP address is performed already.
|
||||
qtype = "A"
|
||||
}
|
||||
|
||||
const (
|
||||
nl = "\n"
|
||||
sc = ";"
|
||||
|
||||
rewriteSuccess = "$dnsrewrite=NOERROR" + sc
|
||||
rewriteSuccessPTR = rewriteSuccess + "PTR" + sc
|
||||
rwSuccess = "^$dnsrewrite=NOERROR;"
|
||||
rwSuccessPTR = "^$dnsrewrite=NOERROR;PTR;"
|
||||
|
||||
modLen = len("||") + len(rwSuccess)
|
||||
modLenPTR = len("||") + len(rwSuccessPTR)
|
||||
)
|
||||
|
||||
var qtype string
|
||||
// The validation of the IP address has been performed earlier so it is
|
||||
// guaranteed to be either an IPv4 or an IPv6.
|
||||
if ip.To4() != nil {
|
||||
qtype = "A"
|
||||
} else {
|
||||
qtype = "AAAA"
|
||||
}
|
||||
|
||||
ipStr := ip.String()
|
||||
fqdn := dns.Fqdn(host)
|
||||
|
||||
for _, ruleData := range [...][]string{{
|
||||
// A/AAAA.
|
||||
rules.MaskStartURL,
|
||||
ruleBuilder := &strings.Builder{}
|
||||
ruleBuilder.Grow(modLen + len(host) + len(qtype) + len(ipStr))
|
||||
stringutil.WriteToBuilder(
|
||||
ruleBuilder,
|
||||
"||",
|
||||
host,
|
||||
rules.MaskSeparator,
|
||||
rewriteSuccess,
|
||||
rwSuccess,
|
||||
qtype,
|
||||
sc,
|
||||
";",
|
||||
ipStr,
|
||||
nl,
|
||||
}, {
|
||||
// PTR.
|
||||
rules.MaskStartURL,
|
||||
arpa,
|
||||
rules.MaskSeparator,
|
||||
rewriteSuccessPTR,
|
||||
fqdn,
|
||||
nl,
|
||||
}} {
|
||||
stringutil.WriteToBuilder(hp.rules, ruleData...)
|
||||
}
|
||||
)
|
||||
added = ruleBuilder.String()
|
||||
|
||||
log.Debug("%s: added ip-host pair %q/%q", hostsContainerPref, ip, host)
|
||||
ruleBuilder.Reset()
|
||||
ruleBuilder.Grow(modLenPTR + len(arpa) + len(fqdn))
|
||||
stringutil.WriteToBuilder(
|
||||
ruleBuilder,
|
||||
"||",
|
||||
arpa,
|
||||
rwSuccessPTR,
|
||||
fqdn,
|
||||
)
|
||||
addedPtr = ruleBuilder.String()
|
||||
|
||||
hp.rulesBuilder.Grow(len(added) + len(addedPtr) + 2*len(nl))
|
||||
stringutil.WriteToBuilder(hp.rulesBuilder, added, nl, addedPtr, nl)
|
||||
|
||||
return added, addedPtr
|
||||
}
|
||||
|
||||
// equalSet returns true if the internal hosts table just parsed equals target.
|
||||
|
|
@ -385,15 +502,16 @@ func (hp *hostsParser) sendUpd(ch chan *netutil.IPMap) {
|
|||
case ch <- upd:
|
||||
// The previous update was just read and the next one pushed. Go on.
|
||||
default:
|
||||
log.Debug("%s: the channel is broken", hostsContainerPref)
|
||||
log.Error("%s: the updates channel is broken", hostsContainerPref)
|
||||
}
|
||||
}
|
||||
|
||||
// newStrg creates a new rules storage from parsed data.
|
||||
func (hp *hostsParser) newStrg() (s *filterlist.RuleStorage, err error) {
|
||||
return filterlist.NewRuleStorage([]filterlist.RuleList{&filterlist.StringRuleList{
|
||||
// TODO(e.burkov): Make configurable.
|
||||
ID: -1,
|
||||
RulesText: hp.rules.String(),
|
||||
RulesText: hp.rulesBuilder.String(),
|
||||
IgnoreCosmetic: true,
|
||||
}})
|
||||
}
|
||||
|
|
@ -424,15 +542,7 @@ func (hc *HostsContainer) refresh() (err error) {
|
|||
return fmt.Errorf("initializing rules storage: %w", err)
|
||||
}
|
||||
|
||||
hc.resetEng(rulesStrg)
|
||||
hc.resetEng(rulesStrg, hp.translations)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (hc *HostsContainer) resetEng(rulesStrg *filterlist.RuleStorage) {
|
||||
hc.engLock.Lock()
|
||||
defer hc.engLock.Unlock()
|
||||
|
||||
hc.rulesStrg = rulesStrg
|
||||
hc.engine = urlfilter.NewDNSEngine(hc.rulesStrg)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package aghnet
|
|||
import (
|
||||
"io/fs"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
|
@ -11,9 +12,9 @@ import (
|
|||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/netutil"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/AdguardTeam/urlfilter"
|
||||
"github.com/AdguardTeam/urlfilter/rules"
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
|
@ -203,129 +204,6 @@ func TestHostsContainer_Refresh(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestHostsContainer_MatchRequest(t *testing.T) {
|
||||
var (
|
||||
ip4 = net.IP{127, 0, 0, 1}
|
||||
ip6 = net.IP{
|
||||
128, 0, 0, 0,
|
||||
0, 0, 0, 0,
|
||||
0, 0, 0, 0,
|
||||
0, 0, 0, 1,
|
||||
}
|
||||
|
||||
hostname4 = "localhost"
|
||||
hostname6 = "localhostv6"
|
||||
hostname4a = "abcd"
|
||||
|
||||
reversed4, _ = netutil.IPToReversedAddr(ip4)
|
||||
reversed6, _ = netutil.IPToReversedAddr(ip6)
|
||||
)
|
||||
|
||||
const filename = "file1"
|
||||
|
||||
gsfs := fstest.MapFS{
|
||||
filename: &fstest.MapFile{Data: []byte(
|
||||
ip4.String() + " " + hostname4 + " " + hostname4a + nl +
|
||||
ip6.String() + " " + hostname6 + nl +
|
||||
`256.256.256.256 fakebroadcast` + nl,
|
||||
)},
|
||||
}
|
||||
|
||||
hc, err := NewHostsContainer(gsfs, &aghtest.FSWatcher{
|
||||
OnEvents: func() (e <-chan struct{}) { panic("not implemented") },
|
||||
OnAdd: func(name string) (err error) {
|
||||
assert.Equal(t, filename, name)
|
||||
|
||||
return nil
|
||||
},
|
||||
OnClose: func() (err error) { panic("not implemented") },
|
||||
}, filename)
|
||||
require.NoError(t, err)
|
||||
|
||||
testCase := []struct {
|
||||
name string
|
||||
want []interface{}
|
||||
req urlfilter.DNSRequest
|
||||
}{{
|
||||
name: "a",
|
||||
want: []interface{}{ip4.To16()},
|
||||
req: urlfilter.DNSRequest{
|
||||
Hostname: hostname4,
|
||||
DNSType: dns.TypeA,
|
||||
},
|
||||
}, {
|
||||
name: "a_for_aaaa",
|
||||
want: []interface{}{
|
||||
ip4.To16(),
|
||||
},
|
||||
req: urlfilter.DNSRequest{
|
||||
Hostname: hostname4,
|
||||
DNSType: dns.TypeAAAA,
|
||||
},
|
||||
}, {
|
||||
name: "aaaa",
|
||||
want: []interface{}{ip6},
|
||||
req: urlfilter.DNSRequest{
|
||||
Hostname: hostname6,
|
||||
DNSType: dns.TypeAAAA,
|
||||
},
|
||||
}, {
|
||||
name: "ptr",
|
||||
want: []interface{}{
|
||||
dns.Fqdn(hostname4),
|
||||
dns.Fqdn(hostname4a),
|
||||
},
|
||||
req: urlfilter.DNSRequest{
|
||||
Hostname: reversed4,
|
||||
DNSType: dns.TypePTR,
|
||||
},
|
||||
}, {
|
||||
name: "ptr_v6",
|
||||
want: []interface{}{dns.Fqdn(hostname6)},
|
||||
req: urlfilter.DNSRequest{
|
||||
Hostname: reversed6,
|
||||
DNSType: dns.TypePTR,
|
||||
},
|
||||
}, {
|
||||
name: "a_alias",
|
||||
want: []interface{}{ip4.To16()},
|
||||
req: urlfilter.DNSRequest{
|
||||
Hostname: hostname4a,
|
||||
DNSType: dns.TypeA,
|
||||
},
|
||||
}}
|
||||
|
||||
for _, tc := range testCase {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res, ok := hc.MatchRequest(tc.req)
|
||||
require.False(t, ok)
|
||||
require.NotNil(t, res)
|
||||
|
||||
rws := res.DNSRewrites()
|
||||
require.Len(t, rws, len(tc.want))
|
||||
|
||||
for i, w := range tc.want {
|
||||
require.NotNil(t, rws[i])
|
||||
|
||||
rw := rws[i].DNSRewrite
|
||||
require.NotNil(t, rw)
|
||||
|
||||
assert.Equal(t, w, rw.Value)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("cname", func(t *testing.T) {
|
||||
res, ok := hc.MatchRequest(urlfilter.DNSRequest{
|
||||
Hostname: hostname4,
|
||||
DNSType: dns.TypeCNAME,
|
||||
})
|
||||
require.False(t, ok)
|
||||
|
||||
assert.Nil(t, res)
|
||||
})
|
||||
}
|
||||
|
||||
func TestHostsContainer_PathsToPatterns(t *testing.T) {
|
||||
const (
|
||||
dir0 = "dir"
|
||||
|
|
@ -412,53 +290,108 @@ func TestHostsContainer_PathsToPatterns(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestUniqueRules_AddPair(t *testing.T) {
|
||||
knownIP := net.IP{1, 2, 3, 4}
|
||||
func TestHostsContainer(t *testing.T) {
|
||||
testdata := os.DirFS("./testdata")
|
||||
|
||||
const knownHost = "host1"
|
||||
nRewrites := func(t *testing.T, res *urlfilter.DNSResult, n int) (rws []*rules.DNSRewrite) {
|
||||
t.Helper()
|
||||
|
||||
ipToHost := netutil.NewIPMap(0)
|
||||
ipToHost.Set(knownIP, *stringutil.NewSet(knownHost))
|
||||
rewrites := res.DNSRewrites()
|
||||
assert.Len(t, rewrites, n)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
host string
|
||||
wantRules string
|
||||
ip net.IP
|
||||
}{{
|
||||
name: "new_one",
|
||||
host: "host2",
|
||||
wantRules: "||host2^$dnsrewrite=NOERROR;A;1.2.3.4\n" +
|
||||
"||4.3.2.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;host2.\n",
|
||||
ip: knownIP,
|
||||
}, {
|
||||
name: "existing_one",
|
||||
host: knownHost,
|
||||
wantRules: "||" + knownHost + "^$dnsrewrite=NOERROR;A;1.2.3.4\n" +
|
||||
"||4.3.2.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;host1.\n",
|
||||
ip: knownIP,
|
||||
}, {
|
||||
name: "new_ip",
|
||||
host: knownHost,
|
||||
wantRules: "||" + knownHost + "^$dnsrewrite=NOERROR;A;1.2.3.5\n" +
|
||||
"||5.3.2.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;" + knownHost + ".\n",
|
||||
ip: net.IP{1, 2, 3, 5},
|
||||
}, {
|
||||
name: "bad_ip",
|
||||
host: knownHost,
|
||||
wantRules: "",
|
||||
ip: net.IP{1, 2, 3, 4, 5},
|
||||
}}
|
||||
for _, rewrite := range rewrites {
|
||||
rw := rewrite.DNSRewrite
|
||||
require.NotNil(t, rw)
|
||||
|
||||
for _, tc := range testCases {
|
||||
hp := hostsParser{
|
||||
rules: &strings.Builder{},
|
||||
table: ipToHost.ShallowClone(),
|
||||
rws = append(rws, rw)
|
||||
}
|
||||
|
||||
return rws
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
testTail func(t *testing.T, res *urlfilter.DNSResult)
|
||||
name string
|
||||
req urlfilter.DNSRequest
|
||||
}{{
|
||||
name: "simple",
|
||||
req: urlfilter.DNSRequest{
|
||||
Hostname: "simplehost",
|
||||
DNSType: dns.TypeA,
|
||||
},
|
||||
testTail: func(t *testing.T, res *urlfilter.DNSResult) {
|
||||
rws := nRewrites(t, res, 2)
|
||||
|
||||
v, ok := rws[0].Value.(net.IP)
|
||||
require.True(t, ok)
|
||||
|
||||
assert.True(t, net.IP{1, 0, 0, 1}.Equal(v))
|
||||
|
||||
v, ok = rws[1].Value.(net.IP)
|
||||
require.True(t, ok)
|
||||
|
||||
// It's ::1.
|
||||
assert.True(t, net.IP(append((&[15]byte{})[:], byte(1))).Equal(v))
|
||||
},
|
||||
}, {
|
||||
name: "hello_alias",
|
||||
req: urlfilter.DNSRequest{
|
||||
Hostname: "hello.world",
|
||||
DNSType: dns.TypeA,
|
||||
},
|
||||
testTail: func(t *testing.T, res *urlfilter.DNSResult) {
|
||||
assert.Equal(t, "hello", nRewrites(t, res, 1)[0].NewCNAME)
|
||||
},
|
||||
}, {
|
||||
name: "lots_of_aliases",
|
||||
req: urlfilter.DNSRequest{
|
||||
Hostname: "for.testing",
|
||||
DNSType: dns.TypeA,
|
||||
},
|
||||
testTail: func(t *testing.T, res *urlfilter.DNSResult) {
|
||||
assert.Equal(t, "a.whole", nRewrites(t, res, 1)[0].NewCNAME)
|
||||
},
|
||||
}, {
|
||||
name: "reverse",
|
||||
req: urlfilter.DNSRequest{
|
||||
Hostname: "1.0.0.1.in-addr.arpa",
|
||||
DNSType: dns.TypePTR,
|
||||
},
|
||||
testTail: func(t *testing.T, res *urlfilter.DNSResult) {
|
||||
rws := nRewrites(t, res, 1)
|
||||
|
||||
assert.Equal(t, dns.TypePTR, rws[0].RRType)
|
||||
assert.Equal(t, "simplehost.", rws[0].Value)
|
||||
},
|
||||
}, {
|
||||
name: "non-existing",
|
||||
req: urlfilter.DNSRequest{
|
||||
Hostname: "nonexisting",
|
||||
DNSType: dns.TypeA,
|
||||
},
|
||||
testTail: func(t *testing.T, res *urlfilter.DNSResult) {
|
||||
require.NotNil(t, res)
|
||||
|
||||
assert.Nil(t, res.DNSRewrites())
|
||||
},
|
||||
}}
|
||||
|
||||
stubWatcher := aghtest.FSWatcher{
|
||||
OnEvents: func() (e <-chan struct{}) { return nil },
|
||||
OnAdd: func(name string) (err error) { return nil },
|
||||
OnClose: func() (err error) { panic("not implemented") },
|
||||
}
|
||||
|
||||
hc, err := NewHostsContainer(testdata, &stubWatcher, "etc_hosts")
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
hp.addPair(tc.ip, tc.host)
|
||||
assert.Equal(t, tc.wantRules, hp.rules.String())
|
||||
res, ok := hc.MatchRequest(tc.req)
|
||||
require.False(t, ok)
|
||||
require.NotNil(t, res)
|
||||
|
||||
tc.testTail(t, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
#
|
||||
# Test /etc/hosts file
|
||||
#
|
||||
|
||||
1.0.0.1 simplehost
|
||||
1.0.0.0 hello hello.world
|
||||
|
||||
# See https://github.com/AdguardTeam/AdGuardHome/issues/3846.
|
||||
1.0.0.2 a.whole lot.of aliases for.testing
|
||||
|
||||
# Same for IPv6.
|
||||
::1 simplehost
|
||||
:: hello hello.world
|
||||
::2 a.whole lot.of aliases for.testing
|
||||
|
|
@ -27,7 +27,7 @@ func (d *DNSFilter) processDNSRewrites(dnsr []*rules.NetworkRule) (res Result) {
|
|||
for _, nr := range dnsr {
|
||||
dr := nr.DNSRewrite
|
||||
if dr.NewCNAME != "" {
|
||||
// NewCNAME rules have a higher priority than the other rules.
|
||||
// NewCNAME rules have a higher priority than other rules.
|
||||
rules = []*ResultRule{{
|
||||
FilterListID: int64(nr.GetFilterListID()),
|
||||
Text: nr.RuleText,
|
||||
|
|
|
|||
|
|
@ -376,16 +376,8 @@ type Result struct {
|
|||
// Rules are applied rules. If Rules are not empty, each rule is not nil.
|
||||
Rules []*ResultRule `json:",omitempty"`
|
||||
|
||||
// ReverseHosts is the reverse lookup rewrite result. It is empty unless
|
||||
// Reason is set to RewrittenAutoHosts.
|
||||
//
|
||||
// TODO(e.burkov): There is no need for AutoHosts-related fields any more
|
||||
// since the hosts container now uses $dnsrewrite rules. These fields are
|
||||
// only used in query log to decode old format.
|
||||
ReverseHosts []string `json:",omitempty"`
|
||||
|
||||
// IPList is the lookup rewrite result. It is empty unless Reason is set to
|
||||
// RewrittenAutoHosts or Rewritten.
|
||||
// Rewritten.
|
||||
IPList []net.IP `json:",omitempty"`
|
||||
|
||||
// CanonName is the CNAME value from the lookup rewrite result. It is empty
|
||||
|
|
@ -464,7 +456,7 @@ func (d *DNSFilter) matchSysHosts(
|
|||
return res, nil
|
||||
}
|
||||
|
||||
dnsres, _ := d.EtcHosts.MatchRequest(urlfilter.DNSRequest{
|
||||
return d.matchSysHostsIntl(&urlfilter.DNSRequest{
|
||||
Hostname: host,
|
||||
SortedClientTags: setts.ClientTags,
|
||||
// TODO(e.burkov): Wait for urlfilter update to pass net.IP.
|
||||
|
|
@ -472,18 +464,34 @@ func (d *DNSFilter) matchSysHosts(
|
|||
ClientName: setts.ClientName,
|
||||
DNSType: qtype,
|
||||
})
|
||||
}
|
||||
|
||||
// matchSysHostsIntl actually matches the request. It's separated to avoid
|
||||
// perfoming checks twice.
|
||||
func (d *DNSFilter) matchSysHostsIntl(
|
||||
req *urlfilter.DNSRequest,
|
||||
) (res Result, err error) {
|
||||
dnsres, _ := d.EtcHosts.MatchRequest(*req)
|
||||
if dnsres == nil {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
if dnsr := dnsres.DNSRewrites(); len(dnsr) > 0 {
|
||||
// Check DNS rewrites first, because the API there is a bit awkward.
|
||||
res = d.processDNSRewrites(dnsr)
|
||||
res.Reason = RewrittenAutoHosts
|
||||
// TODO(e.burkov): Put real hosts-syntax rules.
|
||||
//
|
||||
// See https://github.com/AdguardTeam/AdGuardHome/issues/3846.
|
||||
res.Rules = nil
|
||||
dnsr := dnsres.DNSRewrites()
|
||||
if len(dnsr) == 0 {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
res = d.processDNSRewrites(dnsr)
|
||||
if cn := res.CanonName; cn != "" {
|
||||
// Probably an alias.
|
||||
req.Hostname = cn
|
||||
|
||||
return d.matchSysHostsIntl(req)
|
||||
}
|
||||
|
||||
res.Reason = RewrittenAutoHosts
|
||||
for _, r := range res.Rules {
|
||||
r.Text = stringutil.Coalesce(d.EtcHosts.Translate(r.Text), r.Text)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
|
|
@ -799,7 +807,6 @@ func (d *DNSFilter) matchHost(
|
|||
}
|
||||
|
||||
dnsres, ok := d.filteringEngine.MatchRequest(ureq)
|
||||
|
||||
// Check DNS rewrites first, because the API there is a bit awkward.
|
||||
if dnsr := dnsres.DNSRewrites(); len(dnsr) > 0 {
|
||||
res = d.processDNSRewrites(dnsr)
|
||||
|
|
|
|||
|
|
@ -291,10 +291,13 @@ func decodeResultRules(dec *json.Decoder, ent *logEntry) {
|
|||
}
|
||||
|
||||
if d, ok := keyToken.(json.Delim); ok {
|
||||
if d == '}' {
|
||||
switch d {
|
||||
case '}':
|
||||
i++
|
||||
} else if d == ']' {
|
||||
case ']':
|
||||
return
|
||||
default:
|
||||
// Go on.
|
||||
}
|
||||
|
||||
continue
|
||||
|
|
@ -312,6 +315,11 @@ func decodeResultRules(dec *json.Decoder, ent *logEntry) {
|
|||
}
|
||||
}
|
||||
|
||||
// decodeResultReverseHosts parses the dec's tokens into ent interpreting it as
|
||||
// the result of hosts container's $dnsrewrite rule. It assumes there are no
|
||||
// other occurrences of DNSRewriteResult in the entry since hosts container's
|
||||
// rewrites currently has the highest priority along the entire filtering
|
||||
// pipeline.
|
||||
func decodeResultReverseHosts(dec *json.Decoder, ent *logEntry) {
|
||||
for {
|
||||
itemToken, err := dec.Token()
|
||||
|
|
@ -335,7 +343,25 @@ func decodeResultReverseHosts(dec *json.Decoder, ent *logEntry) {
|
|||
|
||||
return
|
||||
case string:
|
||||
ent.Result.ReverseHosts = append(ent.Result.ReverseHosts, v)
|
||||
v = dns.Fqdn(v)
|
||||
if res := &ent.Result; res.DNSRewriteResult == nil {
|
||||
res.DNSRewriteResult = &filtering.DNSRewriteResult{
|
||||
RCode: dns.RcodeSuccess,
|
||||
Response: filtering.DNSRewriteResultResponse{
|
||||
dns.TypePTR: []rules.RRValue{v},
|
||||
},
|
||||
}
|
||||
|
||||
continue
|
||||
} else {
|
||||
res.DNSRewriteResult.RCode = dns.RcodeSuccess
|
||||
}
|
||||
|
||||
if rres := ent.Result.DNSRewriteResult; rres.Response == nil {
|
||||
rres.Response = filtering.DNSRewriteResultResponse{dns.TypePTR: []rules.RRValue{v}}
|
||||
} else {
|
||||
rres.Response[dns.TypePTR] = append(rres.Response[dns.TypePTR], v)
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
|
|
@ -407,9 +433,9 @@ func decodeResultDNSRewriteResultKey(key string, dec *json.Decoder, ent *logEntr
|
|||
ent.Result.DNSRewriteResult.Response = filtering.DNSRewriteResultResponse{}
|
||||
}
|
||||
|
||||
// TODO(a.garipov): I give up. This whole file is a mess.
|
||||
// Luckily, we can assume that this field is relatively rare and
|
||||
// just use the normal decoding and correct the values.
|
||||
// TODO(a.garipov): I give up. This whole file is a mess. Luckily, we
|
||||
// can assume that this field is relatively rare and just use the normal
|
||||
// decoding and correct the values.
|
||||
err = dec.Decode(&ent.Result.DNSRewriteResult.Response)
|
||||
if err != nil {
|
||||
log.Debug("decodeResultDNSRewriteResultKey response err: %s", err)
|
||||
|
|
@ -463,7 +489,40 @@ func decodeResultDNSRewriteResult(dec *json.Decoder, ent *logEntry) {
|
|||
}
|
||||
}
|
||||
|
||||
// translateResult converts some fields of the ent.Result to the format
|
||||
// consistent with current implementation.
|
||||
func translateResult(ent *logEntry) {
|
||||
res := &ent.Result
|
||||
if res.Reason != filtering.RewrittenAutoHosts || len(res.IPList) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if res.DNSRewriteResult == nil {
|
||||
res.DNSRewriteResult = &filtering.DNSRewriteResult{
|
||||
RCode: dns.RcodeSuccess,
|
||||
}
|
||||
}
|
||||
|
||||
if res.DNSRewriteResult.Response == nil {
|
||||
res.DNSRewriteResult.Response = filtering.DNSRewriteResultResponse{}
|
||||
}
|
||||
|
||||
resp := res.DNSRewriteResult.Response
|
||||
for _, ip := range res.IPList {
|
||||
qType := dns.TypeAAAA
|
||||
if ip.To4() != nil {
|
||||
qType = dns.TypeA
|
||||
}
|
||||
|
||||
resp[qType] = append(resp[qType], ip)
|
||||
}
|
||||
|
||||
res.IPList = nil
|
||||
}
|
||||
|
||||
func decodeResult(dec *json.Decoder, ent *logEntry) {
|
||||
defer translateResult(ent)
|
||||
|
||||
for {
|
||||
keyToken, err := dec.Token()
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ func TestDecodeLogEntry(t *testing.T) {
|
|||
`"Result":{` +
|
||||
`"IsFiltered":true,` +
|
||||
`"Reason":3,` +
|
||||
`"ReverseHosts":["example.net"],` +
|
||||
`"IPList":["127.0.0.2"],` +
|
||||
`"Rules":[{"FilterListID":42,"Text":"||an.yandex.ru","IP":"127.0.0.2"},` +
|
||||
`{"FilterListID":43,"Text":"||an2.yandex.ru","IP":"127.0.0.3"}],` +
|
||||
|
|
@ -58,10 +57,9 @@ func TestDecodeLogEntry(t *testing.T) {
|
|||
ClientProto: "",
|
||||
Answer: ans,
|
||||
Result: filtering.Result{
|
||||
IsFiltered: true,
|
||||
Reason: filtering.FilteredBlockList,
|
||||
ReverseHosts: []string{"example.net"},
|
||||
IPList: []net.IP{net.IPv4(127, 0, 0, 2)},
|
||||
IsFiltered: true,
|
||||
Reason: filtering.FilteredBlockList,
|
||||
IPList: []net.IP{net.IPv4(127, 0, 0, 2)},
|
||||
Rules: []*filtering.ResultRule{{
|
||||
FilterListID: 42,
|
||||
Text: "||an.yandex.ru",
|
||||
|
|
@ -170,8 +168,7 @@ func TestDecodeLogEntry(t *testing.T) {
|
|||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
l := &logEntry{}
|
||||
decodeLogEntry(l, tc.log)
|
||||
decodeLogEntry(new(logEntry), tc.log)
|
||||
|
||||
s := logOutput.String()
|
||||
if tc.want == "" {
|
||||
|
|
@ -185,3 +182,65 @@ func TestDecodeLogEntry(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeLogEntry_backwardCompatability(t *testing.T) {
|
||||
var (
|
||||
a1, a2 = net.IP{127, 0, 0, 1}.To16(), net.IP{127, 0, 0, 2}.To16()
|
||||
aaaa1, aaaa2 = net.ParseIP("::1"), net.ParseIP("::2")
|
||||
)
|
||||
|
||||
testCases := []struct {
|
||||
want *logEntry
|
||||
entry string
|
||||
name string
|
||||
}{{
|
||||
entry: `{"Result":{"ReverseHosts":["example.net","example.org"]}`,
|
||||
want: &logEntry{
|
||||
Result: filtering.Result{DNSRewriteResult: &filtering.DNSRewriteResult{
|
||||
RCode: dns.RcodeSuccess,
|
||||
Response: filtering.DNSRewriteResultResponse{
|
||||
dns.TypePTR: []rules.RRValue{"example.net.", "example.org."},
|
||||
},
|
||||
}},
|
||||
},
|
||||
name: "reverse_hosts",
|
||||
}, {
|
||||
entry: `{"Result":{"IPList":["127.0.0.1","127.0.0.2","::1","::2"],"Reason":10}}`,
|
||||
want: &logEntry{
|
||||
Result: filtering.Result{
|
||||
DNSRewriteResult: &filtering.DNSRewriteResult{
|
||||
RCode: dns.RcodeSuccess,
|
||||
Response: filtering.DNSRewriteResultResponse{
|
||||
dns.TypeA: []rules.RRValue{a1, a2},
|
||||
dns.TypeAAAA: []rules.RRValue{aaaa1, aaaa2},
|
||||
},
|
||||
},
|
||||
Reason: filtering.RewrittenAutoHosts,
|
||||
},
|
||||
},
|
||||
name: "iplist_autohosts",
|
||||
}, {
|
||||
entry: `{"Result":{"IPList":["127.0.0.1","127.0.0.2","::1","::2"],"Reason":9}}`,
|
||||
want: &logEntry{
|
||||
Result: filtering.Result{
|
||||
IPList: []net.IP{
|
||||
a1,
|
||||
a2,
|
||||
aaaa1,
|
||||
aaaa2,
|
||||
},
|
||||
Reason: filtering.Rewritten,
|
||||
},
|
||||
},
|
||||
name: "iplist_rewritten",
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
e := &logEntry{}
|
||||
decodeLogEntry(e, tc.entry)
|
||||
|
||||
assert.Equal(t, tc.want, e)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue