diff --git a/.githooks/pre-commit b/.githooks/pre-commit index f2bbf1c9..278d7d3e 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -2,12 +2,12 @@ set -e -f -u -if [ "$(git diff --cached --name-only '*.js')" ] +if [ "$(git diff --cached --name-only -- '*.js')" ] then make js-lint js-test fi -if [ "$(git diff --cached --name-only '*.go')" ] +if [ "$(git diff --cached --name-only -- '*.go' 'go.mod')" ] then make go-lint go-test fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fdf434a..e63c913c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to ### Added +- `$dnsrewrite` modifier for filters ([#2102]). - The host checking API and the query logs API can now return multiple matched rules ([#2102]). - Detecting of network interface configured to have static IP address via diff --git a/Makefile b/Makefile index 1848fe34..2bdee355 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,7 @@ GPG_KEY := devteam@adguard.com GPG_KEY_PASSPHRASE := GPG_CMD := gpg --detach-sig --default-key $(GPG_KEY) --pinentry-mode loopback --passphrase $(GPG_KEY_PASSPHRASE) VERBOSE := -v +REBUILD_CLIENT = 1 # See release target DIST_DIR=dist @@ -124,7 +125,8 @@ all: build init: git config core.hooksPath .githooks -build: client_with_deps +build: + test '$(REBUILD_CLIENT)' = '1' && $(MAKE) client_with_deps || exit 0 $(GO) mod download PATH=$(GOPATH)/bin:$(PATH) $(GO) generate ./... CGO_ENABLED=0 $(GO) build -ldflags="-s -w -X main.version=$(VERSION) -X main.channel=$(CHANNEL) -X main.goarm=$(GOARM)" diff --git a/README.md b/README.md index b38452d7..eeb8f10e 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ curl -sSL https://raw.githubusercontent.com/AdguardTeam/AdGuardHome/master/scrip * Beta channel builds * Linux: [64-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_amd64.tar.gz), [32-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_386.tar.gz) - * Linux ARM: [32-bit ARMv6](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv6.tar.gz) (recommended for Rapsberry Pi), [64-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_arm64.tar.gz), [32-bit ARMv5](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv5.tar.gz), [32-bit ARMv7](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv7.tar.gz) + * Linux ARM: [32-bit ARMv6](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv6.tar.gz) (recommended for Raspberry Pi), [64-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_arm64.tar.gz), [32-bit ARMv5](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv5.tar.gz), [32-bit ARMv7](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv7.tar.gz) * Linux MIPS: [32-bit MIPS](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_mips_softfloat.tar.gz), [32-bit MIPSLE](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_mipsle_softfloat.tar.gz), [64-bit MIPS](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_mips64_softfloat.tar.gz), [64-bit MIPSLE](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_mips64le_softfloat.tar.gz) * Windows: [64-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_windows_amd64.zip), [32-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_windows_386.zip) * MacOS: [64-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_darwin_amd64.zip), [32-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_darwin_386.zip) @@ -264,7 +264,7 @@ curl -sSL https://raw.githubusercontent.com/AdguardTeam/AdGuardHome/master/scrip * Edge channel builds * Linux: [64-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_amd64.tar.gz), [32-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_386.tar.gz) - * Linux ARM: [32-bit ARMv6](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv6.tar.gz) (recommended for Rapsberry Pi), [64-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_arm64.tar.gz), [32-bit ARMv5](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv5.tar.gz), [32-bit ARMv7](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv7.tar.gz) + * Linux ARM: [32-bit ARMv6](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv6.tar.gz) (recommended for Raspberry Pi), [64-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_arm64.tar.gz), [32-bit ARMv5](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv5.tar.gz), [32-bit ARMv7](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv7.tar.gz) * Linux MIPS: [32-bit MIPS](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_mips_softfloat.tar.gz), [32-bit MIPSLE](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_mipsle_softfloat.tar.gz), [64-bit MIPS](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_mips64_softfloat.tar.gz), [64-bit MIPSLE](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_mips64le_softfloat.tar.gz) * Windows: [64-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_windows_amd64.zip), [32-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_windows_386.zip) * MacOS: [64-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_darwin_amd64.zip), [32-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_darwin_386.zip) diff --git a/go.mod b/go.mod index 81d2e248..10866d88 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( github.com/AdguardTeam/dnsproxy v0.33.7 github.com/AdguardTeam/golibs v0.4.4 - github.com/AdguardTeam/urlfilter v0.13.0 + github.com/AdguardTeam/urlfilter v0.14.0 github.com/NYTimes/gziphandler v1.1.1 github.com/ameshkov/dnscrypt/v2 v2.0.1 github.com/fsnotify/fsnotify v1.4.9 diff --git a/go.sum b/go.sum index 37dd5867..764a2fa0 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/AdguardTeam/golibs v0.4.2/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKU github.com/AdguardTeam/golibs v0.4.4 h1:cM9UySQiYFW79zo5XRwnaIWVzfW4eNXmZktMrWbthpw= github.com/AdguardTeam/golibs v0.4.4/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4= github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU= -github.com/AdguardTeam/urlfilter v0.13.0 h1:MfO46K81JVTkhgP6gRu/buKl5wAOSfusjiDwjT1JN1c= -github.com/AdguardTeam/urlfilter v0.13.0/go.mod h1:klx4JbOfc4EaNb5lWLqOwfg+pVcyRukmoJRvO55lL5U= +github.com/AdguardTeam/urlfilter v0.14.0 h1:+aAhOvZDVGzl5gTERB4pOJCL1zxMyw7vLecJJ6TQTCw= +github.com/AdguardTeam/urlfilter v0.14.0/go.mod h1:klx4JbOfc4EaNb5lWLqOwfg+pVcyRukmoJRvO55lL5U= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= diff --git a/internal/dnsfilter/dnsfilter.go b/internal/dnsfilter/dnsfilter.go index 1735154d..4a1b255e 100644 --- a/internal/dnsfilter/dnsfilter.go +++ b/internal/dnsfilter/dnsfilter.go @@ -1,4 +1,4 @@ -// Package dnsfilter implements a DNS filter. +// Package dnsfilter implements a DNS request and response filter. package dnsfilter import ( @@ -95,8 +95,8 @@ type filtersInitializerParams struct { type DNSFilter struct { rulesStorage *filterlist.RuleStorage filteringEngine *urlfilter.DNSEngine - rulesStorageWhite *filterlist.RuleStorage - filteringEngineWhite *urlfilter.DNSEngine + rulesStorageAllow *filterlist.RuleStorage + filteringEngineAllow *urlfilter.DNSEngine engineLock sync.RWMutex parentalServer string // access via methods @@ -127,16 +127,16 @@ const ( // NotFilteredNotFound - host was not find in any checks, default value for result NotFilteredNotFound Reason = iota - // NotFilteredWhiteList - the host is explicitly whitelisted - NotFilteredWhiteList + // NotFilteredAllowList - the host is explicitly allowed + NotFilteredAllowList // NotFilteredError is returned when there was an error during // checking. Reserved, currently unused. NotFilteredError // reasons for filtering - // FilteredBlackList - the host was matched to be advertising host - FilteredBlackList + // FilteredBlockList - the host was matched to be advertising host + FilteredBlockList // FilteredSafeBrowsing - the host was matched to be malicious/phishing FilteredSafeBrowsing // FilteredParental - the host was matched to be outside of parental control settings @@ -155,16 +155,20 @@ const ( // RewriteAutoHosts is returned when there was a rewrite by // autohosts rules (/etc/hosts and so on). RewriteAutoHosts + + // DNSRewriteRule is returned when a $dnsrewrite filter rule was + // applied. + DNSRewriteRule ) // TODO(a.garipov): Resync with actual code names or replace completely // in HTTP API v1. var reasonNames = []string{ NotFilteredNotFound: "NotFilteredNotFound", - NotFilteredWhiteList: "NotFilteredWhiteList", + NotFilteredAllowList: "NotFilteredWhiteList", NotFilteredError: "NotFilteredError", - FilteredBlackList: "FilteredBlackList", + FilteredBlockList: "FilteredBlackList", FilteredSafeBrowsing: "FilteredSafeBrowsing", FilteredParental: "FilteredParental", FilteredInvalid: "FilteredInvalid", @@ -174,12 +178,15 @@ var reasonNames = []string{ ReasonRewrite: "Rewrite", RewriteAutoHosts: "RewriteEtcHosts", + + DNSRewriteRule: "DNSRewriteRule", } func (r Reason) String() string { - if uint(r) >= uint(len(reasonNames)) { + if r < 0 || int(r) >= len(reasonNames) { return "" } + return reasonNames[r] } @@ -278,16 +285,15 @@ func (d *DNSFilter) reset() { } } - if d.rulesStorageWhite != nil { - err = d.rulesStorageWhite.Close() + if d.rulesStorageAllow != nil { + err = d.rulesStorageAllow.Close() if err != nil { - log.Error("dnsfilter: rulesStorageWhite.Close: %s", err) + log.Error("dnsfilter: rulesStorageAllow.Close: %s", err) } } } type dnsFilterContext struct { - stats Stats safebrowsingCache cache.Cache parentalCache cache.Cache safeSearchCache cache.Cache @@ -339,6 +345,9 @@ type Result struct { // ServiceName is the name of the blocked service. It is empty // unless Reason is set to FilteredBlockedService. ServiceName string `json:",omitempty"` + + // DNSRewriteResult is the $dnsrewrite filter rule result. + DNSRewriteResult *DNSRewriteResult `json:",omitempty"` } // Matched returns true if any match at all was found regardless of @@ -383,9 +392,6 @@ func (d *DNSFilter) CheckHost(host string, qtype uint16, setts *RequestFiltering } } - // Then check the filter lists. - // if request is blocked -- it should be blocked. - // if it is whitelisted -- we should do nothing with it anymore. if setts.FilteringEnabled { result, err = d.matchHost(host, qtype, *setts) if err != nil { @@ -476,9 +482,7 @@ func (d *DNSFilter) checkAutoHosts(host string, qtype uint16, result *Result) (m // . repeat for the new domain name (Note: we return only the last CNAME) // . Find A or AAAA record for a domain name (exact match or by wildcard) // . if found, set IP addresses (IPv4 or IPv6 depending on qtype) in Result.IPList array -func (d *DNSFilter) processRewrites(host string, qtype uint16) Result { - var res Result - +func (d *DNSFilter) processRewrites(host string, qtype uint16) (res Result) { d.confLock.RLock() defer d.confLock.RUnlock() @@ -493,7 +497,8 @@ func (d *DNSFilter) processRewrites(host string, qtype uint16) Result { log.Debug("Rewrite: CNAME for %s is %s", host, rr[0].Answer) if host == rr[0].Answer { // "host == CNAME" is an exception - res.Reason = 0 + res.Reason = NotFilteredNotFound + return res } @@ -616,7 +621,7 @@ func (d *DNSFilter) initFiltering(allowFilters, blockFilters []Filter) error { if err != nil { return err } - rulesStorageWhite, filteringEngineWhite, err := createFilteringEngine(allowFilters) + rulesStorageAllow, filteringEngineAllow, err := createFilteringEngine(allowFilters) if err != nil { return err } @@ -625,8 +630,8 @@ func (d *DNSFilter) initFiltering(allowFilters, blockFilters []Filter) error { d.reset() d.rulesStorage = rulesStorage d.filteringEngine = filteringEngine - d.rulesStorageWhite = rulesStorageWhite - d.filteringEngineWhite = filteringEngineWhite + d.rulesStorageAllow = rulesStorageAllow + d.filteringEngineAllow = filteringEngineAllow d.engineLock.Unlock() // Make sure that the OS reclaims memory as soon as possible @@ -636,9 +641,31 @@ func (d *DNSFilter) initFiltering(allowFilters, blockFilters []Filter) error { return nil } +// matchHostProcessAllowList processes the allowlist logic of host +// matching. +func (d *DNSFilter) matchHostProcessAllowList(host string, dnsres urlfilter.DNSResult) (res Result, err error) { + var rule rules.Rule + if dnsres.NetworkRule != nil { + rule = dnsres.NetworkRule + } else if len(dnsres.HostRulesV4) > 0 { + rule = dnsres.HostRulesV4[0] + } else if len(dnsres.HostRulesV6) > 0 { + rule = dnsres.HostRulesV6[0] + } + + if rule == nil { + return Result{}, fmt.Errorf("invalid dns result: rules are empty") + } + + log.Debug("Filtering: found allowlist rule for host %q: %q list_id: %d", + host, rule.Text(), rule.GetFilterListID()) + + return makeResult(rule, NotFilteredAllowList), nil +} + // matchHost is a low-level way to check only if hostname is filtered by rules, // skipping expensive safebrowsing and parental lookups. -func (d *DNSFilter) matchHost(host string, qtype uint16, setts RequestFilteringSettings) (Result, error) { +func (d *DNSFilter) matchHost(host string, qtype uint16, setts RequestFilteringSettings) (res Result, err error) { d.engineLock.RLock() // Keep in mind that this lock must be held no just when calling Match() // but also while using the rules returned by it. @@ -652,22 +679,10 @@ func (d *DNSFilter) matchHost(host string, qtype uint16, setts RequestFilteringS DNSType: qtype, } - if d.filteringEngineWhite != nil { - rr, ok := d.filteringEngineWhite.MatchRequest(ureq) + if d.filteringEngineAllow != nil { + dnsres, ok := d.filteringEngineAllow.MatchRequest(ureq) if ok { - var rule rules.Rule - if rr.NetworkRule != nil { - rule = rr.NetworkRule - } else if rr.HostRulesV4 != nil { - rule = rr.HostRulesV4[0] - } else if rr.HostRulesV6 != nil { - rule = rr.HostRulesV6[0] - } - - log.Debug("Filtering: found whitelist rule for host %q: %q list_id: %d", - host, rule.Text(), rule.GetFilterListID()) - res := makeResult(rule, NotFilteredWhiteList) - return res, nil + return d.matchHostProcessAllowList(host, dnsres) } } @@ -675,54 +690,65 @@ func (d *DNSFilter) matchHost(host string, qtype uint16, setts RequestFilteringS return Result{}, nil } - rr, ok := d.filteringEngine.MatchRequest(ureq) - if !ok { + dnsres, ok := d.filteringEngine.MatchRequest(ureq) + + // Check DNS rewrites first, because the API there is a bit + // awkward. + if dnsr := dnsres.DNSRewrites(); len(dnsr) > 0 { + res = d.processDNSRewrites(dnsr) + if res.Reason == DNSRewriteRule && res.CanonName == host { + // A rewrite of a host to itself. Go on and + // try matching other things. + } else { + return res, nil + } + } else if !ok { return Result{}, nil } - if rr.NetworkRule != nil { + if dnsres.NetworkRule != nil { log.Debug("Filtering: found rule for host %q: %q list_id: %d", - host, rr.NetworkRule.Text(), rr.NetworkRule.GetFilterListID()) - reason := FilteredBlackList - if rr.NetworkRule.Whitelist { - reason = NotFilteredWhiteList + host, dnsres.NetworkRule.Text(), dnsres.NetworkRule.GetFilterListID()) + reason := FilteredBlockList + if dnsres.NetworkRule.Whitelist { + reason = NotFilteredAllowList } - res := makeResult(rr.NetworkRule, reason) - return res, nil + + return makeResult(dnsres.NetworkRule, reason), nil } - if qtype == dns.TypeA && rr.HostRulesV4 != nil { - rule := rr.HostRulesV4[0] // note that we process only 1 matched rule + if qtype == dns.TypeA && dnsres.HostRulesV4 != nil { + rule := dnsres.HostRulesV4[0] // note that we process only 1 matched rule log.Debug("Filtering: found rule for host %q: %q list_id: %d", host, rule.Text(), rule.GetFilterListID()) - res := makeResult(rule, FilteredBlackList) + res = makeResult(rule, FilteredBlockList) res.Rules[0].IP = rule.IP.To4() return res, nil } - if qtype == dns.TypeAAAA && rr.HostRulesV6 != nil { - rule := rr.HostRulesV6[0] // note that we process only 1 matched rule + if qtype == dns.TypeAAAA && dnsres.HostRulesV6 != nil { + rule := dnsres.HostRulesV6[0] // note that we process only 1 matched rule log.Debug("Filtering: found rule for host %q: %q list_id: %d", host, rule.Text(), rule.GetFilterListID()) - res := makeResult(rule, FilteredBlackList) + res = makeResult(rule, FilteredBlockList) res.Rules[0].IP = rule.IP return res, nil } - if rr.HostRulesV4 != nil || rr.HostRulesV6 != nil { + if dnsres.HostRulesV4 != nil || dnsres.HostRulesV6 != nil { // Question Type doesn't match the host rules // Return the first matched host rule, but without an IP address var rule rules.Rule - if rr.HostRulesV4 != nil { - rule = rr.HostRulesV4[0] - } else if rr.HostRulesV6 != nil { - rule = rr.HostRulesV6[0] + if dnsres.HostRulesV4 != nil { + rule = dnsres.HostRulesV4[0] + } else if dnsres.HostRulesV6 != nil { + rule = dnsres.HostRulesV6[0] } log.Debug("Filtering: found rule for host %q: %q list_id: %d", host, rule.Text(), rule.GetFilterListID()) - res := makeResult(rule, FilteredBlackList) + res = makeResult(rule, FilteredBlockList) res.Rules[0].IP = net.IP{} return res, nil @@ -741,7 +767,7 @@ func makeResult(rule rules.Rule, reason Reason) Result { }}, } - if reason == FilteredBlackList { + if reason == FilteredBlockList { res.IsFiltered = true } diff --git a/internal/dnsfilter/dnsfilter_test.go b/internal/dnsfilter/dnsfilter_test.go index 96376162..2bae12de 100644 --- a/internal/dnsfilter/dnsfilter_test.go +++ b/internal/dnsfilter/dnsfilter_test.go @@ -178,7 +178,6 @@ func TestSafeBrowsing(t *testing.T) { d := NewForTest(&Config{SafeBrowsingEnabled: true}, nil) defer d.Close() - gctx.stats.Safebrowsing.Requests = 0 d.checkMatch(t, "wmconvirus.narod.ru") assert.True(t, strings.Contains(logOutput.String(), "SafeBrowsing lookup for wmconvirus.narod.ru")) @@ -366,7 +365,7 @@ const nl = "\n" const ( blockingRules = `||example.org^` + nl - whitelistRules = `||example.org^` + nl + `@@||test.example.org` + nl + allowlistRules = `||example.org^` + nl + `@@||test.example.org` + nl importantRules = `@@||example.org^` + nl + `||test.example.org^$important` + nl regexRules = `/example\.org/` + nl + `@@||test.example.org^` + nl maskRules = `test*.example.org^` + nl + `exam*.com` + nl @@ -381,49 +380,49 @@ var tests = []struct { reason Reason dnsType uint16 }{ - {"sanity", "||doubleclick.net^", "www.doubleclick.net", true, FilteredBlackList, dns.TypeA}, + {"sanity", "||doubleclick.net^", "www.doubleclick.net", true, FilteredBlockList, dns.TypeA}, {"sanity", "||doubleclick.net^", "nodoubleclick.net", false, NotFilteredNotFound, dns.TypeA}, {"sanity", "||doubleclick.net^", "doubleclick.net.ru", false, NotFilteredNotFound, dns.TypeA}, {"sanity", "||doubleclick.net^", "wmconvirus.narod.ru", false, NotFilteredNotFound, dns.TypeA}, - {"blocking", blockingRules, "example.org", true, FilteredBlackList, dns.TypeA}, - {"blocking", blockingRules, "test.example.org", true, FilteredBlackList, dns.TypeA}, - {"blocking", blockingRules, "test.test.example.org", true, FilteredBlackList, dns.TypeA}, + {"blocking", blockingRules, "example.org", true, FilteredBlockList, dns.TypeA}, + {"blocking", blockingRules, "test.example.org", true, FilteredBlockList, dns.TypeA}, + {"blocking", blockingRules, "test.test.example.org", true, FilteredBlockList, dns.TypeA}, {"blocking", blockingRules, "testexample.org", false, NotFilteredNotFound, dns.TypeA}, {"blocking", blockingRules, "onemoreexample.org", false, NotFilteredNotFound, dns.TypeA}, - {"whitelist", whitelistRules, "example.org", true, FilteredBlackList, dns.TypeA}, - {"whitelist", whitelistRules, "test.example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"whitelist", whitelistRules, "test.test.example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"whitelist", whitelistRules, "testexample.org", false, NotFilteredNotFound, dns.TypeA}, - {"whitelist", whitelistRules, "onemoreexample.org", false, NotFilteredNotFound, dns.TypeA}, + {"allowlist", allowlistRules, "example.org", true, FilteredBlockList, dns.TypeA}, + {"allowlist", allowlistRules, "test.example.org", false, NotFilteredAllowList, dns.TypeA}, + {"allowlist", allowlistRules, "test.test.example.org", false, NotFilteredAllowList, dns.TypeA}, + {"allowlist", allowlistRules, "testexample.org", false, NotFilteredNotFound, dns.TypeA}, + {"allowlist", allowlistRules, "onemoreexample.org", false, NotFilteredNotFound, dns.TypeA}, - {"important", importantRules, "example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"important", importantRules, "test.example.org", true, FilteredBlackList, dns.TypeA}, - {"important", importantRules, "test.test.example.org", true, FilteredBlackList, dns.TypeA}, + {"important", importantRules, "example.org", false, NotFilteredAllowList, dns.TypeA}, + {"important", importantRules, "test.example.org", true, FilteredBlockList, dns.TypeA}, + {"important", importantRules, "test.test.example.org", true, FilteredBlockList, dns.TypeA}, {"important", importantRules, "testexample.org", false, NotFilteredNotFound, dns.TypeA}, {"important", importantRules, "onemoreexample.org", false, NotFilteredNotFound, dns.TypeA}, - {"regex", regexRules, "example.org", true, FilteredBlackList, dns.TypeA}, - {"regex", regexRules, "test.example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"regex", regexRules, "test.test.example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"regex", regexRules, "testexample.org", true, FilteredBlackList, dns.TypeA}, - {"regex", regexRules, "onemoreexample.org", true, FilteredBlackList, dns.TypeA}, + {"regex", regexRules, "example.org", true, FilteredBlockList, dns.TypeA}, + {"regex", regexRules, "test.example.org", false, NotFilteredAllowList, dns.TypeA}, + {"regex", regexRules, "test.test.example.org", false, NotFilteredAllowList, dns.TypeA}, + {"regex", regexRules, "testexample.org", true, FilteredBlockList, dns.TypeA}, + {"regex", regexRules, "onemoreexample.org", true, FilteredBlockList, dns.TypeA}, - {"mask", maskRules, "test.example.org", true, FilteredBlackList, dns.TypeA}, - {"mask", maskRules, "test2.example.org", true, FilteredBlackList, dns.TypeA}, - {"mask", maskRules, "example.com", true, FilteredBlackList, dns.TypeA}, - {"mask", maskRules, "exampleeee.com", true, FilteredBlackList, dns.TypeA}, - {"mask", maskRules, "onemoreexamsite.com", true, FilteredBlackList, dns.TypeA}, + {"mask", maskRules, "test.example.org", true, FilteredBlockList, dns.TypeA}, + {"mask", maskRules, "test2.example.org", true, FilteredBlockList, dns.TypeA}, + {"mask", maskRules, "example.com", true, FilteredBlockList, dns.TypeA}, + {"mask", maskRules, "exampleeee.com", true, FilteredBlockList, dns.TypeA}, + {"mask", maskRules, "onemoreexamsite.com", true, FilteredBlockList, dns.TypeA}, {"mask", maskRules, "example.org", false, NotFilteredNotFound, dns.TypeA}, {"mask", maskRules, "testexample.org", false, NotFilteredNotFound, dns.TypeA}, {"mask", maskRules, "example.co.uk", false, NotFilteredNotFound, dns.TypeA}, {"dnstype", dnstypeRules, "onemoreexample.org", false, NotFilteredNotFound, dns.TypeA}, {"dnstype", dnstypeRules, "example.org", false, NotFilteredNotFound, dns.TypeA}, - {"dnstype", dnstypeRules, "example.org", true, FilteredBlackList, dns.TypeAAAA}, - {"dnstype", dnstypeRules, "test.example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"dnstype", dnstypeRules, "test.example.org", false, NotFilteredWhiteList, dns.TypeAAAA}, + {"dnstype", dnstypeRules, "example.org", true, FilteredBlockList, dns.TypeAAAA}, + {"dnstype", dnstypeRules, "test.example.org", false, NotFilteredAllowList, dns.TypeA}, + {"dnstype", dnstypeRules, "test.example.org", false, NotFilteredAllowList, dns.TypeAAAA}, } func TestMatching(t *testing.T) { @@ -470,7 +469,7 @@ func TestWhitelist(t *testing.T) { // matched by white filter res, err := d.CheckHost("host1", dns.TypeA, &setts) assert.True(t, err == nil) - assert.True(t, !res.IsFiltered && res.Reason == NotFilteredWhiteList) + assert.True(t, !res.IsFiltered && res.Reason == NotFilteredAllowList) if assert.Len(t, res.Rules, 1) { assert.True(t, res.Rules[0].Text == "||host1^") } @@ -478,7 +477,7 @@ func TestWhitelist(t *testing.T) { // not matched by white filter, but matched by block filter res, err = d.CheckHost("host2", dns.TypeA, &setts) assert.True(t, err == nil) - assert.True(t, res.IsFiltered && res.Reason == FilteredBlackList) + assert.True(t, res.IsFiltered && res.Reason == FilteredBlockList) if assert.Len(t, res.Rules, 1) { assert.True(t, res.Rules[0].Text == "||host2^") } @@ -512,8 +511,8 @@ func TestClientSettings(t *testing.T) { // blocked by filters r, _ = d.CheckHost("example.org", dns.TypeA, &setts) - if !r.IsFiltered || r.Reason != FilteredBlackList { - t.Fatalf("CheckHost FilteredBlackList") + if !r.IsFiltered || r.Reason != FilteredBlockList { + t.Fatalf("CheckHost FilteredBlockList") } // blocked by parental diff --git a/internal/dnsfilter/dnsrewrite.go b/internal/dnsfilter/dnsrewrite.go new file mode 100644 index 00000000..1239fbad --- /dev/null +++ b/internal/dnsfilter/dnsrewrite.go @@ -0,0 +1,80 @@ +package dnsfilter + +import ( + "github.com/AdguardTeam/urlfilter/rules" + "github.com/miekg/dns" +) + +// DNSRewriteResult is the result of application of $dnsrewrite rules. +type DNSRewriteResult struct { + RCode rules.RCode `json:",omitempty"` + Response DNSRewriteResultResponse `json:",omitempty"` +} + +// DNSRewriteResultResponse is the collection of DNS response records +// the server returns. +type DNSRewriteResultResponse map[rules.RRType][]rules.RRValue + +// processDNSRewrites processes DNS rewrite rules in dnsr. It returns +// an empty result if dnsr is empty. Otherwise, the result will have +// either CanonName or DNSRewriteResult set. +func (d *DNSFilter) processDNSRewrites(dnsr []*rules.NetworkRule) (res Result) { + if len(dnsr) == 0 { + return Result{} + } + + var rules []*ResultRule + dnsrr := &DNSRewriteResult{ + Response: DNSRewriteResultResponse{}, + } + + for _, nr := range dnsr { + dr := nr.DNSRewrite + if dr.NewCNAME != "" { + // NewCNAME rules have a higher priority than + // the other rules. + rules := []*ResultRule{{ + FilterListID: int64(nr.GetFilterListID()), + Text: nr.RuleText, + }} + + return Result{ + Reason: DNSRewriteRule, + Rules: rules, + CanonName: dr.NewCNAME, + } + } + + switch dr.RCode { + case dns.RcodeSuccess: + dnsrr.RCode = dr.RCode + dnsrr.Response[dr.RRType] = append(dnsrr.Response[dr.RRType], dr.Value) + rules = append(rules, &ResultRule{ + FilterListID: int64(nr.GetFilterListID()), + Text: nr.RuleText, + }) + default: + // RcodeRefused and other such codes have higher + // priority. Return immediately. + rules := []*ResultRule{{ + FilterListID: int64(nr.GetFilterListID()), + Text: nr.RuleText, + }} + dnsrr = &DNSRewriteResult{ + RCode: dr.RCode, + } + + return Result{ + Reason: DNSRewriteRule, + Rules: rules, + DNSRewriteResult: dnsrr, + } + } + } + + return Result{ + Reason: DNSRewriteRule, + Rules: rules, + DNSRewriteResult: dnsrr, + } +} diff --git a/internal/dnsfilter/dnsrewrite_test.go b/internal/dnsfilter/dnsrewrite_test.go new file mode 100644 index 00000000..4918ccc0 --- /dev/null +++ b/internal/dnsfilter/dnsrewrite_test.go @@ -0,0 +1,202 @@ +package dnsfilter + +import ( + "net" + "path" + "testing" + + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" +) + +func TestDNSFilter_CheckHostRules_dnsrewrite(t *testing.T) { + const text = ` +|cname^$dnsrewrite=new_cname + +|a_record^$dnsrewrite=127.0.0.1 + +|aaaa_record^$dnsrewrite=::1 + +|txt_record^$dnsrewrite=NOERROR;TXT;hello_world + +|refused^$dnsrewrite=REFUSED + +|a_records^$dnsrewrite=127.0.0.1 +|a_records^$dnsrewrite=127.0.0.2 + +|aaaa_records^$dnsrewrite=::1 +|aaaa_records^$dnsrewrite=::2 + +|disable_one^$dnsrewrite=127.0.0.1 +|disable_one^$dnsrewrite=127.0.0.2 +@@||disable_one^$dnsrewrite=127.0.0.1 + +|disable_cname^$dnsrewrite=127.0.0.1 +|disable_cname^$dnsrewrite=new_cname +@@||disable_cname^$dnsrewrite=new_cname + +|disable_cname_many^$dnsrewrite=127.0.0.1 +|disable_cname_many^$dnsrewrite=new_cname_1 +|disable_cname_many^$dnsrewrite=new_cname_2 +@@||disable_cname_many^$dnsrewrite=new_cname_1 + +|disable_all^$dnsrewrite=127.0.0.1 +|disable_all^$dnsrewrite=127.0.0.2 +@@||disable_all^$dnsrewrite +` + f := NewForTest(nil, []Filter{{ID: 0, Data: []byte(text)}}) + setts := &RequestFilteringSettings{ + FilteringEnabled: true, + } + + ipv4p1 := net.IPv4(127, 0, 0, 1) + ipv4p2 := net.IPv4(127, 0, 0, 2) + ipv6p1 := net.IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} + ipv6p2 := net.IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2} + + t.Run("cname", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + assert.Equal(t, "new_cname", res.CanonName) + }) + + t.Run("a_record", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 1) { + assert.Equal(t, ipv4p1, ipVals[0]) + } + } + }) + + t.Run("aaaa_record", func(t *testing.T) { + dtyp := dns.TypeAAAA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 1) { + assert.Equal(t, ipv6p1, ipVals[0]) + } + } + }) + + t.Run("txt_record", func(t *testing.T) { + dtyp := dns.TypeTXT + host := path.Base(t.Name()) + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if strVals := dnsrr.Response[dtyp]; assert.Len(t, strVals, 1) { + assert.Equal(t, "hello_world", strVals[0]) + } + } + }) + + t.Run("refused", func(t *testing.T) { + host := path.Base(t.Name()) + res, err := f.CheckHostRules(host, dns.TypeA, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeRefused, dnsrr.RCode) + } + }) + + t.Run("a_records", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 2) { + assert.Equal(t, ipv4p1, ipVals[0]) + assert.Equal(t, ipv4p2, ipVals[1]) + } + } + }) + + t.Run("aaaa_records", func(t *testing.T) { + dtyp := dns.TypeAAAA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 2) { + assert.Equal(t, ipv6p1, ipVals[0]) + assert.Equal(t, ipv6p2, ipVals[1]) + } + } + }) + + t.Run("disable_one", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 1) { + assert.Equal(t, ipv4p2, ipVals[0]) + } + } + }) + + t.Run("disable_cname", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + assert.Equal(t, "", res.CanonName) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 1) { + assert.Equal(t, ipv4p1, ipVals[0]) + } + } + }) + + t.Run("disable_cname_many", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + assert.Equal(t, "new_cname_2", res.CanonName) + assert.Nil(t, res.DNSRewriteResult) + }) + + t.Run("disable_all", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + assert.Equal(t, "", res.CanonName) + assert.Len(t, res.Rules, 0) + }) +} diff --git a/internal/dnsforward/dns.go b/internal/dnsforward/dns.go index 0f9c764b..d7208d1c 100644 --- a/internal/dnsforward/dns.go +++ b/internal/dnsforward/dns.go @@ -366,7 +366,9 @@ func processFilteringAfterResponse(ctx *dnsContext) int { var err error switch res.Reason { - case dnsfilter.ReasonRewrite: + case dnsfilter.ReasonRewrite, + dnsfilter.DNSRewriteRule: + if len(ctx.origQuestion.Name) == 0 { // origQuestion is set in case we get only CNAME without IP from rewrites table break @@ -378,11 +380,11 @@ func processFilteringAfterResponse(ctx *dnsContext) int { if len(d.Res.Answer) != 0 { answer := []dns.RR{} answer = append(answer, s.genCNAMEAnswer(d.Req, res.CanonName)) - answer = append(answer, d.Res.Answer...) // host -> IP + answer = append(answer, d.Res.Answer...) d.Res.Answer = answer } - case dnsfilter.NotFilteredWhiteList: + case dnsfilter.NotFilteredAllowList: // nothing default: diff --git a/internal/dnsforward/dnsrewrite.go b/internal/dnsforward/dnsrewrite.go new file mode 100644 index 00000000..01895323 --- /dev/null +++ b/internal/dnsforward/dnsrewrite.go @@ -0,0 +1,79 @@ +package dnsforward + +import ( + "fmt" + "net" + + "github.com/AdguardTeam/AdGuardHome/internal/agherr" + "github.com/AdguardTeam/AdGuardHome/internal/dnsfilter" + "github.com/AdguardTeam/dnsproxy/proxy" + "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/urlfilter/rules" + "github.com/miekg/dns" +) + +// filterDNSRewriteResponse handles a single DNS rewrite response entry. +// It returns the constructed answer resource record. +func (s *Server) filterDNSRewriteResponse(req *dns.Msg, rr rules.RRType, v rules.RRValue) (ans dns.RR, err error) { + switch rr { + case dns.TypeA, dns.TypeAAAA: + ip, ok := v.(net.IP) + if !ok { + return nil, fmt.Errorf("value has type %T, not net.IP", v) + } + + if rr == dns.TypeA { + return s.genAAnswer(req, ip.To4()), nil + } + + return s.genAAAAAnswer(req, ip), nil + case dns.TypeTXT: + str, ok := v.(string) + if !ok { + return nil, fmt.Errorf("value has type %T, not string", v) + } + + return s.genTXTAnswer(req, []string{str}), nil + default: + log.Debug("don't know how to handle dns rr type %d, skipping", rr) + + return nil, nil + } +} + +// filterDNSRewrite handles dnsrewrite filters. It constructs a DNS +// response and sets it into d.Res. +func (s *Server) filterDNSRewrite(req *dns.Msg, res dnsfilter.Result, d *proxy.DNSContext) (err error) { + resp := s.makeResponse(req) + dnsrr := res.DNSRewriteResult + if dnsrr == nil { + return agherr.Error("no dns rewrite rule content") + } + + resp.Rcode = dnsrr.RCode + if resp.Rcode != dns.RcodeSuccess { + d.Res = resp + + return nil + } + + if dnsrr.Response == nil { + return agherr.Error("no dns rewrite rule responses") + } + + rr := req.Question[0].Qtype + values := dnsrr.Response[rr] + for i, v := range values { + var ans dns.RR + ans, err = s.filterDNSRewriteResponse(req, rr, v) + if err != nil { + return fmt.Errorf("dns rewrite response for %d[%d]: %w", rr, i, err) + } + + resp.Answer = append(resp.Answer, ans) + } + + d.Res = resp + + return nil +} diff --git a/internal/dnsforward/filter.go b/internal/dnsforward/filter.go index 83effc60..5cd0090a 100644 --- a/internal/dnsforward/filter.go +++ b/internal/dnsforward/filter.go @@ -42,7 +42,8 @@ func (s *Server) getClientRequestFilteringSettings(d *proxy.DNSContext) *dnsfilt return &setts } -// filterDNSRequest applies the dnsFilter and sets d.Res if the request was filtered +// filterDNSRequest applies the dnsFilter and sets d.Res if the request +// was filtered. func (s *Server) filterDNSRequest(ctx *dnsContext) (*dnsfilter.Result, error) { d := ctx.proxyCtx req := d.Req @@ -54,9 +55,13 @@ func (s *Server) filterDNSRequest(ctx *dnsContext) (*dnsfilter.Result, error) { } else if res.IsFiltered { log.Tracef("Host %s is filtered, reason - %q, matched rule: %q", host, res.Reason, res.Rules[0].Text) d.Res = s.genDNSFilterMessage(d, &res) - } else if res.Reason == dnsfilter.ReasonRewrite && len(res.CanonName) != 0 && len(res.IPList) == 0 { + } else if res.Reason.In(dnsfilter.ReasonRewrite, dnsfilter.DNSRewriteRule) && + res.CanonName != "" && + len(res.IPList) == 0 { + // Resolve the new canonical name, not the original host + // name. The original question is readded in + // processFilteringAfterResponse. ctx.origQuestion = d.Req.Question[0] - // resolve canonical name, not the original host name d.Req.Question[0].Name = dns.Fqdn(res.CanonName) } else if res.Reason == dnsfilter.RewriteAutoHosts && len(res.ReverseHosts) != 0 { resp := s.makeResponse(req) @@ -99,6 +104,11 @@ func (s *Server) filterDNSRequest(ctx *dnsContext) (*dnsfilter.Result, error) { } d.Res = resp + } else if res.Reason == dnsfilter.DNSRewriteRule { + err = s.filterDNSRewrite(req, res, d) + if err != nil { + return nil, err + } } return &res, err diff --git a/internal/dnsforward/msg.go b/internal/dnsforward/msg.go index 2e72c40c..f8200056 100644 --- a/internal/dnsforward/msg.go +++ b/internal/dnsforward/msg.go @@ -11,12 +11,17 @@ import ( ) // Create a DNS response by DNS request and set necessary flags -func (s *Server) makeResponse(req *dns.Msg) *dns.Msg { - resp := dns.Msg{} +func (s *Server) makeResponse(req *dns.Msg) (resp *dns.Msg) { + resp = &dns.Msg{ + MsgHdr: dns.MsgHdr{ + RecursionAvailable: true, + }, + Compress: true, + } + resp.SetReply(req) - resp.RecursionAvailable = true - resp.Compress = true - return &resp + + return resp } // genDNSFilterMessage generates a DNS message corresponding to the filtering result @@ -121,6 +126,18 @@ func (s *Server) genAAAAAnswer(req *dns.Msg, ip net.IP) *dns.AAAA { return answer } +func (s *Server) genTXTAnswer(req *dns.Msg, strs []string) (answer *dns.TXT) { + return &dns.TXT{ + Hdr: dns.RR_Header{ + Name: req.Question[0].Name, + Rrtype: dns.TypeTXT, + Ttl: s.conf.BlockedResponseTTL, + Class: dns.ClassINET, + }, + Txt: strs, + } +} + // generate DNS response message with an IP address func (s *Server) genResponseWithIP(req *dns.Msg, ip net.IP) *dns.Msg { if req.Question[0].Qtype == dns.TypeA && ip.To4() != nil { diff --git a/internal/dnsforward/stats.go b/internal/dnsforward/stats.go index c2b8921f..c447be05 100644 --- a/internal/dnsforward/stats.go +++ b/internal/dnsforward/stats.go @@ -91,7 +91,7 @@ func (s *Server) updateStats(d *proxy.DNSContext, elapsed time.Duration, res dns case dnsfilter.FilteredSafeSearch: e.Result = stats.RSafeSearch - case dnsfilter.FilteredBlackList: + case dnsfilter.FilteredBlockList: fallthrough case dnsfilter.FilteredInvalid: fallthrough diff --git a/internal/home/controlfiltering.go b/internal/home/controlfiltering.go index 3fe07e7e..1d0172e8 100644 --- a/internal/home/controlfiltering.go +++ b/internal/home/controlfiltering.go @@ -359,6 +359,9 @@ type checkHostResp struct { // Deprecated: Use Rules[*].FilterListID. FilterID int64 `json:"filter_id"` + // Rule is the text of the matched rule. + // + // Deprecated: Use Rules[*].Text. Rule string `json:"rule"` Rules []*checkHostRespRule `json:"rules"` @@ -386,12 +389,15 @@ func (f *Filtering) handleCheckHost(w http.ResponseWriter, r *http.Request) { resp := checkHostResp{} resp.Reason = result.Reason.String() - resp.FilterID = result.Rules[0].FilterListID - resp.Rule = result.Rules[0].Text resp.SvcName = result.ServiceName resp.CanonName = result.CanonName resp.IPList = result.IPList + if len(result.Rules) > 0 { + resp.FilterID = result.Rules[0].FilterListID + resp.Rule = result.Rules[0].Text + } + resp.Rules = make([]*checkHostRespRule, len(result.Rules)) for i, r := range result.Rules { resp.Rules[i] = &checkHostRespRule{ diff --git a/internal/querylog/decode.go b/internal/querylog/decode.go index f5937781..ed721489 100644 --- a/internal/querylog/decode.go +++ b/internal/querylog/decode.go @@ -4,11 +4,14 @@ import ( "encoding/base64" "encoding/json" "io" + "net" "strings" "time" "github.com/AdguardTeam/AdGuardHome/internal/dnsfilter" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/urlfilter/rules" + "github.com/miekg/dns" ) type logEntryHandler (func(t json.Token, ent *logEntry) error) @@ -165,13 +168,285 @@ var resultHandlers = map[string]logEntryHandler{ return nil }, "ServiceName": func(t json.Token, ent *logEntry) error { - v, ok := t.(string) + s, ok := t.(string) if !ok { return nil } - ent.Result.ServiceName = v + + ent.Result.ServiceName = s + return nil }, + "CanonName": func(t json.Token, ent *logEntry) error { + s, ok := t.(string) + if !ok { + return nil + } + + ent.Result.CanonName = s + + return nil + }, +} + +func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) { + switch key { + case "FilterListID": + vToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultRuleKey %s err: %s", key, err) + } + + return + } + + if len(ent.Result.Rules) < i+1 { + ent.Result.Rules = append(ent.Result.Rules, &dnsfilter.ResultRule{}) + } + + if n, ok := vToken.(json.Number); ok { + ent.Result.Rules[i].FilterListID, _ = n.Int64() + } + case "IP": + vToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultRuleKey %s err: %s", key, err) + } + + return + } + + if len(ent.Result.Rules) < i+1 { + ent.Result.Rules = append(ent.Result.Rules, &dnsfilter.ResultRule{}) + } + + if ipStr, ok := vToken.(string); ok { + ent.Result.Rules[i].IP = net.ParseIP(ipStr) + } + case "Text": + vToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultRuleKey %s err: %s", key, err) + } + + return + } + + if len(ent.Result.Rules) < i+1 { + ent.Result.Rules = append(ent.Result.Rules, &dnsfilter.ResultRule{}) + } + + if s, ok := vToken.(string); ok { + ent.Result.Rules[i].Text = s + } + default: + // Go on. + } +} + +func decodeResultRules(dec *json.Decoder, ent *logEntry) { + for { + delimToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultRules err: %s", err) + } + + return + } + + if d, ok := delimToken.(json.Delim); ok { + if d != '[' { + log.Debug("decodeResultRules: unexpected delim %q", d) + } + } else { + return + } + + i := 0 + for { + keyToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultRules err: %s", err) + } + + return + } + + if d, ok := keyToken.(json.Delim); ok { + if d == '}' { + i++ + } else if d == ']' { + return + } + + continue + } + + key, ok := keyToken.(string) + if !ok { + log.Debug("decodeResultRules: keyToken is %T (%[1]v) and not string", keyToken) + + return + } + + decodeResultRuleKey(key, i, dec, ent) + } + } +} + +func decodeResultReverseHosts(dec *json.Decoder, ent *logEntry) { + for { + itemToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultReverseHosts err: %s", err) + } + + return + } + + switch v := itemToken.(type) { + case json.Delim: + if v == '[' { + continue + } else if v == ']' { + return + } + + log.Debug("decodeResultReverseHosts: unexpected delim %q", v) + + return + case string: + ent.Result.ReverseHosts = append(ent.Result.ReverseHosts, v) + default: + continue + } + } +} + +func decodeResultIPList(dec *json.Decoder, ent *logEntry) { + for { + itemToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultIPList err: %s", err) + } + + return + } + + switch v := itemToken.(type) { + case json.Delim: + if v == '[' { + continue + } else if v == ']' { + return + } + + log.Debug("decodeResultIPList: unexpected delim %q", v) + + return + case string: + ip := net.ParseIP(v) + if ip != nil { + ent.Result.IPList = append(ent.Result.IPList, ip) + } + default: + continue + } + } +} + +func decodeResultDNSRewriteResult(dec *json.Decoder, ent *logEntry) { + for { + keyToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultDNSRewriteResult err: %s", err) + } + + return + } + + if d, ok := keyToken.(json.Delim); ok { + if d == '}' { + return + } + + continue + } + + key, ok := keyToken.(string) + if !ok { + log.Debug("decodeResultDNSRewriteResult: keyToken is %T (%[1]v) and not string", keyToken) + + return + } + + // TODO(a.garipov): Refactor this into a separate + // function à la decodeResultRuleKey if we keep this + // code for a longer time than planned. + switch key { + case "RCode": + vToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultDNSRewriteResult err: %s", err) + } + + return + } + + if ent.Result.DNSRewriteResult == nil { + ent.Result.DNSRewriteResult = &dnsfilter.DNSRewriteResult{} + } + + if n, ok := vToken.(json.Number); ok { + rcode64, _ := n.Int64() + ent.Result.DNSRewriteResult.RCode = rules.RCode(rcode64) + } + + continue + case "Response": + if ent.Result.DNSRewriteResult == nil { + ent.Result.DNSRewriteResult = &dnsfilter.DNSRewriteResult{} + } + + if ent.Result.DNSRewriteResult.Response == nil { + ent.Result.DNSRewriteResult.Response = dnsfilter.DNSRewriteResultResponse{} + } + + // TODO(a.garipov): I give up. This whole file + // is a mess. Luckily, we can assume that this + // field is relatively rare and just use the + // normal decoding and correct the values. + err = dec.Decode(&ent.Result.DNSRewriteResult.Response) + if err != nil { + log.Debug("decodeResultDNSRewriteResult response err: %s", err) + } + + for rrType, rrValues := range ent.Result.DNSRewriteResult.Response { + switch rrType { + case dns.TypeA, dns.TypeAAAA: + for i, v := range rrValues { + s, _ := v.(string) + rrValues[i] = net.ParseIP(s) + } + default: + // Go on. + } + } + + continue + default: + // Go on. + } + } } func decodeResult(dec *json.Decoder, ent *logEntry) { @@ -200,6 +475,27 @@ func decodeResult(dec *json.Decoder, ent *logEntry) { return } + switch key { + case "ReverseHosts": + decodeResultReverseHosts(dec, ent) + + continue + case "IPList": + decodeResultIPList(dec, ent) + + continue + case "Rules": + decodeResultRules(dec, ent) + + continue + case "DNSRewriteResult": + decodeResultDNSRewriteResult(dec, ent) + + continue + default: + // Go on. + } + handler, ok := resultHandlers[key] if !ok { continue diff --git a/internal/querylog/decode_test.go b/internal/querylog/decode_test.go index d9cbd600..ffcf94dc 100644 --- a/internal/querylog/decode_test.go +++ b/internal/querylog/decode_test.go @@ -2,95 +2,181 @@ package querylog import ( "bytes" + "encoding/base64" + "net" "strings" "testing" + "time" + "github.com/AdguardTeam/AdGuardHome/internal/dnsfilter" "github.com/AdguardTeam/AdGuardHome/internal/testutil" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/urlfilter/rules" + "github.com/miekg/dns" "github.com/stretchr/testify/assert" ) -func TestDecode_decodeQueryLog(t *testing.T) { +func TestDecodeLogEntry(t *testing.T) { logOutput := &bytes.Buffer{} testutil.ReplaceLogWriter(t, logOutput) testutil.ReplaceLogLevel(t, log.DEBUG) + t.Run("success", func(t *testing.T) { + const ansStr = `Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==` + const data = `{"IP":"127.0.0.1",` + + `"T":"2020-11-25T18:55:56.519796+03:00",` + + `"QH":"an.yandex.ru",` + + `"QT":"A",` + + `"QC":"IN",` + + `"CP":"",` + + `"Answer":"` + ansStr + `",` + + `"Result":{` + + `"IsFiltered":true,` + + `"Reason":3,` + + `"ReverseHosts":["example.net"],` + + `"IPList":["127.0.0.2"],` + + `"Rules":[{"FilterListID":42,"Text":"||an.yandex.ru","IP":"127.0.0.2"},` + + `{"FilterListID":43,"Text":"||an2.yandex.ru","IP":"127.0.0.3"}],` + + `"CanonName":"example.com",` + + `"ServiceName":"example.org",` + + `"DNSRewriteResult":{"RCode":0,"Response":{"1":["127.0.0.2"]}}},` + + `"Elapsed":837429}` + + ans, err := base64.StdEncoding.DecodeString(ansStr) + assert.Nil(t, err) + + want := &logEntry{ + IP: "127.0.0.1", + Time: time.Date(2020, 11, 25, 15, 55, 56, 519796000, time.UTC), + QHost: "an.yandex.ru", + QType: "A", + QClass: "IN", + ClientProto: "", + Answer: ans, + Result: dnsfilter.Result{ + IsFiltered: true, + Reason: dnsfilter.FilteredBlockList, + ReverseHosts: []string{"example.net"}, + IPList: []net.IP{net.IPv4(127, 0, 0, 2)}, + Rules: []*dnsfilter.ResultRule{{ + FilterListID: 42, + Text: "||an.yandex.ru", + IP: net.IPv4(127, 0, 0, 2), + }, { + FilterListID: 43, + Text: "||an2.yandex.ru", + IP: net.IPv4(127, 0, 0, 3), + }}, + CanonName: "example.com", + ServiceName: "example.org", + DNSRewriteResult: &dnsfilter.DNSRewriteResult{ + RCode: dns.RcodeSuccess, + Response: dnsfilter.DNSRewriteResultResponse{ + dns.TypeA: []rules.RRValue{net.IPv4(127, 0, 0, 2)}, + }, + }, + }, + Elapsed: 837429, + } + + got := &logEntry{} + decodeLogEntry(got, data) + + s := logOutput.String() + assert.Equal(t, "", s) + + // Correct for time zones. + got.Time = got.Time.UTC() + assert.Equal(t, want, got) + }) + testCases := []struct { name string log string want string }{{ - name: "all_right", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + name: "all_right_old_rule", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1,"ReverseHosts":["example.com"],"IPList":["127.0.0.1"]},"Elapsed":837429}`, + want: "", }, { - name: "bad_filter_id", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1.5},"Elapsed":837429}`, + name: "bad_filter_id_old_rule", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"FilterID":1.5},"Elapsed":837429}`, want: "decodeResult handler err: strconv.ParseInt: parsing \"1.5\": invalid syntax\n", }, { name: "bad_is_filtered", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":trooe,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":trooe,"Reason":3},"Elapsed":837429}`, want: "decodeLogEntry err: invalid character 'o' in literal true (expecting 'u')\n", }, { name: "bad_elapsed", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":-1}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":-1}`, + want: "", }, { name: "bad_ip", - log: `{"IP":127001,"T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":127001,"T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "bad_time", - log: `{"IP":"127.0.0.1","T":"12/09/1998T15:00:00.000000+05:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, + log: `{"IP":"127.0.0.1","T":"12/09/1998T15:00:00.000000+05:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, want: "decodeLogEntry handler err: parsing time \"12/09/1998T15:00:00.000000+05:00\" as \"2006-01-02T15:04:05Z07:00\": cannot parse \"9/1998T15:00:00.000000+05:00\" as \"2006\"\n", }, { name: "bad_host", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":6,"QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":6,"QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "bad_type", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":true,"QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":true,"QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "bad_class", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":false,"CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":false,"CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "bad_client_proto", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":8,"Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":8,"Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "very_bad_client_proto", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"dog","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"dog","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, want: "decodeLogEntry handler err: invalid client proto: \"dog\"\n", }, { name: "bad_answer", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":0.9,"Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":0.9,"Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "very_bad_answer", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, want: "decodeLogEntry handler err: illegal base64 data at input byte 61\n", }, { name: "bad_rule", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":false,"FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":false},"Elapsed":837429}`, + want: "", }, { name: "bad_reason", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":true,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":true},"Elapsed":837429}`, + want: "", + }, { + name: "bad_reverse_hosts", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"ReverseHosts":[{}]},"Elapsed":837429}`, + want: "decodeResultReverseHosts: unexpected delim \"{\"\n", + }, { + name: "bad_ip_list", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"ReverseHosts":["example.net"],"IPList":[{}]},"Elapsed":837429}`, + want: "decodeResultIPList: unexpected delim \"{\"\n", }} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, err := logOutput.Write([]byte("default")) - assert.Nil(t, err) - l := &logEntry{} decodeLogEntry(l, tc.log) - assert.True(t, strings.HasSuffix(logOutput.String(), tc.want), "%q\ndoes not end with\n%q", logOutput.String(), tc.want) + s := logOutput.String() + if tc.want == "" { + assert.Equal(t, "", s) + } else { + assert.True(t, strings.HasSuffix(s, tc.want), + "got %q", s) + } logOutput.Reset() }) diff --git a/internal/querylog/searchcriteria.go b/internal/querylog/searchcriteria.go index 52b76459..b98e0838 100644 --- a/internal/querylog/searchcriteria.go +++ b/internal/querylog/searchcriteria.go @@ -115,14 +115,14 @@ func (c *searchCriteria) ctFilteringStatusCase(res dnsfilter.Result) bool { case filteringStatusFiltered: return res.IsFiltered || res.Reason.In( - dnsfilter.NotFilteredWhiteList, + dnsfilter.NotFilteredAllowList, dnsfilter.ReasonRewrite, dnsfilter.RewriteAutoHosts, ) case filteringStatusBlocked: return res.IsFiltered && - res.Reason.In(dnsfilter.FilteredBlackList, dnsfilter.FilteredBlockedService) + res.Reason.In(dnsfilter.FilteredBlockList, dnsfilter.FilteredBlockedService) case filteringStatusBlockedService: return res.IsFiltered && res.Reason == dnsfilter.FilteredBlockedService @@ -134,7 +134,7 @@ func (c *searchCriteria) ctFilteringStatusCase(res dnsfilter.Result) bool { return res.IsFiltered && res.Reason == dnsfilter.FilteredSafeBrowsing case filteringStatusWhitelisted: - return res.Reason == dnsfilter.NotFilteredWhiteList + return res.Reason == dnsfilter.NotFilteredAllowList case filteringStatusRewritten: return res.Reason.In(dnsfilter.ReasonRewrite, dnsfilter.RewriteAutoHosts) @@ -144,9 +144,9 @@ func (c *searchCriteria) ctFilteringStatusCase(res dnsfilter.Result) bool { case filteringStatusProcessed: return !res.Reason.In( - dnsfilter.FilteredBlackList, + dnsfilter.FilteredBlockList, dnsfilter.FilteredBlockedService, - dnsfilter.NotFilteredWhiteList, + dnsfilter.NotFilteredAllowList, ) default: diff --git a/openapi/CHANGELOG.md b/openapi/CHANGELOG.md index f40dd490..076d9896 100644 --- a/openapi/CHANGELOG.md +++ b/openapi/CHANGELOG.md @@ -4,15 +4,21 @@ ## v0.105: API changes +### New `"reason"` in `GET /filtering/check_host` and `GET /querylog` + +* The new `DNSRewriteRule` reason is added to `GET /filtering/check_host` and + `GET /querylog`. + +* Also, the reason which was incorrectly documented as `"ReasonRewrite"` is now + correctly documented as `"Rewrite"`, and the previously undocumented + `"RewriteEtcHosts"` is now documented as well. + ### Multiple matched rules in `GET /filtering/check_host` and `GET /querylog` - - * The properties `rule` and `filter_id` are now deprecated. API users should - inspect the newly-added `rules` object array instead. Currently, it's either - empty or contains one object, which contains the same things as the old two - properties did, but under more correct names: + inspect the newly-added `rules` object array instead. For most rules, it's + either empty or contains one object, which contains the same things as the old + two properties did, but under more correct names: ```js { @@ -30,6 +36,30 @@ checked in. --> } ``` + For `$dnsrewrite` rules, they contain all rules that contributed to the + result. For example, if you have the following filtering rules: + + ``` + ||example.com^$dnsrewrite=127.0.0.1 + ||example.com^$dnsrewrite=127.0.0.2 + ``` + + The `"rules"` will be something like: + + ```js + { + // … + + "rules": [{ + "text": "||example.com^$dnsrewrite=127.0.0.1", + "filter_list_id": 0 + }, { + "text": "||example.com^$dnsrewrite=127.0.0.2", + "filter_list_id": 0 + }] + } + ``` + The old fields will be removed in v0.106.0. ## v0.103: API changes diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 17e9f3f5..61db836e 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -523,7 +523,7 @@ Reload filtering rules from URLs. This might be needed if new URL was just added and you dont want to wait for automatic refresh to kick in. This API request is ratelimited, so you can call it freely as often as - you like, it wont create unneccessary burden on servers that host the + you like, it wont create unnecessary burden on servers that host the URL. This should work as intended, a `force` parameter is offered as last-resort attempt to make filter lists fresh. If you ever find yourself using `force` to make something work that otherwise wont, this @@ -1246,7 +1246,7 @@ 'properties': 'reason': 'type': 'string' - 'description': 'DNS filter status' + 'description': 'Request filtering status.' 'enum': - 'NotFilteredNotFound' - 'NotFilteredWhiteList' @@ -1257,7 +1257,9 @@ - 'FilteredInvalid' - 'FilteredSafeSearch' - 'FilteredBlockedService' - - 'ReasonRewrite' + - 'Rewrite' + - 'RewriteEtcHosts' + - 'DNSRewriteRule' 'filter_id': 'deprecated': true 'description': > @@ -1284,12 +1286,12 @@ 'description': 'Set if reason=FilteredBlockedService' 'cname': 'type': 'string' - 'description': 'Set if reason=ReasonRewrite' + 'description': 'Set if reason=Rewrite' 'ip_addrs': 'type': 'array' 'items': 'type': 'string' - 'description': 'Set if reason=ReasonRewrite' + 'description': 'Set if reason=Rewrite' 'FilterRefreshResponse': 'type': 'object' 'description': '/filtering/refresh response data' @@ -1648,7 +1650,7 @@ '$ref': '#/components/schemas/ResultRule' 'reason': 'type': 'string' - 'description': 'DNS filter status' + 'description': 'Request filtering status.' 'enum': - 'NotFilteredNotFound' - 'NotFilteredWhiteList' @@ -1659,7 +1661,9 @@ - 'FilteredInvalid' - 'FilteredSafeSearch' - 'FilteredBlockedService' - - 'ReasonRewrite' + - 'Rewrite' + - 'RewriteEtcHosts' + - 'DNSRewriteRule' 'service_name': 'type': 'string' 'description': 'Set if reason=FilteredBlockedService' diff --git a/scripts/go-lint.sh b/scripts/go-lint.sh index c133379b..ca30e1e7 100644 --- a/scripts/go-lint.sh +++ b/scripts/go-lint.sh @@ -95,7 +95,7 @@ ineffassign . unparam ./... -misspell --error ./... +git ls-files -- '*.go' '*.md' '*.yaml' '*.yml' | xargs misspell --error looppointer ./... diff --git a/scripts/translations/README.md b/scripts/translations/README.md index 2c8e5636..3a5e336c 100644 --- a/scripts/translations/README.md +++ b/scripts/translations/README.md @@ -1,4 +1,4 @@ -## Twosky intergration script +## Twosky integration script ### Usage