Pull request 1837: AG-21462-imp-safebrowsing-parental
Merge in DNS/adguard-home from AG-21462-imp-safebrowsing-parental to master
Squashed commit of the following:
commit 85016d4f1105e21a407efade0bd45b8362808061
Merge: 0e61edade 620b51e3e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Thu Apr 27 16:36:30 2023 +0300
Merge branch 'master' into AG-21462-imp-safebrowsing-parental
commit 0e61edadeff34f6305e941c1db94575c82f238d9
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Thu Apr 27 14:51:37 2023 +0300
filtering: imp tests
commit 994255514cc0f67dfe33d5a0892432e8924d1e36
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Thu Apr 27 11:13:19 2023 +0300
filtering: fix typo
commit 96d1069573171538333330d6af94ef0f4208a9c4
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Thu Apr 27 11:00:18 2023 +0300
filtering: imp code more
commit c2a5620b04c4a529eea69983f1520cd2bc82ea9b
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Wed Apr 26 19:13:26 2023 +0300
all: add todo
commit e5dcc2e9701f8bccfde6ef8c01a4a2e7eb31599e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Wed Apr 26 14:36:08 2023 +0300
all: imp code more
commit b6e734ccbeda82669023f6578481260b7c1f7161
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Tue Apr 25 15:01:56 2023 +0300
filtering: imp code
commit 530648dadf836c1a4bd9917e0d3b47256fa8ff52
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Mon Apr 24 20:06:36 2023 +0300
all: imp code
commit 49fa6e587052a40bb431fea457701ee860493527
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Mon Apr 24 14:57:19 2023 +0300
all: rm safe browsing ctx
commit bbcb66cb03e18fa875e3c33cf16295892739e507
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Fri Apr 21 17:54:18 2023 +0300
filtering: add cache item
commit cb7c9fffe8c4ff5e7a21ca912c223c799f61385f
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Thu Apr 20 18:43:02 2023 +0300
filtering: fix hashes
commit 153fec46270212af03f3631bfb42c5d680c4e142
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Thu Apr 20 16:15:15 2023 +0300
filtering: add test cases
commit 09372f92bbb1fc082f1b1283594ee589100209c5
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Thu Apr 20 15:38:05 2023 +0300
filtering: imp code
commit 466bc26d524ea6d1c3efb33692a7785d39e491ca
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Wed Apr 19 18:38:40 2023 +0300
filtering: add tests
commit 24365ecf8c60512fdac65833ee603c80864ae018
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date: Wed Apr 19 11:38:57 2023 +0300
filtering: add hashprefix
This commit is contained in:
Stanislav Chzhen
parent
620b51e3ea
commit
381f2f651d
|
|
@ -23,6 +23,7 @@ import (
|
|||
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/filtering/hashprefix"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/filtering/safesearch"
|
||||
"github.com/AdguardTeam/dnsproxy/proxy"
|
||||
"github.com/AdguardTeam/dnsproxy/upstream"
|
||||
|
|
@ -915,13 +916,23 @@ func TestBlockedByHosts(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestBlockedBySafeBrowsing(t *testing.T) {
|
||||
const hostname = "wmconvirus.narod.ru"
|
||||
const (
|
||||
hostname = "wmconvirus.narod.ru"
|
||||
cacheTime = 10 * time.Minute
|
||||
cacheSize = 10000
|
||||
)
|
||||
|
||||
sbChecker := hashprefix.New(&hashprefix.Config{
|
||||
CacheTime: cacheTime,
|
||||
CacheSize: cacheSize,
|
||||
Upstream: aghtest.NewBlockUpstream(hostname, true),
|
||||
})
|
||||
|
||||
sbUps := aghtest.NewBlockUpstream(hostname, true)
|
||||
ans4, _ := (&aghtest.TestResolver{}).HostToIPs(hostname)
|
||||
|
||||
filterConf := &filtering.Config{
|
||||
SafeBrowsingEnabled: true,
|
||||
SafeBrowsingChecker: sbChecker,
|
||||
}
|
||||
forwardConf := ServerConfig{
|
||||
UDPListenAddrs: []*net.UDPAddr{{}},
|
||||
|
|
@ -935,7 +946,6 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
|
|||
},
|
||||
}
|
||||
s := createTestServer(t, filterConf, forwardConf, nil)
|
||||
s.dnsFilter.SetSafeBrowsingUpstream(sbUps)
|
||||
startDeferStop(t, s)
|
||||
addr := s.dnsProxy.Addr(proxy.ProtoUDP)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,8 +18,6 @@ import (
|
|||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/dnsproxy/upstream"
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/mathutil"
|
||||
|
|
@ -75,6 +73,12 @@ type Resolver interface {
|
|||
|
||||
// Config allows you to configure DNS filtering with New() or just change variables directly.
|
||||
type Config struct {
|
||||
// SafeBrowsingChecker is the safe browsing hash-prefix checker.
|
||||
SafeBrowsingChecker Checker `yaml:"-"`
|
||||
|
||||
// ParentControl is the parental control hash-prefix checker.
|
||||
ParentalControlChecker Checker `yaml:"-"`
|
||||
|
||||
// enabled is used to be returned within Settings.
|
||||
//
|
||||
// It is of type uint32 to be accessed by atomic.
|
||||
|
|
@ -158,8 +162,22 @@ type hostChecker struct {
|
|||
name string
|
||||
}
|
||||
|
||||
// Checker is used for safe browsing or parental control hash-prefix filtering.
|
||||
type Checker interface {
|
||||
// Check returns true if request for the host should be blocked.
|
||||
Check(host string) (block bool, err error)
|
||||
}
|
||||
|
||||
// DNSFilter matches hostnames and DNS requests against filtering rules.
|
||||
type DNSFilter struct {
|
||||
safeSearch SafeSearch
|
||||
|
||||
// safeBrowsingChecker is the safe browsing hash-prefix checker.
|
||||
safeBrowsingChecker Checker
|
||||
|
||||
// parentalControl is the parental control hash-prefix checker.
|
||||
parentalControlChecker Checker
|
||||
|
||||
rulesStorage *filterlist.RuleStorage
|
||||
filteringEngine *urlfilter.DNSEngine
|
||||
|
||||
|
|
@ -168,14 +186,6 @@ type DNSFilter struct {
|
|||
|
||||
engineLock sync.RWMutex
|
||||
|
||||
parentalServer string // access via methods
|
||||
safeBrowsingServer string // access via methods
|
||||
parentalUpstream upstream.Upstream
|
||||
safeBrowsingUpstream upstream.Upstream
|
||||
|
||||
safebrowsingCache cache.Cache
|
||||
parentalCache cache.Cache
|
||||
|
||||
Config // for direct access by library users, even a = assignment
|
||||
// confLock protects Config.
|
||||
confLock sync.RWMutex
|
||||
|
|
@ -192,7 +202,6 @@ type DNSFilter struct {
|
|||
// TODO(e.burkov): Don't use regexp for such a simple text processing task.
|
||||
filterTitleRegexp *regexp.Regexp
|
||||
|
||||
safeSearch SafeSearch
|
||||
hostCheckers []hostChecker
|
||||
}
|
||||
|
||||
|
|
@ -940,19 +949,12 @@ func InitModule() {
|
|||
// be non-nil.
|
||||
func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) {
|
||||
d = &DNSFilter{
|
||||
refreshLock: &sync.Mutex{},
|
||||
filterTitleRegexp: regexp.MustCompile(`^! Title: +(.*)$`),
|
||||
refreshLock: &sync.Mutex{},
|
||||
filterTitleRegexp: regexp.MustCompile(`^! Title: +(.*)$`),
|
||||
safeBrowsingChecker: c.SafeBrowsingChecker,
|
||||
parentalControlChecker: c.ParentalControlChecker,
|
||||
}
|
||||
|
||||
d.safebrowsingCache = cache.New(cache.Config{
|
||||
EnableLRU: true,
|
||||
MaxSize: c.SafeBrowsingCacheSize,
|
||||
})
|
||||
d.parentalCache = cache.New(cache.Config{
|
||||
EnableLRU: true,
|
||||
MaxSize: c.ParentalCacheSize,
|
||||
})
|
||||
|
||||
d.safeSearch = c.SafeSearch
|
||||
|
||||
d.hostCheckers = []hostChecker{{
|
||||
|
|
@ -977,11 +979,6 @@ func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) {
|
|||
|
||||
defer func() { err = errors.Annotate(err, "filtering: %w") }()
|
||||
|
||||
err = d.initSecurityServices()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("initializing services: %s", err)
|
||||
}
|
||||
|
||||
d.Config = *c
|
||||
d.filtersMu = &sync.RWMutex{}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/filtering/hashprefix"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/testutil"
|
||||
"github.com/AdguardTeam/urlfilter/rules"
|
||||
|
|
@ -27,17 +27,6 @@ const (
|
|||
|
||||
// Helpers.
|
||||
|
||||
func purgeCaches(d *DNSFilter) {
|
||||
for _, c := range []cache.Cache{
|
||||
d.safebrowsingCache,
|
||||
d.parentalCache,
|
||||
} {
|
||||
if c != nil {
|
||||
c.Clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newForTest(t testing.TB, c *Config, filters []Filter) (f *DNSFilter, setts *Settings) {
|
||||
setts = &Settings{
|
||||
ProtectionEnabled: true,
|
||||
|
|
@ -58,11 +47,17 @@ func newForTest(t testing.TB, c *Config, filters []Filter) (f *DNSFilter, setts
|
|||
f, err := New(c, filters)
|
||||
require.NoError(t, err)
|
||||
|
||||
purgeCaches(f)
|
||||
|
||||
return f, setts
|
||||
}
|
||||
|
||||
func newChecker(host string) Checker {
|
||||
return hashprefix.New(&hashprefix.Config{
|
||||
CacheTime: 10,
|
||||
CacheSize: 100000,
|
||||
Upstream: aghtest.NewBlockUpstream(host, true),
|
||||
})
|
||||
}
|
||||
|
||||
func (d *DNSFilter) checkMatch(t *testing.T, hostname string, setts *Settings) {
|
||||
t.Helper()
|
||||
|
||||
|
|
@ -175,10 +170,14 @@ func TestSafeBrowsing(t *testing.T) {
|
|||
aghtest.ReplaceLogWriter(t, logOutput)
|
||||
aghtest.ReplaceLogLevel(t, log.DEBUG)
|
||||
|
||||
d, setts := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
|
||||
sbChecker := newChecker(sbBlocked)
|
||||
|
||||
d, setts := newForTest(t, &Config{
|
||||
SafeBrowsingEnabled: true,
|
||||
SafeBrowsingChecker: sbChecker,
|
||||
}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
|
||||
d.checkMatch(t, sbBlocked, setts)
|
||||
|
||||
require.Contains(t, logOutput.String(), fmt.Sprintf("safebrowsing lookup for %q", sbBlocked))
|
||||
|
|
@ -188,18 +187,17 @@ func TestSafeBrowsing(t *testing.T) {
|
|||
d.checkMatchEmpty(t, pcBlocked, setts)
|
||||
|
||||
// Cached result.
|
||||
d.safeBrowsingServer = "127.0.0.1"
|
||||
d.checkMatch(t, sbBlocked, setts)
|
||||
d.checkMatchEmpty(t, pcBlocked, setts)
|
||||
d.safeBrowsingServer = defaultSafebrowsingServer
|
||||
}
|
||||
|
||||
func TestParallelSB(t *testing.T) {
|
||||
d, setts := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
|
||||
d, setts := newForTest(t, &Config{
|
||||
SafeBrowsingEnabled: true,
|
||||
SafeBrowsingChecker: newChecker(sbBlocked),
|
||||
}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
|
||||
|
||||
t.Run("group", func(t *testing.T) {
|
||||
for i := 0; i < 100; i++ {
|
||||
t.Run(fmt.Sprintf("aaa%d", i), func(t *testing.T) {
|
||||
|
|
@ -220,10 +218,12 @@ func TestParentalControl(t *testing.T) {
|
|||
aghtest.ReplaceLogWriter(t, logOutput)
|
||||
aghtest.ReplaceLogLevel(t, log.DEBUG)
|
||||
|
||||
d, setts := newForTest(t, &Config{ParentalEnabled: true}, nil)
|
||||
d, setts := newForTest(t, &Config{
|
||||
ParentalEnabled: true,
|
||||
ParentalControlChecker: newChecker(pcBlocked),
|
||||
}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
d.SetParentalUpstream(aghtest.NewBlockUpstream(pcBlocked, true))
|
||||
d.checkMatch(t, pcBlocked, setts)
|
||||
require.Contains(t, logOutput.String(), fmt.Sprintf("parental lookup for %q", pcBlocked))
|
||||
|
||||
|
|
@ -233,7 +233,6 @@ func TestParentalControl(t *testing.T) {
|
|||
d.checkMatchEmpty(t, "api.jquery.com", setts)
|
||||
|
||||
// Test cached result.
|
||||
d.parentalServer = "127.0.0.1"
|
||||
d.checkMatch(t, pcBlocked, setts)
|
||||
d.checkMatchEmpty(t, "yandex.ru", setts)
|
||||
}
|
||||
|
|
@ -593,8 +592,10 @@ func applyClientSettings(setts *Settings) {
|
|||
func TestClientSettings(t *testing.T) {
|
||||
d, setts := newForTest(t,
|
||||
&Config{
|
||||
ParentalEnabled: true,
|
||||
SafeBrowsingEnabled: false,
|
||||
ParentalEnabled: true,
|
||||
SafeBrowsingEnabled: false,
|
||||
SafeBrowsingChecker: newChecker(sbBlocked),
|
||||
ParentalControlChecker: newChecker(pcBlocked),
|
||||
},
|
||||
[]Filter{{
|
||||
ID: 0, Data: []byte("||example.org^\n"),
|
||||
|
|
@ -602,9 +603,6 @@ func TestClientSettings(t *testing.T) {
|
|||
)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
d.SetParentalUpstream(aghtest.NewBlockUpstream(pcBlocked, true))
|
||||
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
|
||||
|
||||
type testCase struct {
|
||||
name string
|
||||
host string
|
||||
|
|
@ -665,11 +663,12 @@ func TestClientSettings(t *testing.T) {
|
|||
// Benchmarks.
|
||||
|
||||
func BenchmarkSafeBrowsing(b *testing.B) {
|
||||
d, setts := newForTest(b, &Config{SafeBrowsingEnabled: true}, nil)
|
||||
d, setts := newForTest(b, &Config{
|
||||
SafeBrowsingEnabled: true,
|
||||
SafeBrowsingChecker: newChecker(sbBlocked),
|
||||
}, nil)
|
||||
b.Cleanup(d.Close)
|
||||
|
||||
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
|
||||
|
||||
for n := 0; n < b.N; n++ {
|
||||
res, err := d.CheckHost(sbBlocked, dns.TypeA, setts)
|
||||
require.NoError(b, err)
|
||||
|
|
@ -679,11 +678,12 @@ func BenchmarkSafeBrowsing(b *testing.B) {
|
|||
}
|
||||
|
||||
func BenchmarkSafeBrowsingParallel(b *testing.B) {
|
||||
d, setts := newForTest(b, &Config{SafeBrowsingEnabled: true}, nil)
|
||||
d, setts := newForTest(b, &Config{
|
||||
SafeBrowsingEnabled: true,
|
||||
SafeBrowsingChecker: newChecker(sbBlocked),
|
||||
}, nil)
|
||||
b.Cleanup(d.Close)
|
||||
|
||||
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
res, err := d.CheckHost(sbBlocked, dns.TypeA, setts)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,130 @@
|
|||
package hashprefix
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
)
|
||||
|
||||
// expirySize is the size of expiry in cacheItem.
|
||||
const expirySize = 8
|
||||
|
||||
// cacheItem represents an item that we will store in the cache.
|
||||
type cacheItem struct {
|
||||
// expiry is the time when cacheItem will expire.
|
||||
expiry time.Time
|
||||
|
||||
// hashes is the hashed hostnames.
|
||||
hashes []hostnameHash
|
||||
}
|
||||
|
||||
// toCacheItem decodes cacheItem from data. data must be at least equal to
|
||||
// expiry size.
|
||||
func toCacheItem(data []byte) *cacheItem {
|
||||
t := time.Unix(int64(binary.BigEndian.Uint64(data)), 0)
|
||||
|
||||
data = data[expirySize:]
|
||||
hashes := make([]hostnameHash, len(data)/hashSize)
|
||||
|
||||
for i := 0; i < len(data); i += hashSize {
|
||||
var hash hostnameHash
|
||||
copy(hash[:], data[i:i+hashSize])
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
|
||||
return &cacheItem{
|
||||
expiry: t,
|
||||
hashes: hashes,
|
||||
}
|
||||
}
|
||||
|
||||
// fromCacheItem encodes cacheItem into data.
|
||||
func fromCacheItem(item *cacheItem) (data []byte) {
|
||||
data = make([]byte, len(item.hashes)*hashSize+expirySize)
|
||||
expiry := item.expiry.Unix()
|
||||
binary.BigEndian.PutUint64(data[:expirySize], uint64(expiry))
|
||||
|
||||
for _, v := range item.hashes {
|
||||
// nolint:looppointer // The subsilce is used for a copy.
|
||||
data = append(data, v[:]...)
|
||||
}
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
// findInCache finds hashes in the cache. If nothing found returns list of
|
||||
// hashes, prefixes of which will be sent to upstream.
|
||||
func (c *Checker) findInCache(
|
||||
hashes []hostnameHash,
|
||||
) (found, blocked bool, hashesToRequest []hostnameHash) {
|
||||
now := time.Now()
|
||||
|
||||
i := 0
|
||||
for _, hash := range hashes {
|
||||
// nolint:looppointer // The subsilce is used for a safe cache lookup.
|
||||
data := c.cache.Get(hash[:prefixLen])
|
||||
if data == nil {
|
||||
hashes[i] = hash
|
||||
i++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
item := toCacheItem(data)
|
||||
if now.After(item.expiry) {
|
||||
hashes[i] = hash
|
||||
i++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if ok := findMatch(hashes, item.hashes); ok {
|
||||
return true, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
if i == 0 {
|
||||
return true, false, nil
|
||||
}
|
||||
|
||||
return false, false, hashes[:i]
|
||||
}
|
||||
|
||||
// storeInCache caches hashes.
|
||||
func (c *Checker) storeInCache(hashesToRequest, respHashes []hostnameHash) {
|
||||
hashToStore := make(map[prefix][]hostnameHash)
|
||||
|
||||
for _, hash := range respHashes {
|
||||
var pref prefix
|
||||
// nolint:looppointer // The subsilce is used for a copy.
|
||||
copy(pref[:], hash[:])
|
||||
|
||||
hashToStore[pref] = append(hashToStore[pref], hash)
|
||||
}
|
||||
|
||||
for pref, hash := range hashToStore {
|
||||
// nolint:looppointer // The subsilce is used for a safe cache lookup.
|
||||
c.setCache(pref[:], hash)
|
||||
}
|
||||
|
||||
for _, hash := range hashesToRequest {
|
||||
// nolint:looppointer // The subsilce is used for a safe cache lookup.
|
||||
pref := hash[:prefixLen]
|
||||
val := c.cache.Get(pref)
|
||||
if val == nil {
|
||||
c.setCache(pref, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// setCache stores hash in cache.
|
||||
func (c *Checker) setCache(pref []byte, hashes []hostnameHash) {
|
||||
item := &cacheItem{
|
||||
expiry: time.Now().Add(c.cacheTime),
|
||||
hashes: hashes,
|
||||
}
|
||||
|
||||
c.cache.Set(pref, fromCacheItem(item))
|
||||
log.Debug("%s: stored in cache: %v", c.svc, pref)
|
||||
}
|
||||
|
|
@ -0,0 +1,245 @@
|
|||
// Package hashprefix used for safe browsing and parent control.
|
||||
package hashprefix
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/dnsproxy/upstream"
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/netutil"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/miekg/dns"
|
||||
"golang.org/x/exp/slices"
|
||||
"golang.org/x/net/publicsuffix"
|
||||
)
|
||||
|
||||
const (
|
||||
// prefixLen is the length of the hash prefix of the filtered hostname.
|
||||
prefixLen = 2
|
||||
|
||||
// hashSize is the size of hashed hostname.
|
||||
hashSize = sha256.Size
|
||||
|
||||
// hexSize is the size of hexadecimal representation of hashed hostname.
|
||||
hexSize = hashSize * 2
|
||||
)
|
||||
|
||||
// prefix is the type of the SHA256 hash prefix used to match against the
|
||||
// domain-name database.
|
||||
type prefix [prefixLen]byte
|
||||
|
||||
// hostnameHash is the hashed hostname.
|
||||
//
|
||||
// TODO(s.chzhen): Split into prefix and suffix.
|
||||
type hostnameHash [hashSize]byte
|
||||
|
||||
// findMatch returns true if one of the a hostnames matches one of the b.
|
||||
func findMatch(a, b []hostnameHash) (matched bool) {
|
||||
for _, hash := range a {
|
||||
if slices.Contains(b, hash) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Config is the configuration structure for safe browsing and parental
|
||||
// control.
|
||||
type Config struct {
|
||||
// Upstream is the upstream DNS server.
|
||||
Upstream upstream.Upstream
|
||||
|
||||
// ServiceName is the name of the service.
|
||||
ServiceName string
|
||||
|
||||
// TXTSuffix is the TXT suffix for DNS request.
|
||||
TXTSuffix string
|
||||
|
||||
// CacheTime is the time period to store hash.
|
||||
CacheTime time.Duration
|
||||
|
||||
// CacheSize is the maximum size of the cache. If it's zero, cache size is
|
||||
// unlimited.
|
||||
CacheSize uint
|
||||
}
|
||||
|
||||
type Checker struct {
|
||||
// upstream is the upstream DNS server.
|
||||
upstream upstream.Upstream
|
||||
|
||||
// cache stores hostname hashes.
|
||||
cache cache.Cache
|
||||
|
||||
// svc is the name of the service.
|
||||
svc string
|
||||
|
||||
// txtSuffix is the TXT suffix for DNS request.
|
||||
txtSuffix string
|
||||
|
||||
// cacheTime is the time period to store hash.
|
||||
cacheTime time.Duration
|
||||
}
|
||||
|
||||
// New returns Checker.
|
||||
func New(conf *Config) (c *Checker) {
|
||||
return &Checker{
|
||||
upstream: conf.Upstream,
|
||||
cache: cache.New(cache.Config{
|
||||
EnableLRU: true,
|
||||
MaxSize: conf.CacheSize,
|
||||
}),
|
||||
svc: conf.ServiceName,
|
||||
txtSuffix: conf.TXTSuffix,
|
||||
cacheTime: conf.CacheTime,
|
||||
}
|
||||
}
|
||||
|
||||
// Check returns true if request for the host should be blocked.
|
||||
func (c *Checker) Check(host string) (ok bool, err error) {
|
||||
hashes := hostnameToHashes(host)
|
||||
|
||||
found, blocked, hashesToRequest := c.findInCache(hashes)
|
||||
if found {
|
||||
log.Debug("%s: found %q in cache, blocked: %t", c.svc, host, blocked)
|
||||
|
||||
return blocked, nil
|
||||
}
|
||||
|
||||
question := c.getQuestion(hashesToRequest)
|
||||
|
||||
log.Debug("%s: checking %s: %s", c.svc, host, question)
|
||||
req := (&dns.Msg{}).SetQuestion(question, dns.TypeTXT)
|
||||
|
||||
resp, err := c.upstream.Exchange(req)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("getting hashes: %w", err)
|
||||
}
|
||||
|
||||
matched, receivedHashes := c.processAnswer(hashesToRequest, resp, host)
|
||||
|
||||
c.storeInCache(hashesToRequest, receivedHashes)
|
||||
|
||||
return matched, nil
|
||||
}
|
||||
|
||||
// hostnameToHashes returns hashes that should be checked by the hash prefix
|
||||
// filter.
|
||||
func hostnameToHashes(host string) (hashes []hostnameHash) {
|
||||
// subDomainNum defines how many labels should be hashed to match against a
|
||||
// hash prefix filter.
|
||||
const subDomainNum = 4
|
||||
|
||||
pubSuf, icann := publicsuffix.PublicSuffix(host)
|
||||
if !icann {
|
||||
// Check the full private domain space.
|
||||
pubSuf = ""
|
||||
}
|
||||
|
||||
nDots := 0
|
||||
i := strings.LastIndexFunc(host, func(r rune) (ok bool) {
|
||||
if r == '.' {
|
||||
nDots++
|
||||
}
|
||||
|
||||
return nDots == subDomainNum
|
||||
})
|
||||
if i != -1 {
|
||||
host = host[i+1:]
|
||||
}
|
||||
|
||||
sub := netutil.Subdomains(host)
|
||||
|
||||
for _, s := range sub {
|
||||
if s == pubSuf {
|
||||
break
|
||||
}
|
||||
|
||||
sum := sha256.Sum256([]byte(s))
|
||||
hashes = append(hashes, sum)
|
||||
}
|
||||
|
||||
return hashes
|
||||
}
|
||||
|
||||
// getQuestion combines hexadecimal encoded prefixes of hashed hostnames into
|
||||
// string.
|
||||
func (c *Checker) getQuestion(hashes []hostnameHash) (q string) {
|
||||
b := &strings.Builder{}
|
||||
|
||||
for _, hash := range hashes {
|
||||
// nolint:looppointer // The subsilce is used for safe hex encoding.
|
||||
stringutil.WriteToBuilder(b, hex.EncodeToString(hash[:prefixLen]), ".")
|
||||
}
|
||||
|
||||
stringutil.WriteToBuilder(b, c.txtSuffix)
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// processAnswer returns true if DNS response matches the hash, and received
|
||||
// hashed hostnames from the upstream.
|
||||
func (c *Checker) processAnswer(
|
||||
hashesToRequest []hostnameHash,
|
||||
resp *dns.Msg,
|
||||
host string,
|
||||
) (matched bool, receivedHashes []hostnameHash) {
|
||||
txtCount := 0
|
||||
|
||||
for _, a := range resp.Answer {
|
||||
txt, ok := a.(*dns.TXT)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
txtCount++
|
||||
|
||||
receivedHashes = c.appendHashesFromTXT(receivedHashes, txt, host)
|
||||
}
|
||||
|
||||
log.Debug("%s: received answer for %s with %d TXT count", c.svc, host, txtCount)
|
||||
|
||||
matched = findMatch(hashesToRequest, receivedHashes)
|
||||
if matched {
|
||||
log.Debug("%s: matched %s", c.svc, host)
|
||||
|
||||
return true, receivedHashes
|
||||
}
|
||||
|
||||
return false, receivedHashes
|
||||
}
|
||||
|
||||
// appendHashesFromTXT appends received hashed hostnames.
|
||||
func (c *Checker) appendHashesFromTXT(
|
||||
hashes []hostnameHash,
|
||||
txt *dns.TXT,
|
||||
host string,
|
||||
) (receivedHashes []hostnameHash) {
|
||||
log.Debug("%s: received hashes for %s: %v", c.svc, host, txt.Txt)
|
||||
|
||||
for _, t := range txt.Txt {
|
||||
if len(t) != hexSize {
|
||||
log.Debug("%s: wrong hex size %d for %s %s", c.svc, len(t), host, t)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
buf, err := hex.DecodeString(t)
|
||||
if err != nil {
|
||||
log.Debug("%s: decoding hex string %s: %s", c.svc, t, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
var hash hostnameHash
|
||||
copy(hash[:], buf)
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
|
||||
return hashes
|
||||
}
|
||||
|
|
@ -0,0 +1,248 @@
|
|||
package hashprefix
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
const (
|
||||
cacheTime = 10 * time.Minute
|
||||
cacheSize = 10000
|
||||
)
|
||||
|
||||
func TestChcker_getQuestion(t *testing.T) {
|
||||
const suf = "sb.dns.adguard.com."
|
||||
|
||||
// test hostnameToHashes()
|
||||
hashes := hostnameToHashes("1.2.3.sub.host.com")
|
||||
assert.Len(t, hashes, 3)
|
||||
|
||||
hash := sha256.Sum256([]byte("3.sub.host.com"))
|
||||
hexPref1 := hex.EncodeToString(hash[:prefixLen])
|
||||
assert.True(t, slices.Contains(hashes, hash))
|
||||
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
hexPref2 := hex.EncodeToString(hash[:prefixLen])
|
||||
assert.True(t, slices.Contains(hashes, hash))
|
||||
|
||||
hash = sha256.Sum256([]byte("host.com"))
|
||||
hexPref3 := hex.EncodeToString(hash[:prefixLen])
|
||||
assert.True(t, slices.Contains(hashes, hash))
|
||||
|
||||
hash = sha256.Sum256([]byte("com"))
|
||||
assert.False(t, slices.Contains(hashes, hash))
|
||||
|
||||
c := &Checker{
|
||||
svc: "SafeBrowsing",
|
||||
txtSuffix: suf,
|
||||
}
|
||||
|
||||
q := c.getQuestion(hashes)
|
||||
|
||||
assert.Contains(t, q, hexPref1)
|
||||
assert.Contains(t, q, hexPref2)
|
||||
assert.Contains(t, q, hexPref3)
|
||||
assert.True(t, strings.HasSuffix(q, suf))
|
||||
}
|
||||
|
||||
func TestHostnameToHashes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
host string
|
||||
wantLen int
|
||||
}{{
|
||||
name: "basic",
|
||||
host: "example.com",
|
||||
wantLen: 1,
|
||||
}, {
|
||||
name: "sub_basic",
|
||||
host: "www.example.com",
|
||||
wantLen: 2,
|
||||
}, {
|
||||
name: "private_domain",
|
||||
host: "foo.co.uk",
|
||||
wantLen: 1,
|
||||
}, {
|
||||
name: "sub_private_domain",
|
||||
host: "bar.foo.co.uk",
|
||||
wantLen: 2,
|
||||
}, {
|
||||
name: "private_domain_v2",
|
||||
host: "foo.blogspot.co.uk",
|
||||
wantLen: 4,
|
||||
}, {
|
||||
name: "sub_private_domain_v2",
|
||||
host: "bar.foo.blogspot.co.uk",
|
||||
wantLen: 4,
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
hashes := hostnameToHashes(tc.host)
|
||||
assert.Len(t, hashes, tc.wantLen)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChecker_storeInCache(t *testing.T) {
|
||||
c := &Checker{
|
||||
svc: "SafeBrowsing",
|
||||
cacheTime: cacheTime,
|
||||
}
|
||||
conf := cache.Config{}
|
||||
c.cache = cache.New(conf)
|
||||
|
||||
// store in cache hashes for "3.sub.host.com" and "host.com"
|
||||
// and empty data for hash-prefix for "sub.host.com"
|
||||
hashes := []hostnameHash{}
|
||||
hash := sha256.Sum256([]byte("sub.host.com"))
|
||||
hashes = append(hashes, hash)
|
||||
var hashesArray []hostnameHash
|
||||
hash4 := sha256.Sum256([]byte("3.sub.host.com"))
|
||||
hashesArray = append(hashesArray, hash4)
|
||||
hash2 := sha256.Sum256([]byte("host.com"))
|
||||
hashesArray = append(hashesArray, hash2)
|
||||
c.storeInCache(hashes, hashesArray)
|
||||
|
||||
// match "3.sub.host.com" or "host.com" from cache
|
||||
hashes = []hostnameHash{}
|
||||
hash = sha256.Sum256([]byte("3.sub.host.com"))
|
||||
hashes = append(hashes, hash)
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
hashes = append(hashes, hash)
|
||||
hash = sha256.Sum256([]byte("host.com"))
|
||||
hashes = append(hashes, hash)
|
||||
found, blocked, _ := c.findInCache(hashes)
|
||||
assert.True(t, found)
|
||||
assert.True(t, blocked)
|
||||
|
||||
// match "sub.host.com" from cache
|
||||
hashes = []hostnameHash{}
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
hashes = append(hashes, hash)
|
||||
found, blocked, _ = c.findInCache(hashes)
|
||||
assert.True(t, found)
|
||||
assert.False(t, blocked)
|
||||
|
||||
// Match "sub.host.com" from cache. Another hash for "host.example" is not
|
||||
// in the cache, so get data for it from the server.
|
||||
hashes = []hostnameHash{}
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
hashes = append(hashes, hash)
|
||||
hash = sha256.Sum256([]byte("host.example"))
|
||||
hashes = append(hashes, hash)
|
||||
found, _, hashesToRequest := c.findInCache(hashes)
|
||||
assert.False(t, found)
|
||||
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
ok := slices.Contains(hashesToRequest, hash)
|
||||
assert.False(t, ok)
|
||||
|
||||
hash = sha256.Sum256([]byte("host.example"))
|
||||
ok = slices.Contains(hashesToRequest, hash)
|
||||
assert.True(t, ok)
|
||||
|
||||
c = &Checker{
|
||||
svc: "SafeBrowsing",
|
||||
cacheTime: cacheTime,
|
||||
}
|
||||
c.cache = cache.New(cache.Config{})
|
||||
|
||||
hashes = []hostnameHash{}
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
hashes = append(hashes, hash)
|
||||
|
||||
c.cache.Set(hash[:prefixLen], make([]byte, expirySize+hashSize))
|
||||
found, _, _ = c.findInCache(hashes)
|
||||
assert.False(t, found)
|
||||
}
|
||||
|
||||
func TestChecker_Check(t *testing.T) {
|
||||
const hostname = "example.org"
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
wantBlock bool
|
||||
}{{
|
||||
name: "sb_no_block",
|
||||
wantBlock: false,
|
||||
}, {
|
||||
name: "sb_block",
|
||||
wantBlock: true,
|
||||
}, {
|
||||
name: "pc_no_block",
|
||||
wantBlock: false,
|
||||
}, {
|
||||
name: "pc_block",
|
||||
wantBlock: true,
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
c := New(&Config{
|
||||
CacheTime: cacheTime,
|
||||
CacheSize: cacheSize,
|
||||
})
|
||||
|
||||
// Prepare the upstream.
|
||||
ups := aghtest.NewBlockUpstream(hostname, tc.wantBlock)
|
||||
|
||||
var numReq int
|
||||
onExchange := ups.OnExchange
|
||||
ups.OnExchange = func(req *dns.Msg) (resp *dns.Msg, err error) {
|
||||
numReq++
|
||||
|
||||
return onExchange(req)
|
||||
}
|
||||
|
||||
c.upstream = ups
|
||||
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Firstly, check the request blocking.
|
||||
hits := 0
|
||||
res := false
|
||||
res, err := c.Check(hostname)
|
||||
require.NoError(t, err)
|
||||
|
||||
if tc.wantBlock {
|
||||
assert.True(t, res)
|
||||
hits++
|
||||
} else {
|
||||
require.False(t, res)
|
||||
}
|
||||
|
||||
// Check the cache state, check the response is now cached.
|
||||
assert.Equal(t, 1, c.cache.Stats().Count)
|
||||
assert.Equal(t, hits, c.cache.Stats().Hit)
|
||||
|
||||
// There was one request to an upstream.
|
||||
assert.Equal(t, 1, numReq)
|
||||
|
||||
// Now make the same request to check the cache was used.
|
||||
res, err = c.Check(hostname)
|
||||
require.NoError(t, err)
|
||||
|
||||
if tc.wantBlock {
|
||||
assert.True(t, res)
|
||||
} else {
|
||||
require.False(t, res)
|
||||
}
|
||||
|
||||
// Check the cache state, it should've been used.
|
||||
assert.Equal(t, 1, c.cache.Stats().Count)
|
||||
assert.Equal(t, hits+1, c.cache.Stats().Hit)
|
||||
|
||||
// Check that there were no additional requests.
|
||||
assert.Equal(t, 1, numReq)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,307 +1,15 @@
|
|||
package filtering
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
||||
"github.com/AdguardTeam/dnsproxy/upstream"
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/miekg/dns"
|
||||
"golang.org/x/exp/slices"
|
||||
"golang.org/x/net/publicsuffix"
|
||||
)
|
||||
|
||||
// Safe browsing and parental control methods.
|
||||
|
||||
// TODO(a.garipov): Make configurable.
|
||||
const (
|
||||
dnsTimeout = 3 * time.Second
|
||||
defaultSafebrowsingServer = `https://family.adguard-dns.com/dns-query`
|
||||
defaultParentalServer = `https://family.adguard-dns.com/dns-query`
|
||||
sbTXTSuffix = `sb.dns.adguard.com.`
|
||||
pcTXTSuffix = `pc.dns.adguard.com.`
|
||||
)
|
||||
|
||||
// SetParentalUpstream sets the parental upstream for *DNSFilter.
|
||||
//
|
||||
// TODO(e.burkov): Remove this in v1 API to forbid the direct access.
|
||||
func (d *DNSFilter) SetParentalUpstream(u upstream.Upstream) {
|
||||
d.parentalUpstream = u
|
||||
}
|
||||
|
||||
// SetSafeBrowsingUpstream sets the safe browsing upstream for *DNSFilter.
|
||||
//
|
||||
// TODO(e.burkov): Remove this in v1 API to forbid the direct access.
|
||||
func (d *DNSFilter) SetSafeBrowsingUpstream(u upstream.Upstream) {
|
||||
d.safeBrowsingUpstream = u
|
||||
}
|
||||
|
||||
func (d *DNSFilter) initSecurityServices() error {
|
||||
var err error
|
||||
d.safeBrowsingServer = defaultSafebrowsingServer
|
||||
d.parentalServer = defaultParentalServer
|
||||
opts := &upstream.Options{
|
||||
Timeout: dnsTimeout,
|
||||
ServerIPAddrs: []net.IP{
|
||||
{94, 140, 14, 15},
|
||||
{94, 140, 15, 16},
|
||||
net.ParseIP("2a10:50c0::bad1:ff"),
|
||||
net.ParseIP("2a10:50c0::bad2:ff"),
|
||||
},
|
||||
}
|
||||
|
||||
parUps, err := upstream.AddressToUpstream(d.parentalServer, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("converting parental server: %w", err)
|
||||
}
|
||||
d.SetParentalUpstream(parUps)
|
||||
|
||||
sbUps, err := upstream.AddressToUpstream(d.safeBrowsingServer, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("converting safe browsing server: %w", err)
|
||||
}
|
||||
d.SetSafeBrowsingUpstream(sbUps)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
expire byte[4]
|
||||
hash byte[32]
|
||||
...
|
||||
*/
|
||||
func (c *sbCtx) setCache(prefix, hashes []byte) {
|
||||
d := make([]byte, 4+len(hashes))
|
||||
expire := uint(time.Now().Unix()) + c.cacheTime*60
|
||||
binary.BigEndian.PutUint32(d[:4], uint32(expire))
|
||||
copy(d[4:], hashes)
|
||||
c.cache.Set(prefix, d)
|
||||
log.Debug("%s: stored in cache: %v", c.svc, prefix)
|
||||
}
|
||||
|
||||
// findInHash returns 32-byte hash if it's found in hashToHost.
|
||||
func (c *sbCtx) findInHash(val []byte) (hash32 [32]byte, found bool) {
|
||||
for i := 4; i < len(val); i += 32 {
|
||||
hash := val[i : i+32]
|
||||
|
||||
copy(hash32[:], hash[0:32])
|
||||
|
||||
_, found = c.hashToHost[hash32]
|
||||
if found {
|
||||
return hash32, found
|
||||
}
|
||||
}
|
||||
|
||||
return [32]byte{}, false
|
||||
}
|
||||
|
||||
func (c *sbCtx) getCached() int {
|
||||
now := time.Now().Unix()
|
||||
hashesToRequest := map[[32]byte]string{}
|
||||
for k, v := range c.hashToHost {
|
||||
// nolint:looppointer // The subsilce is used for a safe cache lookup.
|
||||
val := c.cache.Get(k[0:2])
|
||||
if val == nil || now >= int64(binary.BigEndian.Uint32(val)) {
|
||||
hashesToRequest[k] = v
|
||||
continue
|
||||
}
|
||||
if hash32, found := c.findInHash(val); found {
|
||||
log.Debug("%s: found in cache: %s: blocked by %v", c.svc, c.host, hash32)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
if len(hashesToRequest) == 0 {
|
||||
log.Debug("%s: found in cache: %s: not blocked", c.svc, c.host)
|
||||
return -1
|
||||
}
|
||||
|
||||
c.hashToHost = hashesToRequest
|
||||
return 0
|
||||
}
|
||||
|
||||
type sbCtx struct {
|
||||
host string
|
||||
svc string
|
||||
hashToHost map[[32]byte]string
|
||||
cache cache.Cache
|
||||
cacheTime uint
|
||||
}
|
||||
|
||||
func hostnameToHashes(host string) map[[32]byte]string {
|
||||
hashes := map[[32]byte]string{}
|
||||
tld, icann := publicsuffix.PublicSuffix(host)
|
||||
if !icann {
|
||||
// private suffixes like cloudfront.net
|
||||
tld = ""
|
||||
}
|
||||
curhost := host
|
||||
|
||||
nDots := 0
|
||||
for i := len(curhost) - 1; i >= 0; i-- {
|
||||
if curhost[i] == '.' {
|
||||
nDots++
|
||||
if nDots == 4 {
|
||||
curhost = curhost[i+1:] // "xxx.a.b.c.d" -> "a.b.c.d"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
if curhost == "" {
|
||||
// we've reached end of string
|
||||
break
|
||||
}
|
||||
if tld != "" && curhost == tld {
|
||||
// we've reached the TLD, don't hash it
|
||||
break
|
||||
}
|
||||
|
||||
sum := sha256.Sum256([]byte(curhost))
|
||||
hashes[sum] = curhost
|
||||
|
||||
pos := strings.IndexByte(curhost, byte('.'))
|
||||
if pos < 0 {
|
||||
break
|
||||
}
|
||||
curhost = curhost[pos+1:]
|
||||
}
|
||||
return hashes
|
||||
}
|
||||
|
||||
// convert hash array to string
|
||||
func (c *sbCtx) getQuestion() string {
|
||||
b := &strings.Builder{}
|
||||
|
||||
for hash := range c.hashToHost {
|
||||
// nolint:looppointer // The subsilce is used for safe hex encoding.
|
||||
stringutil.WriteToBuilder(b, hex.EncodeToString(hash[0:2]), ".")
|
||||
}
|
||||
|
||||
if c.svc == "SafeBrowsing" {
|
||||
stringutil.WriteToBuilder(b, sbTXTSuffix)
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
stringutil.WriteToBuilder(b, pcTXTSuffix)
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Find the target hash in TXT response
|
||||
func (c *sbCtx) processTXT(resp *dns.Msg) (bool, [][]byte) {
|
||||
matched := false
|
||||
hashes := [][]byte{}
|
||||
for _, a := range resp.Answer {
|
||||
txt, ok := a.(*dns.TXT)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
log.Debug("%s: received hashes for %s: %v", c.svc, c.host, txt.Txt)
|
||||
|
||||
for _, t := range txt.Txt {
|
||||
if len(t) != 32*2 {
|
||||
continue
|
||||
}
|
||||
hash, err := hex.DecodeString(t)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
hashes = append(hashes, hash)
|
||||
|
||||
if !matched {
|
||||
var hash32 [32]byte
|
||||
copy(hash32[:], hash)
|
||||
|
||||
var hashHost string
|
||||
hashHost, ok = c.hashToHost[hash32]
|
||||
if ok {
|
||||
log.Debug("%s: matched %s by %s/%s", c.svc, c.host, hashHost, t)
|
||||
matched = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return matched, hashes
|
||||
}
|
||||
|
||||
func (c *sbCtx) storeCache(hashes [][]byte) {
|
||||
slices.SortFunc(hashes, func(a, b []byte) (sortsBefore bool) {
|
||||
return bytes.Compare(a, b) == -1
|
||||
})
|
||||
|
||||
var curData []byte
|
||||
var prevPrefix []byte
|
||||
for i, hash := range hashes {
|
||||
// nolint:looppointer // The subsilce is used for a safe comparison.
|
||||
if !bytes.Equal(hash[0:2], prevPrefix) {
|
||||
if i != 0 {
|
||||
c.setCache(prevPrefix, curData)
|
||||
curData = nil
|
||||
}
|
||||
prevPrefix = hashes[i][0:2]
|
||||
}
|
||||
curData = append(curData, hash...)
|
||||
}
|
||||
|
||||
if len(prevPrefix) != 0 {
|
||||
c.setCache(prevPrefix, curData)
|
||||
}
|
||||
|
||||
for hash := range c.hashToHost {
|
||||
// nolint:looppointer // The subsilce is used for a safe cache lookup.
|
||||
prefix := hash[0:2]
|
||||
val := c.cache.Get(prefix)
|
||||
if val == nil {
|
||||
c.setCache(prefix, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func check(c *sbCtx, r Result, u upstream.Upstream) (Result, error) {
|
||||
c.hashToHost = hostnameToHashes(c.host)
|
||||
switch c.getCached() {
|
||||
case -1:
|
||||
return Result{}, nil
|
||||
case 1:
|
||||
return r, nil
|
||||
}
|
||||
|
||||
question := c.getQuestion()
|
||||
|
||||
log.Tracef("%s: checking %s: %s", c.svc, c.host, question)
|
||||
req := (&dns.Msg{}).SetQuestion(question, dns.TypeTXT)
|
||||
|
||||
resp, err := u.Exchange(req)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
matched, receivedHashes := c.processTXT(resp)
|
||||
|
||||
c.storeCache(receivedHashes)
|
||||
if matched {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
// TODO(a.garipov): Unify with checkParental.
|
||||
func (d *DNSFilter) checkSafeBrowsing(
|
||||
host string,
|
||||
|
|
@ -317,13 +25,6 @@ func (d *DNSFilter) checkSafeBrowsing(
|
|||
defer timer.LogElapsed("safebrowsing lookup for %q", host)
|
||||
}
|
||||
|
||||
sctx := &sbCtx{
|
||||
host: host,
|
||||
svc: "SafeBrowsing",
|
||||
cache: d.safebrowsingCache,
|
||||
cacheTime: d.Config.CacheTime,
|
||||
}
|
||||
|
||||
res = Result{
|
||||
Rules: []*ResultRule{{
|
||||
Text: "adguard-malware-shavar",
|
||||
|
|
@ -333,7 +34,12 @@ func (d *DNSFilter) checkSafeBrowsing(
|
|||
IsFiltered: true,
|
||||
}
|
||||
|
||||
return check(sctx, res, d.safeBrowsingUpstream)
|
||||
block, err := d.safeBrowsingChecker.Check(host)
|
||||
if !block || err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// TODO(a.garipov): Unify with checkSafeBrowsing.
|
||||
|
|
@ -351,13 +57,6 @@ func (d *DNSFilter) checkParental(
|
|||
defer timer.LogElapsed("parental lookup for %q", host)
|
||||
}
|
||||
|
||||
sctx := &sbCtx{
|
||||
host: host,
|
||||
svc: "Parental",
|
||||
cache: d.parentalCache,
|
||||
cacheTime: d.Config.CacheTime,
|
||||
}
|
||||
|
||||
res = Result{
|
||||
Rules: []*ResultRule{{
|
||||
Text: "parental CATEGORY_BLACKLISTED",
|
||||
|
|
@ -367,7 +66,12 @@ func (d *DNSFilter) checkParental(
|
|||
IsFiltered: true,
|
||||
}
|
||||
|
||||
return check(sctx, res, d.parentalUpstream)
|
||||
block, err := d.parentalControlChecker.Check(host)
|
||||
if !block || err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// setProtectedBool sets the value of a boolean pointer under a lock. l must
|
||||
|
|
|
|||
|
|
@ -1,226 +0,0 @@
|
|||
package filtering
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSafeBrowsingHash(t *testing.T) {
|
||||
// test hostnameToHashes()
|
||||
hashes := hostnameToHashes("1.2.3.sub.host.com")
|
||||
assert.Len(t, hashes, 3)
|
||||
_, ok := hashes[sha256.Sum256([]byte("3.sub.host.com"))]
|
||||
assert.True(t, ok)
|
||||
_, ok = hashes[sha256.Sum256([]byte("sub.host.com"))]
|
||||
assert.True(t, ok)
|
||||
_, ok = hashes[sha256.Sum256([]byte("host.com"))]
|
||||
assert.True(t, ok)
|
||||
_, ok = hashes[sha256.Sum256([]byte("com"))]
|
||||
assert.False(t, ok)
|
||||
|
||||
c := &sbCtx{
|
||||
svc: "SafeBrowsing",
|
||||
hashToHost: hashes,
|
||||
}
|
||||
|
||||
q := c.getQuestion()
|
||||
|
||||
assert.Contains(t, q, "7a1b.")
|
||||
assert.Contains(t, q, "af5a.")
|
||||
assert.Contains(t, q, "eb11.")
|
||||
assert.True(t, strings.HasSuffix(q, "sb.dns.adguard.com."))
|
||||
}
|
||||
|
||||
func TestSafeBrowsingCache(t *testing.T) {
|
||||
c := &sbCtx{
|
||||
svc: "SafeBrowsing",
|
||||
cacheTime: 100,
|
||||
}
|
||||
conf := cache.Config{}
|
||||
c.cache = cache.New(conf)
|
||||
|
||||
// store in cache hashes for "3.sub.host.com" and "host.com"
|
||||
// and empty data for hash-prefix for "sub.host.com"
|
||||
hash := sha256.Sum256([]byte("sub.host.com"))
|
||||
c.hashToHost = make(map[[32]byte]string)
|
||||
c.hashToHost[hash] = "sub.host.com"
|
||||
var hashesArray [][]byte
|
||||
hash4 := sha256.Sum256([]byte("3.sub.host.com"))
|
||||
hashesArray = append(hashesArray, hash4[:])
|
||||
hash2 := sha256.Sum256([]byte("host.com"))
|
||||
hashesArray = append(hashesArray, hash2[:])
|
||||
c.storeCache(hashesArray)
|
||||
|
||||
// match "3.sub.host.com" or "host.com" from cache
|
||||
c.hashToHost = make(map[[32]byte]string)
|
||||
hash = sha256.Sum256([]byte("3.sub.host.com"))
|
||||
c.hashToHost[hash] = "3.sub.host.com"
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
c.hashToHost[hash] = "sub.host.com"
|
||||
hash = sha256.Sum256([]byte("host.com"))
|
||||
c.hashToHost[hash] = "host.com"
|
||||
assert.Equal(t, 1, c.getCached())
|
||||
|
||||
// match "sub.host.com" from cache
|
||||
c.hashToHost = make(map[[32]byte]string)
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
c.hashToHost[hash] = "sub.host.com"
|
||||
assert.Equal(t, -1, c.getCached())
|
||||
|
||||
// Match "sub.host.com" from cache. Another hash for "host.example" is not
|
||||
// in the cache, so get data for it from the server.
|
||||
c.hashToHost = make(map[[32]byte]string)
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
c.hashToHost[hash] = "sub.host.com"
|
||||
hash = sha256.Sum256([]byte("host.example"))
|
||||
c.hashToHost[hash] = "host.example"
|
||||
assert.Empty(t, c.getCached())
|
||||
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
_, ok := c.hashToHost[hash]
|
||||
assert.False(t, ok)
|
||||
|
||||
hash = sha256.Sum256([]byte("host.example"))
|
||||
_, ok = c.hashToHost[hash]
|
||||
assert.True(t, ok)
|
||||
|
||||
c = &sbCtx{
|
||||
svc: "SafeBrowsing",
|
||||
cacheTime: 100,
|
||||
}
|
||||
conf = cache.Config{}
|
||||
c.cache = cache.New(conf)
|
||||
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
c.hashToHost = make(map[[32]byte]string)
|
||||
c.hashToHost[hash] = "sub.host.com"
|
||||
|
||||
c.cache.Set(hash[0:2], make([]byte, 32))
|
||||
assert.Empty(t, c.getCached())
|
||||
}
|
||||
|
||||
func TestSBPC_checkErrorUpstream(t *testing.T) {
|
||||
d, _ := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
ups := aghtest.NewErrorUpstream()
|
||||
d.SetSafeBrowsingUpstream(ups)
|
||||
d.SetParentalUpstream(ups)
|
||||
|
||||
setts := &Settings{
|
||||
ProtectionEnabled: true,
|
||||
SafeBrowsingEnabled: true,
|
||||
ParentalEnabled: true,
|
||||
}
|
||||
|
||||
_, err := d.checkSafeBrowsing("smthng.com", dns.TypeA, setts)
|
||||
assert.Error(t, err)
|
||||
|
||||
_, err = d.checkParental("smthng.com", dns.TypeA, setts)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestSBPC(t *testing.T) {
|
||||
d, _ := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
const hostname = "example.org"
|
||||
|
||||
setts := &Settings{
|
||||
ProtectionEnabled: true,
|
||||
SafeBrowsingEnabled: true,
|
||||
ParentalEnabled: true,
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
testCache cache.Cache
|
||||
testFunc func(host string, _ uint16, _ *Settings) (res Result, err error)
|
||||
name string
|
||||
block bool
|
||||
}{{
|
||||
testCache: d.safebrowsingCache,
|
||||
testFunc: d.checkSafeBrowsing,
|
||||
name: "sb_no_block",
|
||||
block: false,
|
||||
}, {
|
||||
testCache: d.safebrowsingCache,
|
||||
testFunc: d.checkSafeBrowsing,
|
||||
name: "sb_block",
|
||||
block: true,
|
||||
}, {
|
||||
testCache: d.parentalCache,
|
||||
testFunc: d.checkParental,
|
||||
name: "pc_no_block",
|
||||
block: false,
|
||||
}, {
|
||||
testCache: d.parentalCache,
|
||||
testFunc: d.checkParental,
|
||||
name: "pc_block",
|
||||
block: true,
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
// Prepare the upstream.
|
||||
ups := aghtest.NewBlockUpstream(hostname, tc.block)
|
||||
|
||||
var numReq int
|
||||
onExchange := ups.OnExchange
|
||||
ups.OnExchange = func(req *dns.Msg) (resp *dns.Msg, err error) {
|
||||
numReq++
|
||||
|
||||
return onExchange(req)
|
||||
}
|
||||
|
||||
d.SetSafeBrowsingUpstream(ups)
|
||||
d.SetParentalUpstream(ups)
|
||||
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Firstly, check the request blocking.
|
||||
hits := 0
|
||||
res, err := tc.testFunc(hostname, dns.TypeA, setts)
|
||||
require.NoError(t, err)
|
||||
|
||||
if tc.block {
|
||||
assert.True(t, res.IsFiltered)
|
||||
require.Len(t, res.Rules, 1)
|
||||
hits++
|
||||
} else {
|
||||
require.False(t, res.IsFiltered)
|
||||
}
|
||||
|
||||
// Check the cache state, check the response is now cached.
|
||||
assert.Equal(t, 1, tc.testCache.Stats().Count)
|
||||
assert.Equal(t, hits, tc.testCache.Stats().Hit)
|
||||
|
||||
// There was one request to an upstream.
|
||||
assert.Equal(t, 1, numReq)
|
||||
|
||||
// Now make the same request to check the cache was used.
|
||||
res, err = tc.testFunc(hostname, dns.TypeA, setts)
|
||||
require.NoError(t, err)
|
||||
|
||||
if tc.block {
|
||||
assert.True(t, res.IsFiltered)
|
||||
require.Len(t, res.Rules, 1)
|
||||
} else {
|
||||
require.False(t, res.IsFiltered)
|
||||
}
|
||||
|
||||
// Check the cache state, it should've been used.
|
||||
assert.Equal(t, 1, tc.testCache.Stats().Count)
|
||||
assert.Equal(t, hits+1, tc.testCache.Stats().Hit)
|
||||
|
||||
// Check that there were no additional requests.
|
||||
assert.Equal(t, 1, numReq)
|
||||
})
|
||||
|
||||
purgeCaches(d)
|
||||
}
|
||||
}
|
||||
|
|
@ -27,11 +27,13 @@ import (
|
|||
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/filtering/hashprefix"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/filtering/safesearch"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/querylog"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/stats"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/updater"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/version"
|
||||
"github.com/AdguardTeam/dnsproxy/upstream"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/netutil"
|
||||
|
|
@ -295,6 +297,59 @@ func setupConfig(opts options) (err error) {
|
|||
config.DNS.DnsfilterConf.UserRules = slices.Clone(config.UserRules)
|
||||
config.DNS.DnsfilterConf.HTTPClient = Context.client
|
||||
|
||||
const (
|
||||
dnsTimeout = 3 * time.Second
|
||||
|
||||
sbService = "safe browsing"
|
||||
defaultSafeBrowsingServer = `https://family.adguard-dns.com/dns-query`
|
||||
sbTXTSuffix = `sb.dns.adguard.com.`
|
||||
|
||||
pcService = "parental control"
|
||||
defaultParentalServer = `https://family.adguard-dns.com/dns-query`
|
||||
pcTXTSuffix = `pc.dns.adguard.com.`
|
||||
)
|
||||
|
||||
cacheTime := time.Duration(config.DNS.DnsfilterConf.CacheTime) * time.Minute
|
||||
|
||||
upsOpts := &upstream.Options{
|
||||
Timeout: dnsTimeout,
|
||||
ServerIPAddrs: []net.IP{
|
||||
{94, 140, 14, 15},
|
||||
{94, 140, 15, 16},
|
||||
net.ParseIP("2a10:50c0::bad1:ff"),
|
||||
net.ParseIP("2a10:50c0::bad2:ff"),
|
||||
},
|
||||
}
|
||||
|
||||
sbUps, err := upstream.AddressToUpstream(defaultSafeBrowsingServer, upsOpts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("converting safe browsing server: %w", err)
|
||||
}
|
||||
|
||||
safeBrowsing := hashprefix.New(&hashprefix.Config{
|
||||
Upstream: sbUps,
|
||||
ServiceName: sbService,
|
||||
TXTSuffix: sbTXTSuffix,
|
||||
CacheTime: cacheTime,
|
||||
CacheSize: config.DNS.DnsfilterConf.SafeBrowsingCacheSize,
|
||||
})
|
||||
|
||||
parUps, err := upstream.AddressToUpstream(defaultParentalServer, upsOpts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("converting parental server: %w", err)
|
||||
}
|
||||
|
||||
parentalControl := hashprefix.New(&hashprefix.Config{
|
||||
Upstream: parUps,
|
||||
ServiceName: pcService,
|
||||
TXTSuffix: pcTXTSuffix,
|
||||
CacheTime: cacheTime,
|
||||
CacheSize: config.DNS.DnsfilterConf.SafeBrowsingCacheSize,
|
||||
})
|
||||
|
||||
config.DNS.DnsfilterConf.SafeBrowsingChecker = safeBrowsing
|
||||
config.DNS.DnsfilterConf.ParentalControlChecker = parentalControl
|
||||
|
||||
config.DNS.DnsfilterConf.SafeSearchConf.CustomResolver = safeSearchResolver{}
|
||||
config.DNS.DnsfilterConf.SafeSearch, err = safesearch.NewDefault(
|
||||
config.DNS.DnsfilterConf.SafeSearchConf,
|
||||
|
|
|
|||
Loading…
Reference in New Issue