Pull request: 2574 external tests vol.1

Merge in DNS/adguard-home from 2574-external-tests to master

Updates #2574.

Squashed commit of the following:

commit cb7017aa7fc1c81c014c1fa7e0972b06748aff29
Merge: 2a073431 eeeb0383
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Fri Jan 29 15:43:41 2021 +0300

    Merge branch 'master' into 2574-external-tests

commit 2a0734311c5a6ddd2d9c3f56a4494a20b4197955
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Fri Jan 29 15:16:40 2021 +0300

    dnsfilter: imp docs

commit 49aff871282e2739d27fcbceefdf1bd005c21174
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Fri Jan 29 14:10:56 2021 +0300

    dnsfilter: imp docs

commit 11be89a5378c0e451f1b88dc48a2e8827ba38358
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Jan 28 21:20:55 2021 +0300

    dnsfilter: fix go version trouble

commit 1ba4afd95bd739c18818ab0bea0ae70d59b880dc
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Jan 28 17:42:29 2021 +0300

    dnsfilter: imp tests

internal/dnsfilter/dnsfilter.go

@@ -2,6 +2,7 @@
 package dnsfilter
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -91,6 +92,12 @@ type filtersInitializerParams struct {
 	blockFilters []Filter
 }
 
+// Resolver is the interface for net.Resolver to simplify testing.
+type Resolver interface {
+	// TODO(e.burkov): Replace with LookupIP after upgrading go to v1.15.
+	LookupIPAddr(ctx context.Context, host string) (ips []net.IPAddr, err error)
+}
+
 // DNSFilter matches hostnames and DNS requests against filtering rules.
 type DNSFilter struct {
 	rulesStorage         *filterlist.RuleStorage
@@ -110,6 +117,11 @@ type DNSFilter struct {
 	// Channel for passing data to filters-initializer goroutine
 	filtersInitializerChan chan filtersInitializerParams
 	filtersInitializerLock sync.Mutex
+
+	// resolver only looks up the IP address of the host while safe search.
+	//
+	// TODO(e.burkov): Use upstream that configured in dnsforward instead.
+	resolver Resolver
 }
 
 // Filter represents a filter list
@@ -805,7 +817,9 @@ func New(c *Config, blockFilters []Filter) *DNSFilter {
 		}
 	}
 
-	d := new(DNSFilter)
+	d := &DNSFilter{
+		resolver: net.DefaultResolver,
+	}
 
 	err := d.initSecurityServices()
 	if err != nil {

internal/dnsfilter/dnsrewrite_test.go

@@ -45,7 +45,7 @@ func TestDNSFilter_CheckHostRules_dnsrewrite(t *testing.T) {
 @@||disable-all^$dnsrewrite
 `
 
-	f := NewForTest(nil, []Filter{{ID: 0, Data: []byte(text)}})
+	f := newForTest(nil, []Filter{{ID: 0, Data: []byte(text)}})
 	setts := &RequestFilteringSettings{
 		FilteringEnabled: true,
 	}

internal/dnsfilter/rewrites_test.go

@@ -9,7 +9,8 @@ import (
 )
 
 func TestRewrites(t *testing.T) {
-	d := DNSFilter{}
+	d := newForTest(nil, nil)
+	t.Cleanup(d.Close)
 	// CNAME, A, AAAA
 	d.Rewrites = []RewriteEntry{
 		{"somecname", "somehost.com", 0, nil},
@@ -104,7 +105,8 @@ func TestRewrites(t *testing.T) {
 }
 
 func TestRewritesLevels(t *testing.T) {
-	d := DNSFilter{}
+	d := newForTest(nil, nil)
+	t.Cleanup(d.Close)
 	// exact host, wildcard L2, wildcard L3
 	d.Rewrites = []RewriteEntry{
 		{"host.com", "1.1.1.1", 0, nil},
@@ -133,7 +135,8 @@ func TestRewritesLevels(t *testing.T) {
 }
 
 func TestRewritesExceptionCNAME(t *testing.T) {
-	d := DNSFilter{}
+	d := newForTest(nil, nil)
+	t.Cleanup(d.Close)
 	// wildcard; exception for a sub-domain
 	d.Rewrites = []RewriteEntry{
 		{"*.host.com", "2.2.2.2", 0, nil},
@@ -153,7 +156,8 @@ func TestRewritesExceptionCNAME(t *testing.T) {
 }
 
 func TestRewritesExceptionWC(t *testing.T) {
-	d := DNSFilter{}
+	d := newForTest(nil, nil)
+	t.Cleanup(d.Close)
 	// wildcard; exception for a sub-wildcard
 	d.Rewrites = []RewriteEntry{
 		{"*.host.com", "2.2.2.2", 0, nil},
@@ -173,7 +177,8 @@ func TestRewritesExceptionWC(t *testing.T) {
 }
 
 func TestRewritesExceptionIP(t *testing.T) {
-	d := DNSFilter{}
+	d := newForTest(nil, nil)
+	t.Cleanup(d.Close)
 	// exception for AAAA record
 	d.Rewrites = []RewriteEntry{
 		{"host.com", "1.2.3.4", 0, nil},

internal/dnsfilter/safebrowsing_test.go

@@ -4,6 +4,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"strings"
+	"sync"
 	"testing"
 
 	"github.com/AdguardTeam/AdGuardHome/internal/agherr"
@@ -121,8 +122,8 @@ func (teu *testErrUpstream) Address() string {
 }
 
 func TestSBPC_checkErrorUpstream(t *testing.T) {
-	d := NewForTest(&Config{SafeBrowsingEnabled: true}, nil)
-	defer d.Close()
+	d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
+	t.Cleanup(d.Close)
 
 	ups := &testErrUpstream{}
 
@@ -142,11 +143,14 @@ type testSbUpstream struct {
 	hostname      string
 	block         bool
 	requestsCount int
+	counterLock   sync.RWMutex
 }
 
 // Exchange returns a message depending on the upstream settings (hostname, block)
 func (u *testSbUpstream) Exchange(r *dns.Msg) (*dns.Msg, error) {
+	u.counterLock.Lock()
 	u.requestsCount++
+	u.counterLock.Unlock()
 
 	hash := sha256.Sum256([]byte(u.hostname))
 	prefix := hash[0:2]
@@ -175,8 +179,8 @@ func (u *testSbUpstream) Address() string {
 }
 
 func TestSBPC_sbValidResponse(t *testing.T) {
-	d := NewForTest(&Config{SafeBrowsingEnabled: true}, nil)
-	defer d.Close()
+	d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
+	t.Cleanup(d.Close)
 
 	ups := &testSbUpstream{}
 	d.safeBrowsingUpstream = ups
@@ -213,8 +217,8 @@ func TestSBPC_sbValidResponse(t *testing.T) {
 }
 
 func TestSBPC_pcBlockedResponse(t *testing.T) {
-	d := NewForTest(&Config{SafeBrowsingEnabled: true}, nil)
-	defer d.Close()
+	d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
+	t.Cleanup(d.Close)
 
 	ups := &testSbUpstream{}
 	d.safeBrowsingUpstream = ups

internal/dnsfilter/safesearch.go

@@ -2,6 +2,7 @@ package dnsfilter
 
 import (
 	"bytes"
+	"context"
 	"encoding/binary"
 	"encoding/gob"
 	"encoding/json"
@@ -101,15 +102,14 @@ func (d *DNSFilter) checkSafeSearch(host string) (Result, error) {
 		return res, nil
 	}
 
-	// TODO this address should be resolved with upstream that was configured in dnsforward
-	ips, err := net.LookupIP(safeHost)
+	ipAddrs, err := d.resolver.LookupIPAddr(context.Background(), safeHost)
 	if err != nil {
 		log.Tracef("SafeSearchDomain for %s was found but failed to lookup for %s cause %s", host, safeHost, err)
 		return Result{}, err
 	}
 
-	for _, ip := range ips {
-		if ipv4 := ip.To4(); ipv4 != nil {
+	for _, ipAddr := range ipAddrs {
+		if ipv4 := ipAddr.IP.To4(); ipv4 != nil {
 			res.Rules[0].IP = ipv4
 
 			l := d.setCacheResult(gctx.safeSearchCache, host, res)