From 623c3bba09a8600178eee1ae291ff6e0c287aa4e Mon Sep 17 00:00:00 2001 From: Aleksey Dmitrevskiy Date: Fri, 22 Feb 2019 16:34:36 +0300 Subject: [PATCH] Fix #576 - Fix safesearch --- dnsfilter/dnsfilter.go | 75 +++++++++++++++++- dnsfilter/dnsfilter_test.go | 146 ++++++++++++++++++++++++++++++++++++ dnsfilter/safesearch.go | 5 ++ 3 files changed, 225 insertions(+), 1 deletion(-) diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go index 90bd61ea..5f0ad3ab 100644 --- a/dnsfilter/dnsfilter.go +++ b/dnsfilter/dnsfilter.go @@ -91,10 +91,11 @@ type LookupStats struct { PendingMax int64 // maximum number of pending HTTP requests } -// Stats store LookupStats for both safebrowsing and parental +// Stats store LookupStats for safebrowsing, parental and safesearch type Stats struct { Safebrowsing LookupStats Parental LookupStats + Safesearch LookupStats } // Dnsfilter holds added rules and performs hostname matches against the rules @@ -155,6 +156,7 @@ var ( stats Stats safebrowsingCache gcache.Cache parentalCache gcache.Cache + safeSearchCache gcache.Cache ) // Result holds state of hostname check @@ -188,6 +190,19 @@ func (d *Dnsfilter) CheckHost(host string) (Result, error) { return result, nil } + // check safeSearch if no match + if d.SafeSearchEnabled { + result, err = d.checkSafeSearch(host) + if err != nil { + log.Printf("Failed to safesearch HTTP lookup, ignoring check: %v", err) + return Result{}, nil + } + + if result.Reason.Matched() { + return result, nil + } + } + // check safebrowsing if no match if d.SafeBrowsingEnabled { result, err = d.checkSafeBrowsing(host) @@ -584,6 +599,64 @@ func hostnameToHashParam(host string, addslash bool) (string, map[string]bool) { return hashparam.String(), hashes } +func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) { + if safeSearchCache == nil { + safeSearchCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build() + } + + // Check cache. Return cached result if it was found + cachedValue, isFound, err := getCachedReason(safeSearchCache, host) + if isFound { + atomic.AddUint64(&stats.Safesearch.CacheHits, 1) + return cachedValue, nil + } + + if err != nil { + return Result{}, err + } + + safeHost, ok := d.SafeSearchDomain(host) + if !ok { + return Result{}, nil + } + + res := Result {IsFiltered: true, Reason: FilteredSafeSearch} + if ip := net.ParseIP(safeHost); ip != nil { + res.IP = ip + err = safeSearchCache.Set(host, res) + if err != nil { + return Result{}, nil + } + + return res, nil + } + + addrs, err := net.LookupIP(safeHost) + if err != nil { + log.Tracef("SafeSearchDomain for %s was found but failed to lookup for %s cause %s", host, safeHost, err) + return Result{}, err + } + + // The next bug may occurs: LookupIP returns DNS64 mapped ipv4 address with zero-prefix + for _, i := range addrs { + if ipv4 := i.To4(); ipv4 != nil && len(i) == net.IPv6len { + res.IP = ipv4 + break + } + } + + if res.IP == nil || len(res.IP) == 0 { + res.IP = addrs[0] + } + + // Cache result + err = safeSearchCache.Set(host, res) + if err != nil { + return Result{}, nil + } + return res, nil +} + func (d *Dnsfilter) checkSafeBrowsing(host string) (Result, error) { // prevent recursion -- checking the host of safebrowsing server makes no sense if host == d.safeBrowsingServer { diff --git a/dnsfilter/dnsfilter_test.go b/dnsfilter/dnsfilter_test.go index 9692a60d..50fd0bab 100644 --- a/dnsfilter/dnsfilter_test.go +++ b/dnsfilter/dnsfilter_test.go @@ -4,6 +4,7 @@ import ( "archive/zip" "bytes" "io/ioutil" + "net" "net/http" "net/http/httptest" "path" @@ -607,6 +608,151 @@ func TestSafeBrowsingCustomServerFail(t *testing.T) { d.checkMatchEmpty(t, "wmconvirus.narod.ru") } +func TestCheckHostSafeSearchYandex(t *testing.T) { + d := NewForTest() + defer d.Destroy() + + // Enable safesearch + d.SafeSearchEnabled = true + + // Slice of yandex domains + yandex := []string{"yAndeX.ru", "YANdex.COM", "yandex.ua", "yandex.by", "yandex.kz", "www.yandex.com"} + + // Check host for each domain + for _, host := range yandex { + result, err := d.CheckHost(host) + if err != nil { + t.Errorf("SafeSearch doesn't work for yandex domain `%s` cause %s", host, err) + } + + if result.IP.String() != "213.180.193.56" { + t.Errorf("SafeSearch doesn't work for yandex domain `%s`", host) + } + } +} + +func TestCheckHostSafeSearchGoogle(t *testing.T) { + d := NewForTest() + defer d.Destroy() + + // Enable safesearch + d.SafeSearchEnabled = true + + // Slice of google domains + googleDomains := []string{"www.google.com", "www.google.im", "www.google.co.in", "www.google.iq", "www.google.is", "www.google.it", "www.google.je"} + + // Check host for each domain + for _, host := range googleDomains { + result, err := d.CheckHost(host) + if err != nil { + t.Errorf("SafeSearch doesn't work for %s cause %s", host, err) + } + + if result.IP == nil { + t.Errorf("SafeSearch doesn't work for %s", host) + } + } +} + +func TestSafeSearchCacheYandex (t *testing.T) { + d := NewForTest() + defer d.Destroy() + domain := "yandex.ru" + + // Check host with disabled safesearch + result, err := d.CheckHost(domain) + if result.IP != nil { + t.Fatalf("SafeSearch is not enabled but there is an answer for `%s` !", domain) + } + + // Enable safesearch + d.SafeSearchEnabled = true + result, err = d.CheckHost(domain) + if err != nil { + t.Fatalf("CheckHost for safesearh domain %s failed cause %s", domain, err) + } + + // Fir yandex we already know valid ip + if result.IP.String() != "213.180.193.56" { + t.Fatalf("Wrong IP for %s safesearch: %s", domain, result.IP.String()) + } + + // Check cache + cachedValue, isFound, err := getCachedReason(safeSearchCache, domain) + + if err != nil { + t.Fatalf("An error occured during cache search for %s: %s", domain, err) + } + + if !isFound { + t.Fatalf("Safesearch cache doesn't work for %s!", domain) + } + + if cachedValue.IP.String() != "213.180.193.56" { + t.Fatalf("Wrong IP in cache for %s safesearch: %s", domain, cachedValue.IP.String()) + } +} + +func TestSafeSearchCacheGoogle (t *testing.T) { + d := NewForTest() + defer d.Destroy() + domain := "www.google.ru" + result, err := d.CheckHost(domain) + if result.IP != nil { + t.Fatalf("SafeSearch is not enabled but there is an answer!") + } + + + // Let's lookup for safesearch domain + safeDomain, ok := d.SafeSearchDomain(domain) + if !ok { + t.Fatalf("Failed to get safesearch domain for %s", domain) + } + + ips, err := net.LookupIP(safeDomain) + if err != nil { + t.Fatalf("Failed to lookup for %s", safeDomain) + } + + var ip net.IP + for _, i := range ips { + if len(i) == net.IPv6len && i.To4() != nil { + ip = i + } + } + + if ip == nil || len(ip) == 0 { + ip = ips[0] + } + + // Enable safesearch and check host + d.SafeSearchEnabled = true + + result, err = d.CheckHost(domain) + if err != nil { + t.Fatalf("CheckHost for safesearh domain %s failed cause %s", domain, err) + } + + if result.IP.String() != ip.String() { + t.Fatalf("Wrong IP for %s safesearch: %s", domain, result.IP.String()) + } + + // Check cache + cachedValue, isFound, err := getCachedReason(safeSearchCache, domain) + + if err != nil { + t.Fatalf("An error occured during cache search for %s: %s", domain, err) + } + + if !isFound { + t.Fatalf("Safesearch cache doesn't work for %s!", domain) + } + + if cachedValue.IP.String() != ip.String() { + t.Fatalf("Wrong IP in cache for %s safesearch: %s", domain, cachedValue.IP.String()) + } +} + func TestParentalControl(t *testing.T) { d := NewForTest() defer d.Destroy() diff --git a/dnsfilter/safesearch.go b/dnsfilter/safesearch.go index 902c0f09..c4834b2f 100644 --- a/dnsfilter/safesearch.go +++ b/dnsfilter/safesearch.go @@ -6,6 +6,11 @@ var safeSearchDomains = map[string]string{ "yandex.ua": "213.180.193.56", "yandex.by": "213.180.193.56", "yandex.kz": "213.180.193.56", + "www.yandex.com": "213.180.193.56", + "www.yandex.ru": "213.180.193.56", + "www.yandex.ua": "213.180.193.56", + "www.yandex.by": "213.180.193.56", + "www.yandex.kz": "213.180.193.56", "www.bing.com": "strict.bing.com",