Fix #576 - Fix safesearch

This commit is contained in:
Aleksey Dmitrevskiy 2019-02-22 16:34:36 +03:00
parent a5b61459cc
commit 623c3bba09
3 changed files with 225 additions and 1 deletions

View File

@ -91,10 +91,11 @@ type LookupStats struct {
PendingMax int64 // maximum number of pending HTTP requests PendingMax int64 // maximum number of pending HTTP requests
} }
// Stats store LookupStats for both safebrowsing and parental // Stats store LookupStats for safebrowsing, parental and safesearch
type Stats struct { type Stats struct {
Safebrowsing LookupStats Safebrowsing LookupStats
Parental LookupStats Parental LookupStats
Safesearch LookupStats
} }
// Dnsfilter holds added rules and performs hostname matches against the rules // Dnsfilter holds added rules and performs hostname matches against the rules
@ -155,6 +156,7 @@ var (
stats Stats stats Stats
safebrowsingCache gcache.Cache safebrowsingCache gcache.Cache
parentalCache gcache.Cache parentalCache gcache.Cache
safeSearchCache gcache.Cache
) )
// Result holds state of hostname check // Result holds state of hostname check
@ -188,6 +190,19 @@ func (d *Dnsfilter) CheckHost(host string) (Result, error) {
return result, nil return result, nil
} }
// check safeSearch if no match
if d.SafeSearchEnabled {
result, err = d.checkSafeSearch(host)
if err != nil {
log.Printf("Failed to safesearch HTTP lookup, ignoring check: %v", err)
return Result{}, nil
}
if result.Reason.Matched() {
return result, nil
}
}
// check safebrowsing if no match // check safebrowsing if no match
if d.SafeBrowsingEnabled { if d.SafeBrowsingEnabled {
result, err = d.checkSafeBrowsing(host) result, err = d.checkSafeBrowsing(host)
@ -584,6 +599,64 @@ func hostnameToHashParam(host string, addslash bool) (string, map[string]bool) {
return hashparam.String(), hashes return hashparam.String(), hashes
} }
func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) {
if safeSearchCache == nil {
safeSearchCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
}
// Check cache. Return cached result if it was found
cachedValue, isFound, err := getCachedReason(safeSearchCache, host)
if isFound {
atomic.AddUint64(&stats.Safesearch.CacheHits, 1)
return cachedValue, nil
}
if err != nil {
return Result{}, err
}
safeHost, ok := d.SafeSearchDomain(host)
if !ok {
return Result{}, nil
}
res := Result {IsFiltered: true, Reason: FilteredSafeSearch}
if ip := net.ParseIP(safeHost); ip != nil {
res.IP = ip
err = safeSearchCache.Set(host, res)
if err != nil {
return Result{}, nil
}
return res, nil
}
addrs, err := net.LookupIP(safeHost)
if err != nil {
log.Tracef("SafeSearchDomain for %s was found but failed to lookup for %s cause %s", host, safeHost, err)
return Result{}, err
}
// The next bug may occurs: LookupIP returns DNS64 mapped ipv4 address with zero-prefix
for _, i := range addrs {
if ipv4 := i.To4(); ipv4 != nil && len(i) == net.IPv6len {
res.IP = ipv4
break
}
}
if res.IP == nil || len(res.IP) == 0 {
res.IP = addrs[0]
}
// Cache result
err = safeSearchCache.Set(host, res)
if err != nil {
return Result{}, nil
}
return res, nil
}
func (d *Dnsfilter) checkSafeBrowsing(host string) (Result, error) { func (d *Dnsfilter) checkSafeBrowsing(host string) (Result, error) {
// prevent recursion -- checking the host of safebrowsing server makes no sense // prevent recursion -- checking the host of safebrowsing server makes no sense
if host == d.safeBrowsingServer { if host == d.safeBrowsingServer {

View File

@ -4,6 +4,7 @@ import (
"archive/zip" "archive/zip"
"bytes" "bytes"
"io/ioutil" "io/ioutil"
"net"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"path" "path"
@ -607,6 +608,151 @@ func TestSafeBrowsingCustomServerFail(t *testing.T) {
d.checkMatchEmpty(t, "wmconvirus.narod.ru") d.checkMatchEmpty(t, "wmconvirus.narod.ru")
} }
func TestCheckHostSafeSearchYandex(t *testing.T) {
d := NewForTest()
defer d.Destroy()
// Enable safesearch
d.SafeSearchEnabled = true
// Slice of yandex domains
yandex := []string{"yAndeX.ru", "YANdex.COM", "yandex.ua", "yandex.by", "yandex.kz", "www.yandex.com"}
// Check host for each domain
for _, host := range yandex {
result, err := d.CheckHost(host)
if err != nil {
t.Errorf("SafeSearch doesn't work for yandex domain `%s` cause %s", host, err)
}
if result.IP.String() != "213.180.193.56" {
t.Errorf("SafeSearch doesn't work for yandex domain `%s`", host)
}
}
}
func TestCheckHostSafeSearchGoogle(t *testing.T) {
d := NewForTest()
defer d.Destroy()
// Enable safesearch
d.SafeSearchEnabled = true
// Slice of google domains
googleDomains := []string{"www.google.com", "www.google.im", "www.google.co.in", "www.google.iq", "www.google.is", "www.google.it", "www.google.je"}
// Check host for each domain
for _, host := range googleDomains {
result, err := d.CheckHost(host)
if err != nil {
t.Errorf("SafeSearch doesn't work for %s cause %s", host, err)
}
if result.IP == nil {
t.Errorf("SafeSearch doesn't work for %s", host)
}
}
}
func TestSafeSearchCacheYandex (t *testing.T) {
d := NewForTest()
defer d.Destroy()
domain := "yandex.ru"
// Check host with disabled safesearch
result, err := d.CheckHost(domain)
if result.IP != nil {
t.Fatalf("SafeSearch is not enabled but there is an answer for `%s` !", domain)
}
// Enable safesearch
d.SafeSearchEnabled = true
result, err = d.CheckHost(domain)
if err != nil {
t.Fatalf("CheckHost for safesearh domain %s failed cause %s", domain, err)
}
// Fir yandex we already know valid ip
if result.IP.String() != "213.180.193.56" {
t.Fatalf("Wrong IP for %s safesearch: %s", domain, result.IP.String())
}
// Check cache
cachedValue, isFound, err := getCachedReason(safeSearchCache, domain)
if err != nil {
t.Fatalf("An error occured during cache search for %s: %s", domain, err)
}
if !isFound {
t.Fatalf("Safesearch cache doesn't work for %s!", domain)
}
if cachedValue.IP.String() != "213.180.193.56" {
t.Fatalf("Wrong IP in cache for %s safesearch: %s", domain, cachedValue.IP.String())
}
}
func TestSafeSearchCacheGoogle (t *testing.T) {
d := NewForTest()
defer d.Destroy()
domain := "www.google.ru"
result, err := d.CheckHost(domain)
if result.IP != nil {
t.Fatalf("SafeSearch is not enabled but there is an answer!")
}
// Let's lookup for safesearch domain
safeDomain, ok := d.SafeSearchDomain(domain)
if !ok {
t.Fatalf("Failed to get safesearch domain for %s", domain)
}
ips, err := net.LookupIP(safeDomain)
if err != nil {
t.Fatalf("Failed to lookup for %s", safeDomain)
}
var ip net.IP
for _, i := range ips {
if len(i) == net.IPv6len && i.To4() != nil {
ip = i
}
}
if ip == nil || len(ip) == 0 {
ip = ips[0]
}
// Enable safesearch and check host
d.SafeSearchEnabled = true
result, err = d.CheckHost(domain)
if err != nil {
t.Fatalf("CheckHost for safesearh domain %s failed cause %s", domain, err)
}
if result.IP.String() != ip.String() {
t.Fatalf("Wrong IP for %s safesearch: %s", domain, result.IP.String())
}
// Check cache
cachedValue, isFound, err := getCachedReason(safeSearchCache, domain)
if err != nil {
t.Fatalf("An error occured during cache search for %s: %s", domain, err)
}
if !isFound {
t.Fatalf("Safesearch cache doesn't work for %s!", domain)
}
if cachedValue.IP.String() != ip.String() {
t.Fatalf("Wrong IP in cache for %s safesearch: %s", domain, cachedValue.IP.String())
}
}
func TestParentalControl(t *testing.T) { func TestParentalControl(t *testing.T) {
d := NewForTest() d := NewForTest()
defer d.Destroy() defer d.Destroy()

View File

@ -6,6 +6,11 @@ var safeSearchDomains = map[string]string{
"yandex.ua": "213.180.193.56", "yandex.ua": "213.180.193.56",
"yandex.by": "213.180.193.56", "yandex.by": "213.180.193.56",
"yandex.kz": "213.180.193.56", "yandex.kz": "213.180.193.56",
"www.yandex.com": "213.180.193.56",
"www.yandex.ru": "213.180.193.56",
"www.yandex.ua": "213.180.193.56",
"www.yandex.by": "213.180.193.56",
"www.yandex.kz": "213.180.193.56",
"www.bing.com": "strict.bing.com", "www.bing.com": "strict.bing.com",