mirrors
/
AdGuardHome
mirror of https://github.com/AdguardTeam/AdGuardHome.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge: dnsfilter: use urlfilter package #714 

* commit '096a95998749b673bc9be638bc9c8f6f0d13be41':
  * dnsforward: use new dnsfilter interface
  * dnsfilter: adapt tests to new interface
  * dnsfilter: use urlfilter package
  * dnsfilter: remove code for filtering rules
  * dns: rename dnsfilter.Filter.Rule -> dnsfilter.Filter.Data
  * dnsforward: use separate ServerConfig object
  * use urlfilter
This commit is contained in:
Simon Zolin 2019-05-20 11:00:45 +03:00
parent 7bb40bca0f 096a959987
commit e3ee7a0c3e
10 changed files with 223 additions and 1398 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dns.go
View File

@ -34,13 +34,13 @@ func generateServerConfig() dnsforward.ServerConfig {
filters := []dnsfilter.Filter{} filters := []dnsfilter.Filter{}
userFilter := userFilter() userFilter := userFilter()
filters = append(filters, dnsfilter.Filter{ filters = append(filters, dnsfilter.Filter{
ID: userFilter.ID, ID: userFilter.ID,
Rules: userFilter.Rules, Data: userFilter.Data,
}) })
for _, filter := range config.Filters { for _, filter := range config.Filters {
filters = append(filters, dnsfilter.Filter{ filters = append(filters, dnsfilter.Filter{
ID: filter.ID, ID: filter.ID,
Rules: filter.Rules, Data: filter.Data,
}) })
} }

536
dnsfilter/dnsfilter.go
View File

@ -11,14 +11,13 @@ import (
"io/ioutil" "io/ioutil"
"net" "net"
"net/http" "net/http"
"regexp"
"strings" "strings"
"sync"
"sync/atomic" "sync/atomic"
"time" "time"
"github.com/AdguardTeam/dnsproxy/upstream" "github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/urlfilter"
"github.com/bluele/gcache" "github.com/bluele/gcache"
"golang.org/x/net/publicsuffix" "golang.org/x/net/publicsuffix"
) )
@ -48,11 +47,12 @@ const enableDelayedCompilation = true // flag for debugging, must be true in pro
// Config allows you to configure DNS filtering with New() or just change variables directly. // Config allows you to configure DNS filtering with New() or just change variables directly.
type Config struct { type Config struct {
ParentalSensitivity int `yaml:"parental_sensitivity"` // must be either 3, 10, 13 or 17 FilteringTempFilename string `yaml:"filtering_temp_filename"` // temporary file for storing unused filtering rules
ParentalEnabled bool `yaml:"parental_enabled"` ParentalSensitivity int `yaml:"parental_sensitivity"` // must be either 3, 10, 13 or 17
SafeSearchEnabled bool `yaml:"safesearch_enabled"` ParentalEnabled bool `yaml:"parental_enabled"`
SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"` SafeSearchEnabled bool `yaml:"safesearch_enabled"`
ResolverAddress string // DNS server address SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"`
ResolverAddress string // DNS server address
} }
type privateConfig struct { type privateConfig struct {
@ -60,33 +60,6 @@ type privateConfig struct {
safeBrowsingServer string // access via methods safeBrowsingServer string // access via methods
} }
type rule struct {
text string // text without @@ decorators or $ options
shortcut string // for speeding up lookup
originalText string // original text for reporting back to applications
ip net.IP // IP address (for the case when we're matching a hosts file)
// options
options []string // optional options after $
// parsed options
apps []string
isWhitelist bool
isImportant bool
// user-supplied data
listID int64
// suffix matching
isSuffix bool
suffix string
// compiled regexp
compiled *regexp.Regexp
sync.RWMutex
}
// LookupStats store stats collected during safebrowsing or parental checks // LookupStats store stats collected during safebrowsing or parental checks
type LookupStats struct { type LookupStats struct {
Requests uint64 // number of HTTP requests that were sent Requests uint64 // number of HTTP requests that were sent
@ -104,13 +77,8 @@ type Stats struct {
// Dnsfilter holds added rules and performs hostname matches against the rules // Dnsfilter holds added rules and performs hostname matches against the rules
type Dnsfilter struct { type Dnsfilter struct {
storage map[string]bool // rule storage, not used for matching, just for filtering out duplicates rulesStorage *urlfilter.RulesStorage
storageMutex sync.RWMutex filteringEngine *urlfilter.DNSEngine
// rules are checked against these lists in the order defined here
important *rulesTable // more important than whitelist and is checked first
whiteList *rulesTable // more important than blacklist
blackList *rulesTable
// HTTP lookups for safebrowsing and parental // HTTP lookups for safebrowsing and parental
client http.Client // handle for http client -- single instance as recommended by docs client http.Client // handle for http client -- single instance as recommended by docs
@ -122,8 +90,8 @@ type Dnsfilter struct {
// Filter represents a filter list // Filter represents a filter list
type Filter struct { type Filter struct {
ID int64 `json:"id"` // auto-assigned when filter is added (see nextFilterID), json by default keeps ID uppercase but we need lowercase ID int64 `json:"id"` // auto-assigned when filter is added (see nextFilterID), json by default keeps ID uppercase but we need lowercase
Rules []string `json:"-" yaml:"-"` // not in yaml or json Data []byte `json:"-" yaml:"-"` // List of rules divided by '\n'
} }
//go:generate stringer -type=Reason //go:generate stringer -type=Reason
@ -242,308 +210,6 @@ func (d *Dnsfilter) CheckHost(host string) (Result, error) {
return Result{}, nil return Result{}, nil
} }
//
// rules table
//
type rulesTable struct {
rulesByHost map[string]*rule
rulesByShortcut map[string][]*rule
rulesLeftovers []*rule
sync.RWMutex
}
func newRulesTable() *rulesTable {
return &rulesTable{
rulesByHost: make(map[string]*rule),
rulesByShortcut: make(map[string][]*rule),
rulesLeftovers: make([]*rule, 0),
}
}
func (r *rulesTable) Add(rule *rule) {
r.Lock()
if rule.ip != nil {
// Hosts syntax
r.rulesByHost[rule.text] = rule
} else if len(rule.shortcut) == shortcutLength && enableFastLookup {
// Adblock syntax with a shortcut
r.rulesByShortcut[rule.shortcut] = append(r.rulesByShortcut[rule.shortcut], rule)
} else {
// Adblock syntax -- too short to have a shortcut
r.rulesLeftovers = append(r.rulesLeftovers, rule)
}
r.Unlock()
}
func (r *rulesTable) matchByHost(host string) (Result, error) {
// First: examine the hosts-syntax rules
res, err := r.searchByHost(host)
if err != nil {
return res, err
}
if res.Reason.Matched() {
return res, nil
}
// Second: examine the adblock-syntax rules with shortcuts
res, err = r.searchShortcuts(host)
if err != nil {
return res, err
}
if res.Reason.Matched() {
return res, nil
}
// Third: examine the others
res, err = r.searchLeftovers(host)
if err != nil {
return res, err
}
if res.Reason.Matched() {
return res, nil
}
return Result{}, nil
}
func (r *rulesTable) searchByHost(host string) (Result, error) {
rule, ok := r.rulesByHost[host]
if ok {
return rule.match(host)
}
return Result{}, nil
}
func (r *rulesTable) searchShortcuts(host string) (Result, error) {
// check in shortcuts first
for i := 0; i < len(host); i++ {
shortcut := host[i:]
if len(shortcut) > shortcutLength {
shortcut = shortcut[:shortcutLength]
}
if len(shortcut) != shortcutLength {
continue
}
rules, ok := r.rulesByShortcut[shortcut]
if !ok {
continue
}
for _, rule := range rules {
res, err := rule.match(host)
// error? stop search
if err != nil {
return res, err
}
// matched? stop search
if res.Reason.Matched() {
return res, err
}
// continue otherwise
}
}
return Result{}, nil
}
func (r *rulesTable) searchLeftovers(host string) (Result, error) {
for _, rule := range r.rulesLeftovers {
res, err := rule.match(host)
// error? stop search
if err != nil {
return res, err
}
// matched? stop search
if res.Reason.Matched() {
return res, err
}
// continue otherwise
}
return Result{}, nil
}
func findOptionIndex(text string) int {
for i, r := range text {
// ignore non-$
if r != '$' {
continue
}
// ignore `\$`
if i > 0 && text[i-1] == '\\' {
continue
}
// ignore `$/`
if i > len(text) && text[i+1] == '/' {
continue
}
return i + 1
}
return -1
}
func (rule *rule) extractOptions() error {
optIndex := findOptionIndex(rule.text)
if optIndex == 0 { // starts with $
return ErrInvalidSyntax
}
if optIndex == len(rule.text) { // ends with $
return ErrInvalidSyntax
}
if optIndex < 0 {
return nil
}
optionsStr := rule.text[optIndex:]
rule.text = rule.text[:optIndex-1] // remove options from text
begin := 0
i := 0
for i = 0; i < len(optionsStr); i++ {
switch optionsStr[i] {
case ',':
if i > 0 {
// it might be escaped, if so, ignore
if optionsStr[i-1] == '\\' {
break // from switch, not for loop
}
}
rule.options = append(rule.options, optionsStr[begin:i])
begin = i + 1
}
}
if begin != i {
// there's still an option remaining
rule.options = append(rule.options, optionsStr[begin:])
}
return nil
}
func (rule *rule) parseOptions() error {
err := rule.extractOptions()
if err != nil {
return err
}
for _, option := range rule.options {
switch {
case option == "important":
rule.isImportant = true
case strings.HasPrefix(option, "app="):
option = strings.TrimPrefix(option, "app=")
rule.apps = strings.Split(option, "|")
default:
return ErrInvalidSyntax
}
}
return nil
}
func (rule *rule) extractShortcut() {
// regex rules have no shortcuts
if rule.text[0] == '/' && rule.text[len(rule.text)-1] == '/' {
return
}
fields := strings.FieldsFunc(rule.text, func(r rune) bool {
switch r {
case '*', '^', '|':
return true
}
return false
})
longestField := ""
for _, field := range fields {
if len(field) > len(longestField) {
longestField = field
}
}
if len(longestField) > shortcutLength {
longestField = longestField[:shortcutLength]
}
rule.shortcut = strings.ToLower(longestField)
}
func (rule *rule) compile() error {
rule.RLock()
isCompiled := rule.isSuffix || rule.compiled != nil
rule.RUnlock()
if isCompiled {
return nil
}
isSuffix, suffix := getSuffix(rule.text)
if isSuffix {
rule.Lock()
rule.isSuffix = isSuffix
rule.suffix = suffix
rule.Unlock()
return nil
}
expr, err := ruleToRegexp(rule.text)
if err != nil {
return err
}
compiled, err := regexp.Compile(expr)
if err != nil {
return err
}
rule.Lock()
rule.compiled = compiled
rule.Unlock()
return nil
}
// Checks if the rule matches the specified host and returns a corresponding Result object
func (rule *rule) match(host string) (Result, error) {
res := Result{}
if rule.ip != nil && rule.text == host {
// This is a hosts-syntax rule -- just check that the hostname matches and return the result
return Result{
IsFiltered: true,
Reason: FilteredBlackList,
Rule: rule.originalText,
IP: rule.ip,
FilterID: rule.listID,
}, nil
}
err := rule.compile()
if err != nil {
return res, err
}
rule.RLock()
matched := false
if rule.isSuffix {
if host == rule.suffix {
matched = true
} else if strings.HasSuffix(host, "."+rule.suffix) {
matched = true
}
} else {
matched = rule.compiled.MatchString(host)
}
rule.RUnlock()
if matched {
res.Reason = FilteredBlackList
res.IsFiltered = true
res.FilterID = rule.listID
res.Rule = rule.originalText
if rule.isWhitelist {
res.Reason = NotFilteredWhiteList
res.IsFiltered = false
}
}
return res, nil
}
func getCachedReason(cache gcache.Cache, host string) (result Result, isFound bool, err error) { func getCachedReason(cache gcache.Cache, host string) (result Result, isFound bool, err error) {
isFound = false // not found yet isFound = false // not found yet
@ -838,135 +504,59 @@ func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gc
// Adding rule and matching against the rules // Adding rule and matching against the rules
// //
// AddRules is a convinience function to add an array of filters in one call // Initialize urlfilter objects
func (d *Dnsfilter) AddRules(filters []Filter) error { func (d *Dnsfilter) initFiltering(filters map[int]string) error {
for _, f := range filters { var err error
for _, rule := range f.Rules { d.rulesStorage, err = urlfilter.NewRuleStorage(d.FilteringTempFilename)
err := d.AddRule(rule, f.ID)
if err == ErrAlreadyExists || err == ErrInvalidSyntax {
continue
}
if err != nil {
log.Printf("Cannot add rule %s: %s", rule, err)
// Just ignore invalid rules
continue
}
}
}
return nil
}
// AddRule adds a rule, checking if it is a valid rule first and if it wasn't added already
func (d *Dnsfilter) AddRule(input string, filterListID int64) error {
input = strings.TrimSpace(input)
d.storageMutex.RLock()
_, exists := d.storage[input]
d.storageMutex.RUnlock()
if exists {
// already added
return ErrAlreadyExists
}
if !isValidRule(input) {
return ErrInvalidSyntax
}
// First, check if this is a hosts-syntax rule
if d.parseEtcHosts(input, filterListID) {
// This is a valid hosts-syntax rule, no need for further parsing
return nil
}
// Start parsing the rule
r := rule{
text: input, // will be modified
originalText: input,
listID: filterListID,
}
// Mark rule as whitelist if it starts with @@
if strings.HasPrefix(r.text, "@@") {
r.isWhitelist = true
r.text = r.text[2:]
}
err := r.parseOptions()
if err != nil { if err != nil {
return err return err
} }
r.extractShortcut() d.filteringEngine = urlfilter.NewDNSEngine(filters, d.rulesStorage)
if !enableDelayedCompilation {
err := r.compile()
if err != nil {
return err
}
}
destination := d.blackList
if r.isImportant {
destination = d.important
} else if r.isWhitelist {
destination = d.whiteList
}
d.storageMutex.Lock()
d.storage[input] = true
d.storageMutex.Unlock()
destination.Add(&r)
return nil return nil
} }
// Parses the hosts-syntax rules. Returns false if the input string is not of hosts-syntax.
func (d *Dnsfilter) parseEtcHosts(input string, filterListID int64) bool {
// Strip the trailing comment
ruleText := input
if pos := strings.IndexByte(ruleText, '#'); pos != -1 {
ruleText = ruleText[0:pos]
}
fields := strings.Fields(ruleText)
if len(fields) < 2 {
return false
}
addr := net.ParseIP(fields[0])
if addr == nil {
return false
}
d.storageMutex.Lock()
d.storage[input] = true
d.storageMutex.Unlock()
for _, host := range fields[1:] {
r := rule{
text: host,
originalText: input,
listID: filterListID,
ip: addr,
}
d.blackList.Add(&r)
}
return true
}
// matchHost is a low-level way to check only if hostname is filtered by rules, skipping expensive safebrowsing and parental lookups // matchHost is a low-level way to check only if hostname is filtered by rules, skipping expensive safebrowsing and parental lookups
func (d *Dnsfilter) matchHost(host string) (Result, error) { func (d *Dnsfilter) matchHost(host string) (Result, error) {
lists := []*rulesTable{ if d.filteringEngine == nil {
d.important, return Result{}, nil
d.whiteList,
d.blackList,
} }
for _, table := range lists { rules, ok := d.filteringEngine.Match(host)
res, err := table.matchByHost(host) if !ok {
if err != nil { return Result{}, nil
return res, err }
}
if res.Reason.Matched() { for _, rule := range rules {
log.Tracef("Found rule for host '%s': '%s' list_id: %d",
host, rule.Text(), rule.GetFilterListID())
res := Result{}
res.Reason = FilteredBlackList
res.IsFiltered = true
res.FilterID = int64(rule.GetFilterListID())
res.Rule = rule.Text()
if netRule, ok := rule.(*urlfilter.NetworkRule); ok {
if netRule.Whitelist {
res.Reason = NotFilteredWhiteList
res.IsFiltered = false
}
return res, nil return res, nil
} else if hostRule, ok := rule.(*urlfilter.HostRule); ok {
res.IP = hostRule.IP
return res, nil
} else {
log.Tracef("Rule type is unsupported: '%s' list_id: %d",
rule.Text(), rule.GetFilterListID())
} }
} }
return Result{}, nil return Result{}, nil
} }
@ -1058,14 +648,9 @@ func (d *Dnsfilter) createCustomDialContext(resolverAddr string) dialFunctionTyp
} }
// New creates properly initialized DNS Filter that is ready to be used // New creates properly initialized DNS Filter that is ready to be used
func New(c *Config) *Dnsfilter { func New(c *Config, filters map[int]string) *Dnsfilter {
d := new(Dnsfilter) d := new(Dnsfilter)
d.storage = make(map[string]bool)
d.important = newRulesTable()
d.whiteList = newRulesTable()
d.blackList = newRulesTable()
// Customize the Transport to have larger connection pool, // Customize the Transport to have larger connection pool,
// We are not (re)using http.DefaultTransport because of race conditions found by tests // We are not (re)using http.DefaultTransport because of race conditions found by tests
d.transport = &http.Transport{ d.transport = &http.Transport{
@ -1090,6 +675,15 @@ func New(c *Config) *Dnsfilter {
d.Config = *c d.Config = *c
} }
if filters != nil {
err := d.initFiltering(filters)
if err != nil {
log.Error("Can't initialize filtering subsystem: %s", err)
d.Destroy()
return nil
}
}
return d return d
} }
@ -1099,6 +693,11 @@ func (d *Dnsfilter) Destroy() {
if d != nil && d.transport != nil { if d != nil && d.transport != nil {
d.transport.CloseIdleConnections() d.transport.CloseIdleConnections()
} }
if d.rulesStorage != nil {
d.rulesStorage.Close()
d.rulesStorage = nil
}
} }
// //
@ -1141,8 +740,3 @@ func (d *Dnsfilter) SafeSearchDomain(host string) (string, bool) {
func (d *Dnsfilter) GetStats() Stats { func (d *Dnsfilter) GetStats() Stats {
return stats return stats
} }
// Count returns number of rules added to filter
func (d *Dnsfilter) Count() int {
return len(d.storage)
}

801
dnsfilter/dnsfilter_test.go
View File

@ -1,278 +1,51 @@
package dnsfilter package dnsfilter
import ( import (
"archive/zip" "fmt"
"bytes"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"path" "path"
"runtime/pprof" "runtime"
"strings"
"testing" "testing"
"time" "time"
"bufio"
"fmt"
"os"
"runtime"
"github.com/AdguardTeam/golibs/log"
"github.com/shirou/gopsutil/process"
"go.uber.org/goleak"
) )
// first in file because it must be run first // HELPERS
func TestLotsOfRulesMemoryUsage(t *testing.T) { // SAFE BROWSING
start := getRSS() // SAFE SEARCH
log.Tracef("RSS before loading rules - %d kB\n", start/1024) // PARENTAL
dumpMemProfile("tests/" + _Func() + "1.pprof") // FILTERING
// BENCHMARKS
d := NewForTest() // HELPERS
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
t.Error(err)
}
afterLoad := getRSS() func purgeCaches() {
log.Tracef("RSS after loading rules - %d kB (%d kB diff)\n", afterLoad/1024, (afterLoad-start)/1024) if safebrowsingCache != nil {
dumpMemProfile("tests/" + _Func() + "2.pprof") safebrowsingCache.Purge()
tests := []struct {
host string
match bool
}{
{"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com", false},
{"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net", true},
} }
for _, testcase := range tests { if parentalCache != nil {
ret, err := d.CheckHost(testcase.host) parentalCache.Purge()
if err != nil {
t.Errorf("Error while matching host %s: %s", testcase.host, err)
}
if !ret.IsFiltered && ret.IsFiltered != testcase.match {
t.Errorf("Expected hostname %s to not match", testcase.host)
}
if ret.IsFiltered && ret.IsFiltered != testcase.match {
t.Errorf("Expected hostname %s to match", testcase.host)
}
}
afterMatch := getRSS()
log.Tracef("RSS after matching - %d kB (%d kB diff)\n", afterMatch/1024, (afterMatch-afterLoad)/1024)
dumpMemProfile("tests/" + _Func() + "3.pprof")
}
func getRSS() uint64 {
proc, err := process.NewProcess(int32(os.Getpid()))
if err != nil {
panic(err)
}
minfo, err := proc.MemoryInfo()
if err != nil {
panic(err)
}
return minfo.RSS
}
func dumpMemProfile(name string) {
runtime.GC()
f, err := os.Create(name)
if err != nil {
panic(err)
}
defer f.Close()
runtime.GC() // update the stats before writing them
err = pprof.WriteHeapProfile(f)
if err != nil {
panic(err)
} }
} }
const topHostsFilename = "tests/top-1m.csv" func _Func() string {
pc := make([]uintptr, 10) // at least 1 entry needed
func fetchTopHostsFromNet() { runtime.Callers(2, pc)
log.Tracef("Fetching top hosts from network") f := runtime.FuncForPC(pc[0])
resp, err := http.Get("http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip") return path.Base(f.Name())
if err != nil {
panic(err)
}
defer resp.Body.Close()
log.Tracef("Reading zipfile body")
zipfile, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
log.Tracef("Opening zipfile")
r, err := zip.NewReader(bytes.NewReader(zipfile), int64(len(zipfile)))
if err != nil {
panic(err)
}
if len(r.File) != 1 {
panic(fmt.Errorf("zipfile must have only one entry: %+v", r))
}
f := r.File[0]
log.Tracef("Unpacking file %s from zipfile", f.Name)
rc, err := f.Open()
if err != nil {
panic(err)
}
log.Tracef("Reading file %s contents", f.Name)
body, err := ioutil.ReadAll(rc)
if err != nil {
panic(err)
}
rc.Close()
log.Tracef("Writing file %s contents to disk", f.Name)
err = ioutil.WriteFile(topHostsFilename+".tmp", body, 0644)
if err != nil {
panic(err)
}
err = os.Rename(topHostsFilename+".tmp", topHostsFilename)
if err != nil {
panic(err)
}
} }
func getTopHosts() { func NewForTest() *Dnsfilter {
// if file doesn't exist, fetch it d := New(nil, nil)
if _, err := os.Stat(topHostsFilename); os.IsNotExist(err) { purgeCaches()
// file does not exist, fetch it return d
fetchTopHostsFromNet()
}
} }
func TestLotsOfRulesLotsOfHostsMemoryUsage(t *testing.T) { func NewForTestFilters(filters map[int]string) *Dnsfilter {
start := getRSS() d := New(nil, filters)
log.Tracef("RSS before loading rules - %d kB\n", start/1024) purgeCaches()
dumpMemProfile("tests/" + _Func() + "1.pprof") return d
d := NewForTest()
defer d.Destroy()
mustLoadTestRules(d)
log.Tracef("Have %d rules", d.Count())
afterLoad := getRSS()
log.Tracef("RSS after loading rules - %d kB (%d kB diff)\n", afterLoad/1024, (afterLoad-start)/1024)
dumpMemProfile("tests/" + _Func() + "2.pprof")
getTopHosts()
hostnames, err := os.Open(topHostsFilename)
if err != nil {
t.Fatal(err)
}
defer hostnames.Close()
afterHosts := getRSS()
log.Tracef("RSS after loading hosts - %d kB (%d kB diff)\n", afterHosts/1024, (afterHosts-afterLoad)/1024)
dumpMemProfile("tests/" + _Func() + "2.pprof")
{
scanner := bufio.NewScanner(hostnames)
for scanner.Scan() {
line := scanner.Text()
records := strings.Split(line, ",")
ret, err := d.CheckHost(records[1] + "." + records[1])
if err != nil {
t.Error(err)
}
if ret.Reason.Matched() {
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
}
}
}
afterMatch := getRSS()
log.Tracef("RSS after matching - %d kB (%d kB diff)\n", afterMatch/1024, (afterMatch-afterLoad)/1024)
dumpMemProfile("tests/" + _Func() + "3.pprof")
}
func TestRuleToRegexp(t *testing.T) {
tests := []struct {
rule string
result string
err error
}{
{"/doubleclick/", "doubleclick", nil},
{"/", "", ErrInvalidSyntax},
{`|double*?.+[]|(){}#$\|`, `^double.*\?\.\+\[\]\|\(\)\{\}\#\$\\$`, nil},
{`||doubleclick.net^`, `(?:^|\.)doubleclick\.net$`, nil},
}
for _, testcase := range tests {
converted, err := ruleToRegexp(testcase.rule)
if err != testcase.err {
t.Error("Errors do not match, got ", err, " expected ", testcase.err)
}
if converted != testcase.result {
t.Error("Results do not match, got ", converted, " expected ", testcase.result)
}
}
}
func TestSuffixRule(t *testing.T) {
for _, testcase := range []struct {
rule string
isSuffix bool
suffix string
}{
{`||doubleclick.net^`, true, `doubleclick.net`}, // entire string or subdomain match
{`||doubleclick.net|`, true, `doubleclick.net`}, // entire string or subdomain match
{`|doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`|*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`||*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`||*doubleclick.net|`, false, ``}, // TODO: ends with doubleclick.net
{`||*doublec*lick.net^`, false, ``}, // has a wildcard inside, has to be regexp
{`||*doublec|lick.net^`, false, ``}, // has a special symbol inside, has to be regexp
{`/abracadabra/`, false, ``}, // regexp, not anchored
{`/abracadabra$/`, false, ``}, // TODO: simplify simple suffix regexes
} {
isSuffix, suffix := getSuffix(testcase.rule)
if testcase.isSuffix != isSuffix {
t.Errorf("Results do not match for \"%s\": got %v expected %v", testcase.rule, isSuffix, testcase.isSuffix)
continue
}
if testcase.isSuffix && testcase.suffix != suffix {
t.Errorf("Result suffix does not match for \"%s\": got \"%s\" expected \"%s\"", testcase.rule, suffix, testcase.suffix)
continue
}
// log.Tracef("\"%s\": %v: %s", testcase.rule, isSuffix, suffix)
}
}
//
// helper functions
//
func (d *Dnsfilter) checkAddRule(t *testing.T, rule string) {
t.Helper()
err := d.AddRule(rule, 0)
if err == nil {
// nothing to report
return
}
if err == ErrInvalidSyntax {
t.Errorf("This rule has invalid syntax: %s", rule)
}
if err != nil {
t.Errorf("Error while adding rule %s: %s", rule, err)
}
}
func (d *Dnsfilter) checkAddRuleFail(t *testing.T, rule string) {
t.Helper()
err := d.AddRule(rule, 0)
if err == ErrInvalidSyntax || err == ErrAlreadyExists {
return
}
if err != nil {
t.Errorf("Error while adding rule %s: %s", rule, err)
}
t.Errorf("Adding this rule should have failed: %s", rule)
} }
func (d *Dnsfilter) checkMatch(t *testing.T, hostname string) { func (d *Dnsfilter) checkMatch(t *testing.T, hostname string) {
@ -311,232 +84,21 @@ func (d *Dnsfilter) checkMatchEmpty(t *testing.T, hostname string) {
} }
} }
func loadTestRules(d *Dnsfilter) error {
filterFileName := "tests/dns.txt"
file, err := os.Open(filterFileName)
if err != nil {
return err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
rule := scanner.Text()
err = d.AddRule(rule, 0)
if err == ErrInvalidSyntax || err == ErrAlreadyExists {
continue
}
if err != nil {
return err
}
}
err = scanner.Err()
return err
}
func mustLoadTestRules(d *Dnsfilter) {
err := loadTestRules(d)
if err != nil {
panic(err)
}
}
func NewForTest() *Dnsfilter {
d := New(nil)
purgeCaches()
return d
}
//
// tests
//
func TestSanityCheck(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatchEmpty(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
d.checkMatchEmpty(t, "wmconvirus.narod.ru")
d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384")
}
func TestEtcHostsMatching(t *testing.T) { func TestEtcHostsMatching(t *testing.T) {
d := NewForTest()
defer d.Destroy()
addr := "216.239.38.120" addr := "216.239.38.120"
text := fmt.Sprintf(" %s google.com www.google.com # enforce google's safesearch ", addr) text := fmt.Sprintf(" %s google.com www.google.com # enforce google's safesearch ", addr)
filters := make(map[int]string)
filters[0] = text
d := NewForTestFilters(filters)
defer d.Destroy()
d.checkAddRule(t, text)
d.checkMatchIP(t, "google.com", addr) d.checkMatchIP(t, "google.com", addr)
d.checkMatchIP(t, "www.google.com", addr) d.checkMatchIP(t, "www.google.com", addr)
d.checkMatchEmpty(t, "subdomain.google.com") d.checkMatchEmpty(t, "subdomain.google.com")
d.checkMatchEmpty(t, "example.org") d.checkMatchEmpty(t, "example.org")
} }
func TestSuffixMatching1(t *testing.T) { // SAFE BROWSING
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatchEmpty(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching2(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "|doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatchEmpty(t, "www.doubleclick.net")
d.checkMatchEmpty(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching3(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching4(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "*doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching5(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "|*doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching6(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||*doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestCount(t *testing.T) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
t.Fatal(err)
}
count := d.Count()
expected := 12747
if count != expected {
t.Fatalf("Number of rules parsed should be %d, but it is %d\n", expected, count)
}
}
func TestDnsFilterBlocking(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||example.org^")
d.checkMatch(t, "example.org")
d.checkMatch(t, "test.example.org")
d.checkMatch(t, "test.test.example.org")
d.checkMatchEmpty(t, "testexample.org")
d.checkMatchEmpty(t, "onemoreexample.org")
}
func TestDnsFilterWhitelist(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||example.org^")
d.checkAddRule(t, "@@||test.example.org")
d.checkMatch(t, "example.org")
d.checkMatchEmpty(t, "test.example.org")
d.checkMatchEmpty(t, "test.test.example.org")
d.checkAddRule(t, "||googleadapis.l.google.com^|")
d.checkMatch(t, "googleadapis.l.google.com")
d.checkMatch(t, "test.googleadapis.l.google.com")
d.checkAddRule(t, "@@||googleadapis.l.google.com|")
d.checkMatchEmpty(t, "googleadapis.l.google.com")
d.checkMatchEmpty(t, "test.googleadapis.l.google.com")
}
func TestDnsFilterImportant(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "@@||example.org^")
d.checkAddRule(t, "||test.example.org^$important")
d.checkMatchEmpty(t, "example.org")
d.checkMatch(t, "test.example.org")
d.checkMatch(t, "test.test.example.org")
d.checkMatchEmpty(t, "testexample.org")
d.checkMatchEmpty(t, "onemoreexample.org")
}
func TestDnsFilterRegexrule(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "/example\\.org/")
d.checkAddRule(t, "@@||test.example.org^")
d.checkMatch(t, "example.org")
d.checkMatchEmpty(t, "test.example.org")
d.checkMatchEmpty(t, "test.test.example.org")
d.checkMatch(t, "testexample.org")
d.checkMatch(t, "onemoreexample.org")
}
func TestDomainMask(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "test*.example.org^")
d.checkAddRule(t, "exam*.com")
d.checkMatch(t, "test.example.org")
d.checkMatch(t, "test2.example.org")
d.checkMatch(t, "example.com")
d.checkMatch(t, "exampleeee.com")
d.checkMatchEmpty(t, "example.org")
d.checkMatchEmpty(t, "testexample.org")
d.checkMatchEmpty(t, "example.co.uk")
}
func TestAddRuleFail(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384")
}
func TestSafeBrowsing(t *testing.T) { func TestSafeBrowsing(t *testing.T) {
testCases := []string{ testCases := []string{
@ -608,6 +170,25 @@ func TestSafeBrowsingCustomServerFail(t *testing.T) {
d.checkMatchEmpty(t, "wmconvirus.narod.ru") d.checkMatchEmpty(t, "wmconvirus.narod.ru")
} }
// SAFE SEARCH
func TestSafeSearch(t *testing.T) {
d := NewForTest()
defer d.Destroy()
_, ok := d.SafeSearchDomain("www.google.com")
if ok {
t.Errorf("Expected safesearch to error when disabled")
}
d.SafeSearchEnabled = true
val, ok := d.SafeSearchDomain("www.google.com")
if !ok {
t.Errorf("Expected safesearch to find result for www.google.com")
}
if val != "forcesafesearch.google.com" {
t.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
}
}
func TestCheckHostSafeSearchYandex(t *testing.T) { func TestCheckHostSafeSearchYandex(t *testing.T) {
d := NewForTest() d := NewForTest()
defer d.Destroy() defer d.Destroy()
@ -757,6 +338,8 @@ func TestSafeSearchCacheGoogle(t *testing.T) {
} }
} }
// PARENTAL
func TestParentalControl(t *testing.T) { func TestParentalControl(t *testing.T) {
d := NewForTest() d := NewForTest()
defer d.Destroy() defer d.Destroy()
@ -785,63 +368,50 @@ func TestParentalControl(t *testing.T) {
d.checkMatchEmpty(t, "api.jquery.com") d.checkMatchEmpty(t, "api.jquery.com")
} }
func TestSafeSearch(t *testing.T) { // FILTERING
d := NewForTest()
defer d.Destroy()
_, ok := d.SafeSearchDomain("www.google.com")
if ok {
t.Errorf("Expected safesearch to error when disabled")
}
d.SafeSearchEnabled = true
val, ok := d.SafeSearchDomain("www.google.com")
if !ok {
t.Errorf("Expected safesearch to find result for www.google.com")
}
if val != "forcesafesearch.google.com" {
t.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
}
}
// var blockingRules = "||example.org^\n"
// parametrized testing var whitelistRules = "||example.org^\n@@||test.example.org\n"
// var importantRules = "@@||example.org^\n||test.example.org^$important\n"
var blockingRules = []string{"||example.org^"} var regexRules = "/example\\.org/\n@@||test.example.org^\n"
var whitelistRules = []string{"||example.org^", "@@||test.example.org"} var maskRules = "test*.example.org^\nexam*.com\n"
var importantRules = []string{"@@||example.org^", "||test.example.org^$important"}
var regexRules = []string{"/example\\.org/", "@@||test.example.org^"}
var maskRules = []string{"test*.example.org^", "exam*.com"}
var tests = []struct { var tests = []struct {
testname string testname string
rules []string rules string
hostname string hostname string
isFiltered bool isFiltered bool
reason Reason reason Reason
}{ }{
{"sanity", []string{"||doubleclick.net^"}, "www.doubleclick.net", true, FilteredBlackList}, {"sanity", "||doubleclick.net^", "www.doubleclick.net", true, FilteredBlackList},
{"sanity", []string{"||doubleclick.net^"}, "nodoubleclick.net", false, NotFilteredNotFound}, {"sanity", "||doubleclick.net^", "nodoubleclick.net", false, NotFilteredNotFound},
{"sanity", []string{"||doubleclick.net^"}, "doubleclick.net.ru", false, NotFilteredNotFound}, {"sanity", "||doubleclick.net^", "doubleclick.net.ru", false, NotFilteredNotFound},
{"sanity", []string{"||doubleclick.net^"}, "wmconvirus.narod.ru", false, NotFilteredNotFound}, {"sanity", "||doubleclick.net^", "wmconvirus.narod.ru", false, NotFilteredNotFound},
{"blocking", blockingRules, "example.org", true, FilteredBlackList}, {"blocking", blockingRules, "example.org", true, FilteredBlackList},
{"blocking", blockingRules, "test.example.org", true, FilteredBlackList}, {"blocking", blockingRules, "test.example.org", true, FilteredBlackList},
{"blocking", blockingRules, "test.test.example.org", true, FilteredBlackList}, {"blocking", blockingRules, "test.test.example.org", true, FilteredBlackList},
{"blocking", blockingRules, "testexample.org", false, NotFilteredNotFound}, {"blocking", blockingRules, "testexample.org", false, NotFilteredNotFound},
{"blocking", blockingRules, "onemoreexample.org", false, NotFilteredNotFound}, {"blocking", blockingRules, "onemoreexample.org", false, NotFilteredNotFound},
{"whitelist", whitelistRules, "example.org", true, FilteredBlackList}, {"whitelist", whitelistRules, "example.org", true, FilteredBlackList},
{"whitelist", whitelistRules, "test.example.org", false, NotFilteredWhiteList}, {"whitelist", whitelistRules, "test.example.org", false, NotFilteredWhiteList},
{"whitelist", whitelistRules, "test.test.example.org", false, NotFilteredWhiteList}, {"whitelist", whitelistRules, "test.test.example.org", false, NotFilteredWhiteList},
{"whitelist", whitelistRules, "testexample.org", false, NotFilteredNotFound}, {"whitelist", whitelistRules, "testexample.org", false, NotFilteredNotFound},
{"whitelist", whitelistRules, "onemoreexample.org", false, NotFilteredNotFound}, {"whitelist", whitelistRules, "onemoreexample.org", false, NotFilteredNotFound},
{"important", importantRules, "example.org", false, NotFilteredWhiteList}, {"important", importantRules, "example.org", false, NotFilteredWhiteList},
{"important", importantRules, "test.example.org", true, FilteredBlackList}, {"important", importantRules, "test.example.org", true, FilteredBlackList},
{"important", importantRules, "test.test.example.org", true, FilteredBlackList}, {"important", importantRules, "test.test.example.org", true, FilteredBlackList},
{"important", importantRules, "testexample.org", false, NotFilteredNotFound}, {"important", importantRules, "testexample.org", false, NotFilteredNotFound},
{"important", importantRules, "onemoreexample.org", false, NotFilteredNotFound}, {"important", importantRules, "onemoreexample.org", false, NotFilteredNotFound},
{"regex", regexRules, "example.org", true, FilteredBlackList}, {"regex", regexRules, "example.org", true, FilteredBlackList},
{"regex", regexRules, "test.example.org", false, NotFilteredWhiteList}, {"regex", regexRules, "test.example.org", false, NotFilteredWhiteList},
{"regex", regexRules, "test.test.example.org", false, NotFilteredWhiteList}, {"regex", regexRules, "test.test.example.org", false, NotFilteredWhiteList},
{"regex", regexRules, "testexample.org", true, FilteredBlackList}, {"regex", regexRules, "testexample.org", true, FilteredBlackList},
{"regex", regexRules, "onemoreexample.org", true, FilteredBlackList}, {"regex", regexRules, "onemoreexample.org", true, FilteredBlackList},
{"mask", maskRules, "test.example.org", true, FilteredBlackList}, {"mask", maskRules, "test.example.org", true, FilteredBlackList},
{"mask", maskRules, "test2.example.org", true, FilteredBlackList}, {"mask", maskRules, "test2.example.org", true, FilteredBlackList},
{"mask", maskRules, "example.com", true, FilteredBlackList}, {"mask", maskRules, "example.com", true, FilteredBlackList},
@ -855,14 +425,11 @@ var tests = []struct {
func TestMatching(t *testing.T) { func TestMatching(t *testing.T) {
for _, test := range tests { for _, test := range tests {
t.Run(fmt.Sprintf("%s-%s", test.testname, test.hostname), func(t *testing.T) { t.Run(fmt.Sprintf("%s-%s", test.testname, test.hostname), func(t *testing.T) {
d := NewForTest() filters := make(map[int]string)
filters[0] = test.rules
d := NewForTestFilters(filters)
defer d.Destroy() defer d.Destroy()
for _, rule := range test.rules {
err := d.AddRule(rule, 0)
if err != nil {
t.Fatal(err)
}
}
ret, err := d.CheckHost(test.hostname) ret, err := d.CheckHost(test.hostname)
if err != nil { if err != nil {
t.Errorf("Error while matching host %s: %s", test.hostname, err) t.Errorf("Error while matching host %s: %s", test.hostname, err)
@ -877,204 +444,7 @@ func TestMatching(t *testing.T) {
} }
} }
// // BENCHMARKS
// benchmarks
//
func BenchmarkAddRule(b *testing.B) {
d := NewForTest()
defer d.Destroy()
for n := 0; n < b.N; n++ {
rule := "||doubleclick.net^"
err := d.AddRule(rule, 0)
switch err {
case nil:
case ErrAlreadyExists: // ignore rules which were already added
case ErrInvalidSyntax: // ignore invalid syntax
default:
b.Fatalf("Error while adding rule %s: %s", rule, err)
}
}
}
func BenchmarkAddRuleParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
rule := "||doubleclick.net^"
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
var err error
for pb.Next() {
err = d.AddRule(rule, 0)
}
switch err {
case nil:
case ErrAlreadyExists: // ignore rules which were already added
case ErrInvalidSyntax: // ignore invalid syntax
default:
b.Fatalf("Error while adding rule %s: %s", rule, err)
}
})
}
func BenchmarkLotsOfRulesNoMatch(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for n := 0; n < b.N; n++ {
hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com"
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if ret.IsFiltered {
b.Errorf("Expected hostname %s to not match", hostname)
}
}
}
func BenchmarkLotsOfRulesNoMatchParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com"
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if ret.IsFiltered {
b.Errorf("Expected hostname %s to not match", hostname)
}
}
})
}
func BenchmarkLotsOfRulesMatch(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for n := 0; n < b.N; n++ {
const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net"
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if !ret.IsFiltered {
b.Errorf("Expected hostname %s to match", hostname)
}
}
}
func BenchmarkLotsOfRulesMatchParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net"
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if !ret.IsFiltered {
b.Errorf("Expected hostname %s to match", hostname)
}
}
})
}
func BenchmarkLotsOfRulesLotsOfHosts(b *testing.B) {
d := NewForTest()
defer d.Destroy()
mustLoadTestRules(d)
getTopHosts()
hostnames, err := os.Open(topHostsFilename)
if err != nil {
b.Fatal(err)
}
defer hostnames.Close()
scanner := bufio.NewScanner(hostnames)
b.ResetTimer()
for n := 0; n < b.N; n++ {
havedata := scanner.Scan()
if !havedata {
_, _ = hostnames.Seek(0, 0)
scanner = bufio.NewScanner(hostnames)
havedata = scanner.Scan()
}
if !havedata {
b.Fatal(scanner.Err())
}
line := scanner.Text()
records := strings.Split(line, ",")
ret, err := d.CheckHost(records[1] + "." + records[1])
if err != nil {
b.Error(err)
}
if ret.Reason.Matched() {
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
}
}
}
func BenchmarkLotsOfRulesLotsOfHostsParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
mustLoadTestRules(d)
getTopHosts()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
hostnames, err := os.Open(topHostsFilename)
if err != nil {
b.Fatal(err)
}
defer hostnames.Close()
scanner := bufio.NewScanner(hostnames)
for pb.Next() {
havedata := scanner.Scan()
if !havedata {
_, _ = hostnames.Seek(0, 0)
scanner = bufio.NewScanner(hostnames)
havedata = scanner.Scan()
}
if !havedata {
b.Fatal(scanner.Err())
}
line := scanner.Text()
records := strings.Split(line, ",")
ret, err := d.CheckHost(records[1] + "." + records[1])
if err != nil {
b.Error(err)
}
if ret.Reason.Matched() {
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
}
}
})
}
func BenchmarkSafeBrowsing(b *testing.B) { func BenchmarkSafeBrowsing(b *testing.B) {
d := NewForTest() d := NewForTest()
@ -1141,26 +511,3 @@ func BenchmarkSafeSearchParallel(b *testing.B) {
} }
}) })
} }
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
//
// helper functions for debugging and testing
//
func purgeCaches() {
if safebrowsingCache != nil {
safebrowsingCache.Purge()
}
if parentalCache != nil {
parentalCache.Purge()
}
}
func _Func() string {
pc := make([]uintptr, 10) // at least 1 entry needed
runtime.Callers(2, pc)
f := runtime.FuncForPC(pc[0])
return path.Base(f.Name())
}

40
dnsfilter/helpers.go
View File

@ -1,49 +1,9 @@
package dnsfilter package dnsfilter
import ( import (
"strings"
"sync/atomic" "sync/atomic"
) )
func isValidRule(rule string) bool {
if len(rule) < 4 {
return false
}
if rule[0] == '!' {
return false
}
if rule[0] == '#' {
return false
}
if strings.HasPrefix(rule, "[Adblock") {
return false
}
// Filter out all sorts of cosmetic rules:
// https://kb.adguard.com/en/general/how-to-create-your-own-ad-filters#cosmetic-rules
masks := []string{
"##",
"#@#",
"#?#",
"#@?#",
"#$#",
"#@$#",
"#?$#",
"#@?$#",
"$$",
"$@$",
"#%#",
"#@%#",
}
for _, mask := range masks {
if strings.Contains(rule, mask) {
return false
}
}
return true
}
func updateMax(valuePtr *int64, maxPtr *int64) { func updateMax(valuePtr *int64, maxPtr *int64) {
for { for {
current := atomic.LoadInt64(valuePtr) current := atomic.LoadInt64(valuePtr)

91
dnsfilter/rule_to_regexp.go
View File

@ -1,91 +0,0 @@
package dnsfilter
import (
"strings"
)
func ruleToRegexp(rule string) (string, error) {
const hostStart = `(?:^|\.)`
const hostEnd = `$`
// empty or short rule -- do nothing
if !isValidRule(rule) {
return "", ErrInvalidSyntax
}
// if starts with / and ends with /, it's already a regexp, just strip the slashes
if rule[0] == '/' && rule[len(rule)-1] == '/' {
return rule[1 : len(rule)-1], nil
}
var sb strings.Builder
if rule[0] == '|' && rule[1] == '|' {
sb.WriteString(hostStart)
rule = rule[2:]
}
for i, r := range rule {
switch {
case r == '?' || r == '.' || r == '+' || r == '[' || r == ']' || r == '(' || r == ')' || r == '{' || r == '}' || r == '#' || r == '\\' || r == '$':
sb.WriteRune('\\')
sb.WriteRune(r)
case r == '|' && i == 0:
// | at start and it's not || at start
sb.WriteRune('^')
case r == '|' && i == len(rule)-1:
// | at end
sb.WriteRune('$')
case r == '|' && i != 0 && i != len(rule)-1:
sb.WriteString(`\|`)
case r == '*':
sb.WriteString(`.*`)
case r == '^':
sb.WriteString(hostEnd)
default:
sb.WriteRune(r)
}
}
return sb.String(), nil
}
// handle suffix rule ||example.com^ -- either entire string is example.com or *.example.com
func getSuffix(rule string) (bool, string) {
// if starts with / and ends with /, it's already a regexp
// TODO: if a regexp is simple `/abracadabra$/`, then simplify it maybe?
if rule[0] == '/' && rule[len(rule)-1] == '/' {
return false, ""
}
// must start with ||
if rule[0] != '|' || rule[1] != '|' {
return false, ""
}
rule = rule[2:]
// suffix rule must end with ^ or |
lastChar := rule[len(rule)-1]
if lastChar != '^' && lastChar != '|' {
return false, ""
}
// last char was checked, eat it
rule = rule[:len(rule)-1]
// it might also end with ^|
if rule[len(rule)-1] == '^' {
rule = rule[:len(rule)-1]
}
// check that it doesn't have any special characters inside
for _, r := range rule {
switch r {
case '|':
return false, ""
case '*':
return false, ""
}
}
return true, rule
}

56
dnsforward/dnsforward.go
View File

@ -44,7 +44,7 @@ type Server struct {
once sync.Once once sync.Once
sync.RWMutex sync.RWMutex
ServerConfig conf ServerConfig
} }
// NewServer creates a new instance of the dnsforward.Server // NewServer creates a new instance of the dnsforward.Server
@ -123,7 +123,7 @@ func (s *Server) Start(config *ServerConfig) error {
// startInternal starts without locking // startInternal starts without locking
func (s *Server) startInternal(config *ServerConfig) error { func (s *Server) startInternal(config *ServerConfig) error {
if config != nil { if config != nil {
s.ServerConfig = *config s.conf = *config
} }
if s.dnsFilter != nil || s.dnsProxy != nil { if s.dnsFilter != nil || s.dnsProxy != nil {
@ -158,21 +158,21 @@ func (s *Server) startInternal(config *ServerConfig) error {
}) })
proxyConfig := proxy.Config{ proxyConfig := proxy.Config{
UDPListenAddr: s.UDPListenAddr, UDPListenAddr: s.conf.UDPListenAddr,
TCPListenAddr: s.TCPListenAddr, TCPListenAddr: s.conf.TCPListenAddr,
Ratelimit: s.Ratelimit, Ratelimit: s.conf.Ratelimit,
RatelimitWhitelist: s.RatelimitWhitelist, RatelimitWhitelist: s.conf.RatelimitWhitelist,
RefuseAny: s.RefuseAny, RefuseAny: s.conf.RefuseAny,
CacheEnabled: true, CacheEnabled: true,
Upstreams: s.Upstreams, Upstreams: s.conf.Upstreams,
DomainsReservedUpstreams: s.DomainsReservedUpstreams, DomainsReservedUpstreams: s.conf.DomainsReservedUpstreams,
Handler: s.handleDNSRequest, Handler: s.handleDNSRequest,
AllServers: s.AllServers, AllServers: s.conf.AllServers,
} }
if s.TLSListenAddr != nil && s.CertificateChain != "" && s.PrivateKey != "" { if s.conf.TLSListenAddr != nil && s.conf.CertificateChain != "" && s.conf.PrivateKey != "" {
proxyConfig.TLSListenAddr = s.TLSListenAddr proxyConfig.TLSListenAddr = s.conf.TLSListenAddr
keypair, err := tls.X509KeyPair([]byte(s.CertificateChain), []byte(s.PrivateKey)) keypair, err := tls.X509KeyPair([]byte(s.conf.CertificateChain), []byte(s.conf.PrivateKey))
if err != nil { if err != nil {
return errorx.Decorate(err, "Failed to parse TLS keypair") return errorx.Decorate(err, "Failed to parse TLS keypair")
} }
@ -202,14 +202,20 @@ func (s *Server) startInternal(config *ServerConfig) error {
// Initializes the DNS filter // Initializes the DNS filter
func (s *Server) initDNSFilter() error { func (s *Server) initDNSFilter() error {
log.Tracef("Creating dnsfilter") log.Tracef("Creating dnsfilter")
s.dnsFilter = dnsfilter.New(&s.Config)
// add rules only if they are enabled var filters map[int]string
if s.FilteringEnabled { filters = nil
err := s.dnsFilter.AddRules(s.Filters) if s.conf.FilteringEnabled {
if err != nil { filters = make(map[int]string)
return errorx.Decorate(err, "could not initialize dnsfilter") for _, f := range s.conf.Filters {
filters[int(f.ID)] = string(f.Data)
} }
} }
s.dnsFilter = dnsfilter.New(&s.conf.Config, filters)
if s.dnsFilter == nil {
return fmt.Errorf("could not initialize dnsfilter")
}
return nil return nil
} }
@ -336,11 +342,11 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error {
msg := d.Req msg := d.Req
// don't log ANY request if refuseAny is enabled // don't log ANY request if refuseAny is enabled
if len(msg.Question) >= 1 && msg.Question[0].Qtype == dns.TypeANY && s.RefuseAny { if len(msg.Question) >= 1 && msg.Question[0].Qtype == dns.TypeANY && s.conf.RefuseAny {
shouldLog = false shouldLog = false
} }
if s.QueryLogEnabled && shouldLog { if s.conf.QueryLogEnabled && shouldLog {
elapsed := time.Since(start) elapsed := time.Since(start)
upstreamAddr := "" upstreamAddr := ""
if d.Upstream != nil { if d.Upstream != nil {
@ -361,7 +367,7 @@ func (s *Server) filterDNSRequest(d *proxy.DNSContext) (*dnsfilter.Result, error
host := strings.TrimSuffix(msg.Question[0].Name, ".") host := strings.TrimSuffix(msg.Question[0].Name, ".")
s.RLock() s.RLock()
protectionEnabled := s.ProtectionEnabled protectionEnabled := s.conf.ProtectionEnabled
dnsFilter := s.dnsFilter dnsFilter := s.dnsFilter
s.RUnlock() s.RUnlock()
@ -402,7 +408,7 @@ func (s *Server) genDNSFilterMessage(d *proxy.DNSContext, result *dnsfilter.Resu
return s.genARecord(m, result.IP) return s.genARecord(m, result.IP)
} }
if s.BlockingMode == "null_ip" { if s.conf.BlockingMode == "null_ip" {
return s.genARecord(m, net.IPv4zero) return s.genARecord(m, net.IPv4zero)
} }
@ -420,7 +426,7 @@ func (s *Server) genServerFailure(request *dns.Msg) *dns.Msg {
func (s *Server) genARecord(request *dns.Msg, ip net.IP) *dns.Msg { func (s *Server) genARecord(request *dns.Msg, ip net.IP) *dns.Msg {
resp := dns.Msg{} resp := dns.Msg{}
resp.SetReply(request) resp.SetReply(request)
answer, err := dns.NewRR(fmt.Sprintf("%s %d A %s", request.Question[0].Name, s.BlockedResponseTTL, ip.String())) answer, err := dns.NewRR(fmt.Sprintf("%s %d A %s", request.Question[0].Name, s.conf.BlockedResponseTTL, ip.String()))
if err != nil { if err != nil {
log.Printf("Couldn't generate A record for replacement host '%s': %s", ip.String(), err) log.Printf("Couldn't generate A record for replacement host '%s': %s", ip.String(), err)
return s.genServerFailure(request) return s.genServerFailure(request)
@ -489,7 +495,7 @@ func (s *Server) genSOA(request *dns.Msg) []dns.RR {
Hdr: dns.RR_Header{ Hdr: dns.RR_Header{
Name: zone, Name: zone,
Rrtype: dns.TypeSOA, Rrtype: dns.TypeSOA,
Ttl: s.BlockedResponseTTL, Ttl: s.conf.BlockedResponseTTL,
Class: dns.ClassINET, Class: dns.ClassINET,
}, },
Mbox: "hostmaster.", // zone will be appended later if it's not empty or "." Mbox: "hostmaster.", // zone will be appended later if it's not empty or "."

22
dnsforward/dnsforward_test.go
View File

@ -86,7 +86,7 @@ func TestDotServer(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
defer removeDataDir(t) defer removeDataDir(t)
s.TLSConfig = TLSConfig{ s.conf.TLSConfig = TLSConfig{
TLSListenAddr: &net.TCPAddr{Port: 0}, TLSListenAddr: &net.TCPAddr{Port: 0},
CertificateChain: string(certPem), CertificateChain: string(certPem),
PrivateKey: string(keyPem), PrivateKey: string(keyPem),
@ -149,7 +149,7 @@ func TestServerRace(t *testing.T) {
func TestSafeSearch(t *testing.T) { func TestSafeSearch(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
s.SafeSearchEnabled = true s.conf.SafeSearchEnabled = true
defer removeDataDir(t) defer removeDataDir(t)
err := s.Start(nil) err := s.Start(nil)
if err != nil { if err != nil {
@ -295,7 +295,7 @@ func TestBlockedRequest(t *testing.T) {
func TestNullBlockedRequest(t *testing.T) { func TestNullBlockedRequest(t *testing.T) {
s := createTestServer(t) s := createTestServer(t)
s.FilteringConfig.BlockingMode = "null_ip" s.conf.FilteringConfig.BlockingMode = "null_ip"
defer removeDataDir(t) defer removeDataDir(t)
err := s.Start(nil) err := s.Start(nil)
if err != nil { if err != nil {
@ -451,14 +451,14 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
func createTestServer(t *testing.T) *Server { func createTestServer(t *testing.T) *Server {
s := NewServer(createDataDir(t)) s := NewServer(createDataDir(t))
s.UDPListenAddr = &net.UDPAddr{Port: 0} s.conf.UDPListenAddr = &net.UDPAddr{Port: 0}
s.TCPListenAddr = &net.TCPAddr{Port: 0} s.conf.TCPListenAddr = &net.TCPAddr{Port: 0}
s.QueryLogEnabled = true s.conf.QueryLogEnabled = true
s.FilteringConfig.FilteringEnabled = true s.conf.FilteringConfig.FilteringEnabled = true
s.FilteringConfig.ProtectionEnabled = true s.conf.FilteringConfig.ProtectionEnabled = true
s.FilteringConfig.SafeBrowsingEnabled = true s.conf.FilteringConfig.SafeBrowsingEnabled = true
s.Filters = make([]dnsfilter.Filter, 0) s.conf.Filters = make([]dnsfilter.Filter, 0)
rules := []string{ rules := []string{
"||nxdomain.example.org^", "||nxdomain.example.org^",
@ -466,7 +466,7 @@ func createTestServer(t *testing.T) *Server {
"127.0.0.1 host.example.org", "127.0.0.1 host.example.org",
} }
filter := dnsfilter.Filter{ID: 1, Rules: rules} filter := dnsfilter.Filter{ID: 1, Rules: rules}
s.Filters = append(s.Filters, filter) s.conf.Filters = append(s.conf.Filters, filter)
return s return s
} }

26
filter.go
View File

@ -35,13 +35,12 @@ type filter struct {
// Creates a helper object for working with the user rules // Creates a helper object for working with the user rules
func userFilter() filter { func userFilter() filter {
return filter{ f := filter{
// User filter always has constant ID=0 // User filter always has constant ID=0
Enabled: true, Enabled: true,
Filter: dnsfilter.Filter{
Rules: config.UserRules,
},
} }
f.Filter.Data = []byte(strings.Join(config.UserRules, "\n"))
return f
} }
// Enable or disable a filter // Enable or disable a filter
@ -242,7 +241,7 @@ func refreshFiltersIfNecessary(force bool) int {
log.Info("Updated filter #%d. Rules: %d -> %d", log.Info("Updated filter #%d. Rules: %d -> %d",
f.ID, f.RulesCount, uf.RulesCount) f.ID, f.RulesCount, uf.RulesCount)
f.Name = uf.Name f.Name = uf.Name
f.Rules = uf.Rules f.Data = uf.Data
f.RulesCount = uf.RulesCount f.RulesCount = uf.RulesCount
f.checksum = uf.checksum f.checksum = uf.checksum
updateCount++ updateCount++
@ -261,7 +260,7 @@ func refreshFiltersIfNecessary(force bool) int {
} }
// A helper function that parses filter contents and returns a number of rules and a filter name (if there's any) // A helper function that parses filter contents and returns a number of rules and a filter name (if there's any)
func parseFilterContents(contents []byte) (int, string, []string) { func parseFilterContents(contents []byte) (int, string) {
lines := strings.Split(string(contents), "\n") lines := strings.Split(string(contents), "\n")
rulesCount := 0 rulesCount := 0
name := "" name := ""
@ -286,7 +285,7 @@ func parseFilterContents(contents []byte) (int, string, []string) {
} }
} }
return rulesCount, name, lines return rulesCount, name
} }
// Perform upgrade on a filter // Perform upgrade on a filter
@ -327,13 +326,13 @@ func (filter *filter) update() (bool, error) {
} }
// Extract filter name and count number of rules // Extract filter name and count number of rules
rulesCount, filterName, rules := parseFilterContents(body) rulesCount, filterName := parseFilterContents(body)
log.Printf("Filter %d has been updated: %d bytes, %d rules", filter.ID, len(body), rulesCount) log.Printf("Filter %d has been updated: %d bytes, %d rules", filter.ID, len(body), rulesCount)
if filterName != "" { if filterName != "" {
filter.Name = filterName filter.Name = filterName
} }
filter.RulesCount = rulesCount filter.RulesCount = rulesCount
filter.Rules = rules filter.Data = body
filter.checksum = checksum filter.checksum = checksum
return true, nil return true, nil
@ -343,9 +342,8 @@ func (filter *filter) update() (bool, error) {
func (filter *filter) save() error { func (filter *filter) save() error {
filterFilePath := filter.Path() filterFilePath := filter.Path()
log.Printf("Saving filter %d contents to: %s", filter.ID, filterFilePath) log.Printf("Saving filter %d contents to: %s", filter.ID, filterFilePath)
body := []byte(strings.Join(filter.Rules, "\n"))
err := file.SafeWrite(filterFilePath, body) err := file.SafeWrite(filterFilePath, filter.Data)
// update LastUpdated field after saving the file // update LastUpdated field after saving the file
filter.LastUpdated = filter.LastTimeUpdated() filter.LastUpdated = filter.LastTimeUpdated()
@ -368,10 +366,10 @@ func (filter *filter) load() error {
} }
log.Tracef("File %s, id %d, length %d", filterFilePath, filter.ID, len(filterFileContents)) log.Tracef("File %s, id %d, length %d", filterFilePath, filter.ID, len(filterFileContents))
rulesCount, _, rules := parseFilterContents(filterFileContents) rulesCount, _ := parseFilterContents(filterFileContents)
filter.RulesCount = rulesCount filter.RulesCount = rulesCount
filter.Rules = rules filter.Data = filterFileContents
filter.checksum = crc32.ChecksumIEEE(filterFileContents) filter.checksum = crc32.ChecksumIEEE(filterFileContents)
filter.LastUpdated = filter.LastTimeUpdated() filter.LastUpdated = filter.LastTimeUpdated()
@ -380,7 +378,7 @@ func (filter *filter) load() error {
// Clear filter rules // Clear filter rules
func (filter *filter) unload() { func (filter *filter) unload() {
filter.Rules = []string{} filter.Data = nil
filter.RulesCount = 0 filter.RulesCount = 0
} }

10
go.mod
View File

@ -5,10 +5,9 @@ go 1.12
require ( require (
github.com/AdguardTeam/dnsproxy v0.12.0 github.com/AdguardTeam/dnsproxy v0.12.0
github.com/AdguardTeam/golibs v0.1.3 github.com/AdguardTeam/golibs v0.1.3
github.com/AdguardTeam/urlfilter v0.3.0
github.com/NYTimes/gziphandler v1.1.1 github.com/NYTimes/gziphandler v1.1.1
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect
github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0 github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0
github.com/go-ole/go-ole v1.2.1 // indirect
github.com/go-test/deep v1.0.1 github.com/go-test/deep v1.0.1
github.com/gobuffalo/packr v1.19.0 github.com/gobuffalo/packr v1.19.0
github.com/joomcode/errorx v0.1.0 github.com/joomcode/errorx v0.1.0
@ -16,13 +15,10 @@ require (
github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b
github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414 github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414
github.com/miekg/dns v1.1.1 github.com/miekg/dns v1.1.1
github.com/shirou/gopsutil v2.18.10+incompatible
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0
github.com/stretchr/testify v1.3.0 github.com/stretchr/testify v1.3.0
go.uber.org/goleak v0.10.0 golang.org/x/net v0.0.0-20190313220215-9f648a60d977
golang.org/x/net v0.0.0-20190119204137-ed066c81e75e golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a
golang.org/x/sys v0.0.0-20190122071731-054c452bb702
gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477 gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477
gopkg.in/yaml.v2 v2.2.1 gopkg.in/yaml.v2 v2.2.1
) )

31
go.sum
View File

@ -3,10 +3,12 @@ github.com/AdguardTeam/dnsproxy v0.12.0/go.mod h1:lcZM2QPwcWGEL3pz8RYy06nQdbjj4p
github.com/AdguardTeam/golibs v0.1.2/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko= github.com/AdguardTeam/golibs v0.1.2/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko=
github.com/AdguardTeam/golibs v0.1.3 h1:hmapdTtMtIk3T8eQDwTOLdqZLGDKNKk9325uC8z12xg= github.com/AdguardTeam/golibs v0.1.3 h1:hmapdTtMtIk3T8eQDwTOLdqZLGDKNKk9325uC8z12xg=
github.com/AdguardTeam/golibs v0.1.3/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko= github.com/AdguardTeam/golibs v0.1.3/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko=
github.com/AdguardTeam/urlfilter v0.3.0 h1:WNd3uZEYWwxylUuA8QS6V5DqHNsVFw3ZD/E2rd5HGpo=
github.com/AdguardTeam/urlfilter v0.3.0/go.mod h1:9xfZ6R2vB8LlT8G9LxtbNhDsbr/xybUOSwmJvpXhl/c=
github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8DgGXC5B7ILL8y51fci/qYz2B4j8iLY= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s=
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 h1:52m0LGchQBBVqJRyYYufQuIbVqRawmubW3OFGqK1ekw= github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 h1:52m0LGchQBBVqJRyYYufQuIbVqRawmubW3OFGqK1ekw=
@ -15,6 +17,9 @@ github.com/ameshkov/dnscrypt v1.0.6 h1:55wfnNF8c4E3JXDNlwPl2Pbs7UPPIh+kI6KK3THqY
github.com/ameshkov/dnscrypt v1.0.6/go.mod h1:ZvT9LaNaJfDNXKIbkYFf24HUgHuQR6MNT6nwVvN4jMQ= github.com/ameshkov/dnscrypt v1.0.6/go.mod h1:ZvT9LaNaJfDNXKIbkYFf24HUgHuQR6MNT6nwVvN4jMQ=
github.com/ameshkov/dnsstamps v1.0.1 h1:LhGvgWDzhNJh+kBQd/AfUlq1vfVe109huiXw4JhnPug= github.com/ameshkov/dnsstamps v1.0.1 h1:LhGvgWDzhNJh+kBQd/AfUlq1vfVe109huiXw4JhnPug=
github.com/ameshkov/dnsstamps v1.0.1/go.mod h1:Ii3eUu73dx4Vw5O4wjzmT5+lkCwovjzaEZZ4gKyIH5A= github.com/ameshkov/dnsstamps v1.0.1/go.mod h1:Ii3eUu73dx4Vw5O4wjzmT5+lkCwovjzaEZZ4gKyIH5A=
github.com/ameshkov/goproxy v0.0.0-20190328085534-e9f6fabc24d4/go.mod h1:tKA6C/1BQYejT7L6ZX0klDrqloYenYETv3BCk8xCbrQ=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6 h1:KXlsf+qt/X5ttPGEjR0tPH1xaWWoKBEg9Q1THAj2h3I= github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6 h1:KXlsf+qt/X5ttPGEjR0tPH1xaWWoKBEg9Q1THAj2h3I=
github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA=
github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0 h1:vUdUwmQLnT/yuk8PsDhhMVkrfr4aMdcv/0GWzIqOjEY= github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0 h1:vUdUwmQLnT/yuk8PsDhhMVkrfr4aMdcv/0GWzIqOjEY=
@ -22,8 +27,8 @@ github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0/go.mod h1:8c4/i2Vlov
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg=
github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/gobuffalo/envy v1.6.7 h1:XMZGuFqTupAXhZTriQ+qO38QvNOSU/0rl3hEPCFci/4= github.com/gobuffalo/envy v1.6.7 h1:XMZGuFqTupAXhZTriQ+qO38QvNOSU/0rl3hEPCFci/4=
@ -32,6 +37,9 @@ github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264 h1:roWyi0eEdiFreSq
github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264/go.mod h1:Yf2toFaISlyQrr5TfO3h6DB9pl9mZRmyvBGQb/aQ/pI= github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264/go.mod h1:Yf2toFaISlyQrr5TfO3h6DB9pl9mZRmyvBGQb/aQ/pI=
github.com/gobuffalo/packr v1.19.0 h1:3UDmBDxesCOPF8iZdMDBBWKfkBoYujIMIZePnobqIUI= github.com/gobuffalo/packr v1.19.0 h1:3UDmBDxesCOPF8iZdMDBBWKfkBoYujIMIZePnobqIUI=
github.com/gobuffalo/packr v1.19.0/go.mod h1:MstrNkfCQhd5o+Ct4IJ0skWlxN8emOq8DsoT1G98VIU= github.com/gobuffalo/packr v1.19.0/go.mod h1:MstrNkfCQhd5o+Ct4IJ0skWlxN8emOq8DsoT1G98VIU=
github.com/google/pprof v0.0.0-20190309163659-77426154d546/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/go-vhost v0.0.0-20160627193104-06d84117953b/go.mod h1:aA6DnFhALT3zH0y+A39we+zbrdMC2N0X/q21e6FI0LU=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@ -55,8 +63,9 @@ github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/shirou/gopsutil v2.18.10+incompatible h1:cy84jW6EVRPa5g9HAHrlbxMSIjBhDSX0OFYyMYminYs= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM=
github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 h1:mu7brOsdaH5Dqf93vdch+mr/0To8Sgc+yInt/jE/RJM= github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 h1:mu7brOsdaH5Dqf93vdch+mr/0To8Sgc+yInt/jE/RJM=
@ -70,15 +79,18 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= golang.org/x/arch v0.0.0-20190312162104-788fe5ffcd8c/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190122013713-64072686203f h1:u1CmMhe3a44hy8VIgpInORnI01UVaUYheqR7x9BxT3c= golang.org/x/crypto v0.0.0-20190122013713-64072686203f h1:u1CmMhe3a44hy8VIgpInORnI01UVaUYheqR7x9BxT3c=
golang.org/x/crypto v0.0.0-20190122013713-64072686203f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190122013713-64072686203f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190119204137-ed066c81e75e h1:MDa3fSUp6MdYHouVmCCNz/zaH2a6CRcxY3VhT/K3C5Q= golang.org/x/net v0.0.0-20190119204137-ed066c81e75e h1:MDa3fSUp6MdYHouVmCCNz/zaH2a6CRcxY3VhT/K3C5Q=
golang.org/x/net v0.0.0-20190119204137-ed066c81e75e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190119204137-ed066c81e75e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
@ -87,6 +99,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FY
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190122071731-054c452bb702 h1:Lk4tbZFnlyPgV+sLgTw5yGfzrlOn9kx4vSombi2FFlY= golang.org/x/sys v0.0.0-20190122071731-054c452bb702 h1:Lk4tbZFnlyPgV+sLgTw5yGfzrlOn9kx4vSombi2FFlY=
golang.org/x/sys v0.0.0-20190122071731-054c452bb702/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190122071731-054c452bb702/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477 h1:5xUJw+lg4zao9W4HIDzlFbMYgSgtvNVHh00MEHvbGpQ= gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477 h1:5xUJw+lg4zao9W4HIDzlFbMYgSgtvNVHh00MEHvbGpQ=
@ -95,3 +109,4 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=