2018-08-30 15:25:33 +01:00
|
|
|
package dnsfilter
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"io/ioutil"
|
2018-10-29 12:46:58 +00:00
|
|
|
"net"
|
2018-08-30 15:25:33 +01:00
|
|
|
"net/http"
|
2019-07-05 15:35:40 +01:00
|
|
|
"os"
|
2019-10-22 12:58:20 +01:00
|
|
|
"runtime"
|
2020-05-12 22:46:35 +01:00
|
|
|
"runtime/debug"
|
2018-08-30 15:25:33 +01:00
|
|
|
"strings"
|
2019-10-09 17:51:26 +01:00
|
|
|
"sync"
|
2019-06-18 14:18:13 +01:00
|
|
|
|
2020-03-20 12:05:43 +00:00
|
|
|
"github.com/AdguardTeam/AdGuardHome/util"
|
2019-04-18 12:31:13 +01:00
|
|
|
"github.com/AdguardTeam/dnsproxy/upstream"
|
2019-08-22 13:09:43 +01:00
|
|
|
"github.com/AdguardTeam/golibs/cache"
|
2019-02-25 13:44:22 +00:00
|
|
|
"github.com/AdguardTeam/golibs/log"
|
2019-05-15 14:46:11 +01:00
|
|
|
"github.com/AdguardTeam/urlfilter"
|
2019-11-27 12:11:46 +00:00
|
|
|
"github.com/AdguardTeam/urlfilter/filterlist"
|
|
|
|
"github.com/AdguardTeam/urlfilter/rules"
|
2019-05-22 10:38:17 +01:00
|
|
|
"github.com/miekg/dns"
|
2018-08-30 15:25:33 +01:00
|
|
|
)
|
|
|
|
|
2019-07-23 10:21:37 +01:00
|
|
|
// ServiceEntry - blocked service array element
|
|
|
|
type ServiceEntry struct {
|
|
|
|
Name string
|
2019-11-27 12:11:46 +00:00
|
|
|
Rules []*rules.NetworkRule
|
2019-07-23 10:21:37 +01:00
|
|
|
}
|
|
|
|
|
2019-07-29 09:37:16 +01:00
|
|
|
// RequestFilteringSettings is custom filtering settings
|
2019-05-28 12:14:12 +01:00
|
|
|
type RequestFilteringSettings struct {
|
|
|
|
FilteringEnabled bool
|
|
|
|
SafeSearchEnabled bool
|
|
|
|
SafeBrowsingEnabled bool
|
|
|
|
ParentalEnabled bool
|
2020-01-28 11:06:52 +00:00
|
|
|
ClientTags []string
|
2019-07-23 10:21:37 +01:00
|
|
|
ServicesRules []ServiceEntry
|
2019-05-28 12:14:12 +01:00
|
|
|
}
|
|
|
|
|
2018-11-30 10:32:51 +00:00
|
|
|
// Config allows you to configure DNS filtering with New() or just change variables directly.
|
|
|
|
type Config struct {
|
2019-07-04 12:00:20 +01:00
|
|
|
ParentalEnabled bool `yaml:"parental_enabled"`
|
|
|
|
SafeSearchEnabled bool `yaml:"safesearch_enabled"`
|
|
|
|
SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"`
|
2019-10-09 17:51:26 +01:00
|
|
|
ResolverAddress string `yaml:"-"` // DNS server address
|
2019-05-28 12:14:12 +01:00
|
|
|
|
2019-08-22 13:09:43 +01:00
|
|
|
SafeBrowsingCacheSize uint `yaml:"safebrowsing_cache_size"` // (in bytes)
|
|
|
|
SafeSearchCacheSize uint `yaml:"safesearch_cache_size"` // (in bytes)
|
|
|
|
ParentalCacheSize uint `yaml:"parental_cache_size"` // (in bytes)
|
|
|
|
CacheTime uint `yaml:"cache_time"` // Element's TTL (in minutes)
|
|
|
|
|
2019-07-29 09:37:16 +01:00
|
|
|
Rewrites []RewriteEntry `yaml:"rewrites"`
|
|
|
|
|
2020-02-18 17:17:35 +00:00
|
|
|
// Names of services to block (globally).
|
|
|
|
// Per-client settings can override this configuration.
|
|
|
|
BlockedServices []string `yaml:"blocked_services"`
|
|
|
|
|
2020-03-20 12:05:43 +00:00
|
|
|
// IP-hostname pairs taken from system configuration (e.g. /etc/hosts) files
|
|
|
|
AutoHosts *util.AutoHosts `yaml:"-"`
|
|
|
|
|
2019-10-09 17:51:26 +01:00
|
|
|
// Called when the configuration is changed by HTTP request
|
|
|
|
ConfigModified func() `yaml:"-"`
|
2018-11-30 10:32:51 +00:00
|
|
|
|
2019-10-09 17:51:26 +01:00
|
|
|
// Register an HTTP handler
|
|
|
|
HTTPRegister func(string, string, func(http.ResponseWriter, *http.Request)) `yaml:"-"`
|
2018-08-30 15:25:33 +01:00
|
|
|
}
|
|
|
|
|
2018-09-14 14:50:56 +01:00
|
|
|
// LookupStats store stats collected during safebrowsing or parental checks
|
2018-08-30 15:25:33 +01:00
|
|
|
type LookupStats struct {
|
|
|
|
Requests uint64 // number of HTTP requests that were sent
|
|
|
|
CacheHits uint64 // number of lookups that didn't need HTTP requests
|
|
|
|
Pending int64 // number of currently pending HTTP requests
|
|
|
|
PendingMax int64 // maximum number of pending HTTP requests
|
|
|
|
}
|
|
|
|
|
2019-02-22 13:34:36 +00:00
|
|
|
// Stats store LookupStats for safebrowsing, parental and safesearch
|
2018-08-30 15:25:33 +01:00
|
|
|
type Stats struct {
|
|
|
|
Safebrowsing LookupStats
|
|
|
|
Parental LookupStats
|
2019-02-22 13:34:36 +00:00
|
|
|
Safesearch LookupStats
|
2018-08-30 15:25:33 +01:00
|
|
|
}
|
|
|
|
|
2019-10-09 17:51:26 +01:00
|
|
|
// Parameters to pass to filters-initializer goroutine
|
|
|
|
type filtersInitializerParams struct {
|
2020-02-26 16:58:25 +00:00
|
|
|
allowFilters []Filter
|
|
|
|
blockFilters []Filter
|
2019-10-09 17:51:26 +01:00
|
|
|
}
|
|
|
|
|
2018-08-30 15:25:33 +01:00
|
|
|
// Dnsfilter holds added rules and performs hostname matches against the rules
|
|
|
|
type Dnsfilter struct {
|
2020-02-26 16:58:25 +00:00
|
|
|
rulesStorage *filterlist.RuleStorage
|
|
|
|
filteringEngine *urlfilter.DNSEngine
|
|
|
|
rulesStorageWhite *filterlist.RuleStorage
|
|
|
|
filteringEngineWhite *urlfilter.DNSEngine
|
|
|
|
engineLock sync.RWMutex
|
2019-05-15 14:46:11 +01:00
|
|
|
|
2019-10-16 10:57:49 +01:00
|
|
|
parentalServer string // access via methods
|
|
|
|
safeBrowsingServer string // access via methods
|
|
|
|
parentalUpstream upstream.Upstream
|
|
|
|
safeBrowsingUpstream upstream.Upstream
|
2019-10-09 17:51:26 +01:00
|
|
|
|
|
|
|
Config // for direct access by library users, even a = assignment
|
|
|
|
confLock sync.RWMutex
|
|
|
|
|
|
|
|
// Channel for passing data to filters-initializer goroutine
|
|
|
|
filtersInitializerChan chan filtersInitializerParams
|
|
|
|
filtersInitializerLock sync.Mutex
|
2018-08-30 15:25:33 +01:00
|
|
|
}
|
|
|
|
|
2019-01-24 17:11:01 +00:00
|
|
|
// Filter represents a filter list
|
2018-11-30 10:24:42 +00:00
|
|
|
type Filter struct {
|
2019-09-04 12:12:00 +01:00
|
|
|
ID int64 // auto-assigned when filter is added (see nextFilterID)
|
|
|
|
Data []byte `yaml:"-"` // List of rules divided by '\n'
|
|
|
|
FilePath string `yaml:"-"` // Path to a filtering rules file
|
2018-11-30 10:24:42 +00:00
|
|
|
}
|
|
|
|
|
2018-09-14 14:50:56 +01:00
|
|
|
// Reason holds an enum detailing why it was filtered or not filtered
|
2018-08-30 15:25:33 +01:00
|
|
|
type Reason int
|
|
|
|
|
|
|
|
const (
|
|
|
|
// reasons for not filtering
|
2019-01-24 17:11:01 +00:00
|
|
|
|
|
|
|
// NotFilteredNotFound - host was not find in any checks, default value for result
|
|
|
|
NotFilteredNotFound Reason = iota
|
|
|
|
// NotFilteredWhiteList - the host is explicitly whitelisted
|
|
|
|
NotFilteredWhiteList
|
|
|
|
// NotFilteredError - there was a transitive error during check
|
|
|
|
NotFilteredError
|
2018-08-30 15:25:33 +01:00
|
|
|
|
|
|
|
// reasons for filtering
|
2019-01-24 17:11:01 +00:00
|
|
|
|
|
|
|
// FilteredBlackList - the host was matched to be advertising host
|
|
|
|
FilteredBlackList
|
|
|
|
// FilteredSafeBrowsing - the host was matched to be malicious/phishing
|
|
|
|
FilteredSafeBrowsing
|
|
|
|
// FilteredParental - the host was matched to be outside of parental control settings
|
|
|
|
FilteredParental
|
|
|
|
// FilteredInvalid - the request was invalid and was not processed
|
|
|
|
FilteredInvalid
|
|
|
|
// FilteredSafeSearch - the host was replaced with safesearch variant
|
|
|
|
FilteredSafeSearch
|
2019-07-23 10:21:37 +01:00
|
|
|
// FilteredBlockedService - the host is blocked by "blocked services" settings
|
|
|
|
FilteredBlockedService
|
2019-07-29 09:37:16 +01:00
|
|
|
|
|
|
|
// ReasonRewrite - rewrite rule was applied
|
|
|
|
ReasonRewrite
|
2020-03-20 12:05:43 +00:00
|
|
|
|
|
|
|
// RewriteEtcHosts - rewrite by /etc/hosts rule
|
|
|
|
RewriteEtcHosts
|
2018-08-30 15:25:33 +01:00
|
|
|
)
|
|
|
|
|
2019-10-09 17:51:26 +01:00
|
|
|
var reasonNames = []string{
|
|
|
|
"NotFilteredNotFound",
|
|
|
|
"NotFilteredWhiteList",
|
|
|
|
"NotFilteredError",
|
|
|
|
|
|
|
|
"FilteredBlackList",
|
|
|
|
"FilteredSafeBrowsing",
|
|
|
|
"FilteredParental",
|
|
|
|
"FilteredInvalid",
|
|
|
|
"FilteredSafeSearch",
|
|
|
|
"FilteredBlockedService",
|
|
|
|
|
|
|
|
"Rewrite",
|
2020-03-20 12:05:43 +00:00
|
|
|
"RewriteEtcHosts",
|
2019-10-09 17:51:26 +01:00
|
|
|
}
|
|
|
|
|
2019-08-19 22:55:32 +01:00
|
|
|
func (r Reason) String() string {
|
2019-10-09 17:51:26 +01:00
|
|
|
if uint(r) >= uint(len(reasonNames)) {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
return reasonNames[r]
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetConfig - get configuration
|
|
|
|
func (d *Dnsfilter) GetConfig() RequestFilteringSettings {
|
|
|
|
c := RequestFilteringSettings{}
|
|
|
|
// d.confLock.RLock()
|
|
|
|
c.SafeSearchEnabled = d.Config.SafeSearchEnabled
|
|
|
|
c.SafeBrowsingEnabled = d.Config.SafeBrowsingEnabled
|
|
|
|
c.ParentalEnabled = d.Config.ParentalEnabled
|
|
|
|
// d.confLock.RUnlock()
|
|
|
|
return c
|
|
|
|
}
|
|
|
|
|
|
|
|
// WriteDiskConfig - write configuration
|
|
|
|
func (d *Dnsfilter) WriteDiskConfig(c *Config) {
|
2020-01-16 09:51:35 +00:00
|
|
|
d.confLock.Lock()
|
2019-10-09 17:51:26 +01:00
|
|
|
*c = d.Config
|
2020-01-16 09:51:35 +00:00
|
|
|
c.Rewrites = rewriteArrayDup(d.Config.Rewrites)
|
2020-02-18 17:17:35 +00:00
|
|
|
// BlockedServices
|
2020-01-16 09:51:35 +00:00
|
|
|
d.confLock.Unlock()
|
2019-10-09 17:51:26 +01:00
|
|
|
}
|
2019-07-29 09:37:16 +01:00
|
|
|
|
2019-10-09 17:51:26 +01:00
|
|
|
// SetFilters - set new filters (synchronously or asynchronously)
|
|
|
|
// When filters are set asynchronously, the old filters continue working until the new filters are ready.
|
|
|
|
// In this case the caller must ensure that the old filter files are intact.
|
2020-02-26 16:58:25 +00:00
|
|
|
func (d *Dnsfilter) SetFilters(blockFilters []Filter, allowFilters []Filter, async bool) error {
|
2019-10-09 17:51:26 +01:00
|
|
|
if async {
|
|
|
|
params := filtersInitializerParams{
|
2020-02-26 16:58:25 +00:00
|
|
|
allowFilters: allowFilters,
|
|
|
|
blockFilters: blockFilters,
|
2019-10-09 17:51:26 +01:00
|
|
|
}
|
2019-07-29 09:37:16 +01:00
|
|
|
|
2019-10-09 17:51:26 +01:00
|
|
|
d.filtersInitializerLock.Lock() // prevent multiple writers from adding more than 1 task
|
|
|
|
// remove all pending tasks
|
|
|
|
stop := false
|
|
|
|
for !stop {
|
|
|
|
select {
|
|
|
|
case <-d.filtersInitializerChan:
|
|
|
|
//
|
|
|
|
default:
|
|
|
|
stop = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
d.filtersInitializerChan <- params
|
|
|
|
d.filtersInitializerLock.Unlock()
|
|
|
|
return nil
|
2019-07-23 09:43:30 +01:00
|
|
|
}
|
2019-10-09 17:51:26 +01:00
|
|
|
|
2020-02-26 16:58:25 +00:00
|
|
|
err := d.initFiltering(allowFilters, blockFilters)
|
2019-10-09 17:51:26 +01:00
|
|
|
if err != nil {
|
|
|
|
log.Error("Can't initialize filtering subsystem: %s", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Starts initializing new filters by signal from channel
|
|
|
|
func (d *Dnsfilter) filtersInitializer() {
|
|
|
|
for {
|
|
|
|
params := <-d.filtersInitializerChan
|
2020-02-26 16:58:25 +00:00
|
|
|
err := d.initFiltering(params.allowFilters, params.blockFilters)
|
2019-10-09 17:51:26 +01:00
|
|
|
if err != nil {
|
|
|
|
log.Error("Can't initialize filtering subsystem: %s", err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close - close the object
|
|
|
|
func (d *Dnsfilter) Close() {
|
2020-03-05 10:12:21 +00:00
|
|
|
d.engineLock.Lock()
|
|
|
|
defer d.engineLock.Unlock()
|
2020-02-26 16:58:25 +00:00
|
|
|
d.reset()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Dnsfilter) reset() {
|
2019-10-09 17:51:26 +01:00
|
|
|
if d.rulesStorage != nil {
|
2020-02-05 11:30:43 +00:00
|
|
|
_ = d.rulesStorage.Close()
|
2019-07-23 09:43:30 +01:00
|
|
|
}
|
2020-02-26 16:58:25 +00:00
|
|
|
if d.rulesStorageWhite != nil {
|
|
|
|
d.rulesStorageWhite.Close()
|
|
|
|
}
|
2019-07-23 09:43:30 +01:00
|
|
|
}
|
|
|
|
|
2019-06-27 08:48:12 +01:00
|
|
|
type dnsFilterContext struct {
|
2018-08-30 15:25:33 +01:00
|
|
|
stats Stats
|
2019-08-22 13:09:43 +01:00
|
|
|
safebrowsingCache cache.Cache
|
|
|
|
parentalCache cache.Cache
|
|
|
|
safeSearchCache cache.Cache
|
2019-06-24 17:00:03 +01:00
|
|
|
}
|
|
|
|
|
2019-06-27 08:48:12 +01:00
|
|
|
var gctx dnsFilterContext // global dnsfilter context
|
2018-08-30 15:25:33 +01:00
|
|
|
|
2018-09-14 14:50:56 +01:00
|
|
|
// Result holds state of hostname check
|
2018-08-30 15:25:33 +01:00
|
|
|
type Result struct {
|
2018-10-29 12:46:58 +00:00
|
|
|
IsFiltered bool `json:",omitempty"` // True if the host name is filtered
|
|
|
|
Reason Reason `json:",omitempty"` // Reason for blocking / unblocking
|
|
|
|
Rule string `json:",omitempty"` // Original rule text
|
2019-01-24 17:11:01 +00:00
|
|
|
IP net.IP `json:",omitempty"` // Not nil only in the case of a hosts file syntax
|
2018-10-30 14:16:20 +00:00
|
|
|
FilterID int64 `json:",omitempty"` // Filter ID the rule belongs to
|
2019-07-29 09:37:16 +01:00
|
|
|
|
|
|
|
// for ReasonRewrite:
|
2020-04-16 16:56:47 +01:00
|
|
|
CanonName string `json:",omitempty"` // CNAME value
|
|
|
|
|
|
|
|
// for RewriteEtcHosts:
|
|
|
|
ReverseHost string `json:",omitempty"`
|
|
|
|
|
|
|
|
// for ReasonRewrite & RewriteEtcHosts:
|
|
|
|
IPList []net.IP `json:",omitempty"` // list of IP addresses
|
2019-07-23 10:21:37 +01:00
|
|
|
|
|
|
|
// for FilteredBlockedService:
|
|
|
|
ServiceName string `json:",omitempty"` // Name of the blocked service
|
2018-08-30 15:25:33 +01:00
|
|
|
}
|
|
|
|
|
2018-09-14 14:50:56 +01:00
|
|
|
// Matched can be used to see if any match at all was found, no matter filtered or not
|
2018-08-30 15:25:33 +01:00
|
|
|
func (r Reason) Matched() bool {
|
|
|
|
return r != NotFilteredNotFound
|
|
|
|
}
|
|
|
|
|
2019-12-23 12:59:49 +00:00
|
|
|
// CheckHostRules tries to match the host against filtering rules only
|
|
|
|
func (d *Dnsfilter) CheckHostRules(host string, qtype uint16, setts *RequestFilteringSettings) (Result, error) {
|
|
|
|
if !setts.FilteringEnabled {
|
|
|
|
return Result{}, nil
|
|
|
|
}
|
|
|
|
|
2020-01-28 11:06:52 +00:00
|
|
|
return d.matchHost(host, qtype, setts.ClientTags)
|
2019-12-23 12:59:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// CheckHost tries to match the host against filtering rules,
|
|
|
|
// then safebrowsing and parental if they are enabled
|
2019-07-25 14:37:06 +01:00
|
|
|
func (d *Dnsfilter) CheckHost(host string, qtype uint16, setts *RequestFilteringSettings) (Result, error) {
|
2018-10-05 05:31:56 +01:00
|
|
|
// sometimes DNS clients will try to resolve ".", which is a request to get root servers
|
2018-08-30 15:25:33 +01:00
|
|
|
if host == "" {
|
2018-10-05 05:31:56 +01:00
|
|
|
return Result{Reason: NotFilteredNotFound}, nil
|
2018-08-30 15:25:33 +01:00
|
|
|
}
|
2018-09-10 18:34:42 +01:00
|
|
|
host = strings.ToLower(host)
|
2018-08-30 15:25:33 +01:00
|
|
|
|
2019-05-28 12:14:12 +01:00
|
|
|
var result Result
|
|
|
|
var err error
|
2019-07-29 09:37:16 +01:00
|
|
|
|
2020-03-02 08:49:26 +00:00
|
|
|
result = d.processRewrites(host)
|
2019-07-29 09:37:16 +01:00
|
|
|
if result.Reason == ReasonRewrite {
|
|
|
|
return result, nil
|
|
|
|
}
|
|
|
|
|
2020-03-20 12:05:43 +00:00
|
|
|
if d.Config.AutoHosts != nil {
|
2020-04-16 16:56:47 +01:00
|
|
|
ips := d.Config.AutoHosts.Process(host, qtype)
|
2020-03-20 12:05:43 +00:00
|
|
|
if ips != nil {
|
|
|
|
result.Reason = RewriteEtcHosts
|
|
|
|
result.IPList = ips
|
|
|
|
return result, nil
|
|
|
|
}
|
2020-04-16 16:56:47 +01:00
|
|
|
|
|
|
|
revHost := d.Config.AutoHosts.ProcessReverse(host, qtype)
|
|
|
|
if len(revHost) != 0 {
|
|
|
|
result.Reason = RewriteEtcHosts
|
|
|
|
result.ReverseHost = revHost + "."
|
|
|
|
return result, nil
|
|
|
|
}
|
2020-03-20 12:05:43 +00:00
|
|
|
}
|
|
|
|
|
2019-05-28 12:14:12 +01:00
|
|
|
// try filter lists first
|
|
|
|
if setts.FilteringEnabled {
|
2020-01-28 11:06:52 +00:00
|
|
|
result, err = d.matchHost(host, qtype, setts.ClientTags)
|
2019-05-28 12:14:12 +01:00
|
|
|
if err != nil {
|
|
|
|
return result, err
|
|
|
|
}
|
|
|
|
if result.Reason.Matched() {
|
|
|
|
return result, nil
|
|
|
|
}
|
2018-08-30 15:25:33 +01:00
|
|
|
}
|
|
|
|
|
2019-07-23 10:21:37 +01:00
|
|
|
if len(setts.ServicesRules) != 0 {
|
|
|
|
result = matchBlockedServicesRules(host, setts.ServicesRules)
|
|
|
|
if result.Reason.Matched() {
|
|
|
|
return result, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-28 12:14:12 +01:00
|
|
|
if setts.SafeSearchEnabled {
|
2019-02-22 13:34:36 +00:00
|
|
|
result, err = d.checkSafeSearch(host)
|
|
|
|
if err != nil {
|
2019-10-16 10:57:49 +01:00
|
|
|
log.Info("SafeSearch: failed: %v", err)
|
2019-02-22 13:34:36 +00:00
|
|
|
return Result{}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
if result.Reason.Matched() {
|
|
|
|
return result, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-28 12:14:12 +01:00
|
|
|
if setts.SafeBrowsingEnabled {
|
2018-08-30 15:25:33 +01:00
|
|
|
result, err = d.checkSafeBrowsing(host)
|
|
|
|
if err != nil {
|
2019-10-16 10:57:49 +01:00
|
|
|
log.Info("SafeBrowsing: failed: %v", err)
|
2018-08-30 15:25:33 +01:00
|
|
|
return Result{}, nil
|
|
|
|
}
|
|
|
|
if result.Reason.Matched() {
|
|
|
|
return result, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-28 12:14:12 +01:00
|
|
|
if setts.ParentalEnabled {
|
2018-08-30 15:25:33 +01:00
|
|
|
result, err = d.checkParental(host)
|
|
|
|
if err != nil {
|
2019-10-16 10:57:49 +01:00
|
|
|
log.Printf("Parental: failed: %v", err)
|
2018-08-30 15:25:33 +01:00
|
|
|
return Result{}, nil
|
|
|
|
}
|
|
|
|
if result.Reason.Matched() {
|
|
|
|
return result, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return Result{}, nil
|
|
|
|
}
|
|
|
|
|
2019-07-29 09:37:16 +01:00
|
|
|
// Process rewrites table
|
2020-01-16 09:51:35 +00:00
|
|
|
// . Find CNAME for a domain name (exact match or by wildcard)
|
2019-07-29 09:37:16 +01:00
|
|
|
// . if found, set domain name to canonical name
|
2020-01-16 09:51:35 +00:00
|
|
|
// . repeat for the new domain name (Note: we return only the last CNAME)
|
|
|
|
// . Find A or AAAA record for a domain name (exact match or by wildcard)
|
2020-03-02 08:49:26 +00:00
|
|
|
// . if found, return IP addresses (both IPv4 and IPv6)
|
|
|
|
func (d *Dnsfilter) processRewrites(host string) Result {
|
2019-07-29 09:37:16 +01:00
|
|
|
var res Result
|
|
|
|
|
2019-10-09 17:51:26 +01:00
|
|
|
d.confLock.RLock()
|
|
|
|
defer d.confLock.RUnlock()
|
|
|
|
|
2020-01-16 09:51:35 +00:00
|
|
|
rr := findRewrites(d.Rewrites, host)
|
|
|
|
if len(rr) != 0 {
|
|
|
|
res.Reason = ReasonRewrite
|
2019-07-29 09:37:16 +01:00
|
|
|
}
|
|
|
|
|
2020-01-16 09:51:35 +00:00
|
|
|
cnames := map[string]bool{}
|
|
|
|
origHost := host
|
|
|
|
for len(rr) != 0 && rr[0].Type == dns.TypeCNAME {
|
|
|
|
log.Debug("Rewrite: CNAME for %s is %s", host, rr[0].Answer)
|
|
|
|
host = rr[0].Answer
|
|
|
|
_, ok := cnames[host]
|
|
|
|
if ok {
|
|
|
|
log.Info("Rewrite: breaking CNAME redirection loop: %s. Question: %s", host, origHost)
|
|
|
|
return res
|
2019-07-29 09:37:16 +01:00
|
|
|
}
|
2020-01-16 09:51:35 +00:00
|
|
|
cnames[host] = false
|
|
|
|
res.CanonName = rr[0].Answer
|
|
|
|
rr = findRewrites(d.Rewrites, host)
|
2019-07-29 09:37:16 +01:00
|
|
|
}
|
|
|
|
|
2020-01-16 09:51:35 +00:00
|
|
|
for _, r := range rr {
|
2020-03-02 08:49:26 +00:00
|
|
|
if r.Type != dns.TypeCNAME {
|
2020-01-16 09:51:35 +00:00
|
|
|
res.IPList = append(res.IPList, r.IP)
|
|
|
|
log.Debug("Rewrite: A/AAAA for %s is %s", host, r.IP)
|
|
|
|
}
|
2019-07-29 09:37:16 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return res
|
|
|
|
}
|
|
|
|
|
2019-07-23 10:21:37 +01:00
|
|
|
func matchBlockedServicesRules(host string, svcs []ServiceEntry) Result {
|
2019-11-27 12:11:46 +00:00
|
|
|
req := rules.NewRequestForHostname(host)
|
2019-07-23 10:21:37 +01:00
|
|
|
res := Result{}
|
|
|
|
|
|
|
|
for _, s := range svcs {
|
|
|
|
for _, rule := range s.Rules {
|
|
|
|
if rule.Match(req) {
|
|
|
|
res.Reason = FilteredBlockedService
|
|
|
|
res.IsFiltered = true
|
|
|
|
res.ServiceName = s.Name
|
|
|
|
res.Rule = rule.Text()
|
|
|
|
log.Debug("Blocked Services: matched rule: %s host: %s service: %s",
|
|
|
|
res.Rule, host, s.Name)
|
|
|
|
return res
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return res
|
|
|
|
}
|
|
|
|
|
2018-08-30 15:25:33 +01:00
|
|
|
//
|
|
|
|
// Adding rule and matching against the rules
|
|
|
|
//
|
|
|
|
|
2019-07-05 15:35:40 +01:00
|
|
|
// Return TRUE if file exists
|
|
|
|
func fileExists(fn string) bool {
|
|
|
|
_, err := os.Stat(fn)
|
|
|
|
if err != nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2020-02-26 16:58:25 +00:00
|
|
|
func createFilteringEngine(filters []Filter) (*filterlist.RuleStorage, *urlfilter.DNSEngine, error) {
|
2019-11-27 12:11:46 +00:00
|
|
|
listArray := []filterlist.RuleList{}
|
2020-02-26 16:58:25 +00:00
|
|
|
for _, f := range filters {
|
2019-11-27 12:11:46 +00:00
|
|
|
var list filterlist.RuleList
|
2019-07-05 15:35:40 +01:00
|
|
|
|
2020-02-26 16:58:25 +00:00
|
|
|
if f.ID == 0 {
|
2019-11-27 12:11:46 +00:00
|
|
|
list = &filterlist.StringRuleList{
|
2019-07-04 12:00:20 +01:00
|
|
|
ID: 0,
|
2020-02-26 16:58:25 +00:00
|
|
|
RulesText: string(f.Data),
|
2019-10-22 12:58:20 +01:00
|
|
|
IgnoreCosmetic: true,
|
2019-07-04 12:00:20 +01:00
|
|
|
}
|
2019-07-05 15:35:40 +01:00
|
|
|
|
2020-02-26 16:58:25 +00:00
|
|
|
} else if !fileExists(f.FilePath) {
|
2019-11-27 12:11:46 +00:00
|
|
|
list = &filterlist.StringRuleList{
|
2020-02-26 16:58:25 +00:00
|
|
|
ID: int(f.ID),
|
2019-10-22 12:58:20 +01:00
|
|
|
IgnoreCosmetic: true,
|
|
|
|
}
|
|
|
|
|
|
|
|
} else if runtime.GOOS == "windows" {
|
|
|
|
// On Windows we don't pass a file to urlfilter because
|
|
|
|
// it's difficult to update this file while it's being used.
|
2020-02-26 16:58:25 +00:00
|
|
|
data, err := ioutil.ReadFile(f.FilePath)
|
2019-10-22 12:58:20 +01:00
|
|
|
if err != nil {
|
2020-02-26 16:58:25 +00:00
|
|
|
return nil, nil, fmt.Errorf("ioutil.ReadFile(): %s: %s", f.FilePath, err)
|
2019-10-22 12:58:20 +01:00
|
|
|
}
|
2019-11-27 12:11:46 +00:00
|
|
|
list = &filterlist.StringRuleList{
|
2020-02-26 16:58:25 +00:00
|
|
|
ID: int(f.ID),
|
2019-10-22 12:58:20 +01:00
|
|
|
RulesText: string(data),
|
|
|
|
IgnoreCosmetic: true,
|
2019-07-05 15:35:40 +01:00
|
|
|
}
|
|
|
|
|
2019-07-04 12:00:20 +01:00
|
|
|
} else {
|
|
|
|
var err error
|
2020-02-26 16:58:25 +00:00
|
|
|
list, err = filterlist.NewFileRuleList(int(f.ID), f.FilePath, true)
|
2019-07-04 12:00:20 +01:00
|
|
|
if err != nil {
|
2020-02-26 16:58:25 +00:00
|
|
|
return nil, nil, fmt.Errorf("filterlist.NewFileRuleList(): %s: %s", f.FilePath, err)
|
2019-07-04 12:00:20 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
listArray = append(listArray, list)
|
|
|
|
}
|
|
|
|
|
2019-11-27 12:11:46 +00:00
|
|
|
rulesStorage, err := filterlist.NewRuleStorage(listArray)
|
2019-05-15 14:46:11 +01:00
|
|
|
if err != nil {
|
2020-02-26 16:58:25 +00:00
|
|
|
return nil, nil, fmt.Errorf("filterlist.NewRuleStorage(): %s", err)
|
2019-05-15 14:46:11 +01:00
|
|
|
}
|
2019-10-09 17:51:26 +01:00
|
|
|
filteringEngine := urlfilter.NewDNSEngine(rulesStorage)
|
2020-02-26 16:58:25 +00:00
|
|
|
return rulesStorage, filteringEngine, nil
|
|
|
|
}
|
2019-10-09 17:51:26 +01:00
|
|
|
|
2020-02-26 16:58:25 +00:00
|
|
|
// Initialize urlfilter objects
|
|
|
|
func (d *Dnsfilter) initFiltering(allowFilters, blockFilters []Filter) error {
|
2019-10-09 17:51:26 +01:00
|
|
|
d.engineLock.Lock()
|
2020-03-05 10:12:21 +00:00
|
|
|
defer d.engineLock.Unlock()
|
2020-02-26 16:58:25 +00:00
|
|
|
d.reset()
|
|
|
|
rulesStorage, filteringEngine, err := createFilteringEngine(blockFilters)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
rulesStorageWhite, filteringEngineWhite, err := createFilteringEngine(allowFilters)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
2019-10-09 17:51:26 +01:00
|
|
|
}
|
|
|
|
d.rulesStorage = rulesStorage
|
|
|
|
d.filteringEngine = filteringEngine
|
2020-02-26 16:58:25 +00:00
|
|
|
d.rulesStorageWhite = rulesStorageWhite
|
|
|
|
d.filteringEngineWhite = filteringEngineWhite
|
2020-05-12 22:46:35 +01:00
|
|
|
|
|
|
|
// Make sure that the OS reclaims memory as soon as possible
|
|
|
|
debug.FreeOSMemory()
|
2019-10-09 17:51:26 +01:00
|
|
|
log.Debug("initialized filtering engine")
|
|
|
|
|
2018-11-30 10:48:53 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-08-30 15:25:33 +01:00
|
|
|
// matchHost is a low-level way to check only if hostname is filtered by rules, skipping expensive safebrowsing and parental lookups
|
2020-01-28 11:06:52 +00:00
|
|
|
func (d *Dnsfilter) matchHost(host string, qtype uint16, ctags []string) (Result, error) {
|
2019-10-09 17:51:26 +01:00
|
|
|
d.engineLock.RLock()
|
2020-01-30 16:06:09 +00:00
|
|
|
// Keep in mind that this lock must be held no just when calling Match()
|
|
|
|
// but also while using the rules returned by it.
|
2019-10-09 17:51:26 +01:00
|
|
|
defer d.engineLock.RUnlock()
|
2020-02-26 16:58:25 +00:00
|
|
|
|
|
|
|
if d.filteringEngineWhite != nil {
|
|
|
|
rr, ok := d.filteringEngineWhite.Match(host, ctags)
|
|
|
|
if ok {
|
|
|
|
var rule rules.Rule
|
|
|
|
if rr.NetworkRule != nil {
|
|
|
|
rule = rr.NetworkRule
|
|
|
|
} else if rr.HostRulesV4 != nil {
|
|
|
|
rule = rr.HostRulesV4[0]
|
|
|
|
} else if rr.HostRulesV6 != nil {
|
|
|
|
rule = rr.HostRulesV6[0]
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Debug("Filtering: found whitelist rule for host '%s': '%s' list_id: %d",
|
|
|
|
host, rule.Text(), rule.GetFilterListID())
|
|
|
|
res := makeResult(rule, NotFilteredWhiteList)
|
|
|
|
return res, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-15 14:46:11 +01:00
|
|
|
if d.filteringEngine == nil {
|
|
|
|
return Result{}, nil
|
|
|
|
}
|
|
|
|
|
2020-01-30 16:06:09 +00:00
|
|
|
rr, ok := d.filteringEngine.Match(host, ctags)
|
2019-05-15 14:46:11 +01:00
|
|
|
if !ok {
|
|
|
|
return Result{}, nil
|
|
|
|
}
|
|
|
|
|
2020-01-30 16:06:09 +00:00
|
|
|
if rr.NetworkRule != nil {
|
|
|
|
log.Debug("Filtering: found rule for host '%s': '%s' list_id: %d",
|
|
|
|
host, rr.NetworkRule.Text(), rr.NetworkRule.GetFilterListID())
|
2020-02-26 16:58:25 +00:00
|
|
|
reason := FilteredBlackList
|
2020-01-30 16:06:09 +00:00
|
|
|
if rr.NetworkRule.Whitelist {
|
2020-02-26 16:58:25 +00:00
|
|
|
reason = NotFilteredWhiteList
|
2020-01-30 16:06:09 +00:00
|
|
|
}
|
2020-02-26 16:58:25 +00:00
|
|
|
res := makeResult(rr.NetworkRule, reason)
|
2020-01-30 16:06:09 +00:00
|
|
|
return res, nil
|
|
|
|
}
|
2019-05-15 14:46:11 +01:00
|
|
|
|
2020-01-30 16:06:09 +00:00
|
|
|
if qtype == dns.TypeA && rr.HostRulesV4 != nil {
|
|
|
|
rule := rr.HostRulesV4[0] // note that we process only 1 matched rule
|
2020-02-26 16:58:25 +00:00
|
|
|
log.Debug("Filtering: found rule for host '%s': '%s' list_id: %d",
|
|
|
|
host, rule.Text(), rule.GetFilterListID())
|
|
|
|
res := makeResult(rule, FilteredBlackList)
|
2020-01-30 16:06:09 +00:00
|
|
|
res.IP = rule.IP.To4()
|
|
|
|
return res, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
if qtype == dns.TypeAAAA && rr.HostRulesV6 != nil {
|
|
|
|
rule := rr.HostRulesV6[0] // note that we process only 1 matched rule
|
2020-02-26 16:58:25 +00:00
|
|
|
log.Debug("Filtering: found rule for host '%s': '%s' list_id: %d",
|
|
|
|
host, rule.Text(), rule.GetFilterListID())
|
|
|
|
res := makeResult(rule, FilteredBlackList)
|
2020-01-30 16:06:09 +00:00
|
|
|
res.IP = rule.IP
|
|
|
|
return res, nil
|
|
|
|
}
|
2019-05-15 14:46:11 +01:00
|
|
|
|
2020-01-30 16:06:09 +00:00
|
|
|
if rr.HostRulesV4 != nil || rr.HostRulesV6 != nil {
|
|
|
|
// Question Type doesn't match the host rules
|
|
|
|
// Return the first matched host rule, but without an IP address
|
|
|
|
var rule rules.Rule
|
|
|
|
if rr.HostRulesV4 != nil {
|
|
|
|
rule = rr.HostRulesV4[0]
|
|
|
|
} else if rr.HostRulesV6 != nil {
|
|
|
|
rule = rr.HostRulesV6[0]
|
2019-05-15 14:46:11 +01:00
|
|
|
}
|
2020-02-26 16:58:25 +00:00
|
|
|
log.Debug("Filtering: found rule for host '%s': '%s' list_id: %d",
|
|
|
|
host, rule.Text(), rule.GetFilterListID())
|
|
|
|
res := makeResult(rule, FilteredBlackList)
|
2020-01-30 16:06:09 +00:00
|
|
|
res.IP = net.IP{}
|
|
|
|
return res, nil
|
2019-05-15 14:46:11 +01:00
|
|
|
}
|
|
|
|
|
2018-08-30 15:25:33 +01:00
|
|
|
return Result{}, nil
|
|
|
|
}
|
|
|
|
|
2020-02-26 16:58:25 +00:00
|
|
|
// Construct Result object
|
|
|
|
func makeResult(rule rules.Rule, reason Reason) Result {
|
|
|
|
res := Result{}
|
|
|
|
res.FilterID = int64(rule.GetFilterListID())
|
|
|
|
res.Rule = rule.Text()
|
|
|
|
res.Reason = reason
|
|
|
|
if reason == FilteredBlackList {
|
|
|
|
res.IsFiltered = true
|
|
|
|
}
|
|
|
|
return res
|
|
|
|
}
|
|
|
|
|
2020-04-27 11:21:16 +01:00
|
|
|
// InitModule() - manually initialize blocked services map
|
|
|
|
func InitModule() {
|
|
|
|
initBlockedServices()
|
|
|
|
}
|
|
|
|
|
2018-09-14 14:50:56 +01:00
|
|
|
// New creates properly initialized DNS Filter that is ready to be used
|
2020-02-26 16:58:25 +00:00
|
|
|
func New(c *Config, blockFilters []Filter) *Dnsfilter {
|
2019-06-24 17:00:03 +01:00
|
|
|
|
|
|
|
if c != nil {
|
2019-08-22 13:09:43 +01:00
|
|
|
cacheConf := cache.Config{
|
|
|
|
EnableLRU: true,
|
|
|
|
}
|
|
|
|
|
2019-06-24 17:00:03 +01:00
|
|
|
// initialize objects only once
|
2019-08-22 13:09:43 +01:00
|
|
|
|
2019-07-15 10:10:43 +01:00
|
|
|
if gctx.safebrowsingCache == nil {
|
2019-08-22 13:09:43 +01:00
|
|
|
cacheConf.MaxSize = c.SafeBrowsingCacheSize
|
|
|
|
gctx.safebrowsingCache = cache.New(cacheConf)
|
2019-06-24 17:00:03 +01:00
|
|
|
}
|
2019-08-22 13:09:43 +01:00
|
|
|
|
2019-07-15 10:10:43 +01:00
|
|
|
if gctx.safeSearchCache == nil {
|
2019-08-22 13:09:43 +01:00
|
|
|
cacheConf.MaxSize = c.SafeSearchCacheSize
|
|
|
|
gctx.safeSearchCache = cache.New(cacheConf)
|
2019-06-24 17:00:03 +01:00
|
|
|
}
|
2019-08-22 13:09:43 +01:00
|
|
|
|
2019-07-15 10:10:43 +01:00
|
|
|
if gctx.parentalCache == nil {
|
2019-08-22 13:09:43 +01:00
|
|
|
cacheConf.MaxSize = c.ParentalCacheSize
|
|
|
|
gctx.parentalCache = cache.New(cacheConf)
|
2019-06-24 17:00:03 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-30 15:25:33 +01:00
|
|
|
d := new(Dnsfilter)
|
|
|
|
|
2019-10-16 10:57:49 +01:00
|
|
|
err := d.initSecurityServices()
|
|
|
|
if err != nil {
|
|
|
|
log.Error("dnsfilter: initialize services: %s", err)
|
|
|
|
return nil
|
2018-08-30 15:25:33 +01:00
|
|
|
}
|
2019-10-16 10:57:49 +01:00
|
|
|
|
2018-11-30 10:47:26 +00:00
|
|
|
if c != nil {
|
|
|
|
d.Config = *c
|
2020-01-16 09:51:35 +00:00
|
|
|
d.prepareRewrites()
|
2018-11-30 10:47:26 +00:00
|
|
|
}
|
2018-09-10 18:34:42 +01:00
|
|
|
|
2020-02-18 17:17:35 +00:00
|
|
|
bsvcs := []string{}
|
|
|
|
for _, s := range d.BlockedServices {
|
|
|
|
if !BlockedSvcKnown(s) {
|
|
|
|
log.Debug("skipping unknown blocked-service '%s'", s)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
bsvcs = append(bsvcs, s)
|
|
|
|
}
|
|
|
|
d.BlockedServices = bsvcs
|
|
|
|
|
2020-02-26 16:58:25 +00:00
|
|
|
if blockFilters != nil {
|
|
|
|
err := d.initFiltering(nil, blockFilters)
|
2019-05-15 14:46:11 +01:00
|
|
|
if err != nil {
|
|
|
|
log.Error("Can't initialize filtering subsystem: %s", err)
|
2019-10-09 17:51:26 +01:00
|
|
|
d.Close()
|
2019-05-15 14:46:11 +01:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-16 11:25:40 +00:00
|
|
|
return d
|
|
|
|
}
|
|
|
|
|
2020-01-30 16:06:09 +00:00
|
|
|
// Start - start the module:
|
|
|
|
// . start async filtering initializer goroutine
|
|
|
|
// . register web handlers
|
2020-01-16 11:25:40 +00:00
|
|
|
func (d *Dnsfilter) Start() {
|
2019-10-09 17:51:26 +01:00
|
|
|
d.filtersInitializerChan = make(chan filtersInitializerParams, 1)
|
|
|
|
go d.filtersInitializer()
|
2018-08-30 15:25:33 +01:00
|
|
|
|
2019-10-09 17:51:26 +01:00
|
|
|
if d.Config.HTTPRegister != nil { // for tests
|
|
|
|
d.registerSecurityHandlers()
|
|
|
|
d.registerRewritesHandlers()
|
2020-02-18 17:17:35 +00:00
|
|
|
d.registerBlockedServicesHandlers()
|
2019-05-15 14:46:11 +01:00
|
|
|
}
|
2018-08-30 15:25:33 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// stats
|
|
|
|
//
|
|
|
|
|
2018-09-14 14:50:56 +01:00
|
|
|
// GetStats return dns filtering stats since startup
|
2018-08-30 15:25:33 +01:00
|
|
|
func (d *Dnsfilter) GetStats() Stats {
|
2019-06-24 17:00:03 +01:00
|
|
|
return gctx.stats
|
2018-08-30 15:25:33 +01:00
|
|
|
}
|