From ff04b2a7d3fc2ed75f63b14bdec4f2f5563c36c4 Mon Sep 17 00:00:00 2001 From: Stanislav Chzhen Date: Mon, 13 Feb 2023 18:15:33 +0300 Subject: [PATCH] Pull request 1731: 4299-stats-ignore Merge in DNS/adguard-home from 4299-stats-ignore to master Updates #1717. Updates #4299. Squashed commit of the following: commit 1d1212d088c944e995deae2fd599eccb0a075033 Author: Stanislav Chzhen Date: Mon Feb 13 17:53:36 2023 +0300 fix changelog commit 5f56852c21d794bd87c13192d3857757be10f9b2 Author: Stanislav Chzhen Date: Mon Feb 13 17:39:02 2023 +0300 add todo; fix data race commit 89b8b16ddf5a43ebf68174cbaf9e8a53365f8cbe Merge: e0a6bb49 ec19a85e Author: Stanislav Chzhen Date: Fri Feb 10 17:21:38 2023 +0300 Merge branch 'master' into 4299-stats-ignore commit e0a6bb490b651d1cf31589a7f17095fff4cb4dbb Author: Stanislav Chzhen Date: Fri Feb 10 17:21:06 2023 +0300 interval under mutex commit c569c7bc237f11b23fe47c98a20a1c5cb36751cb Author: Stanislav Chzhen Date: Fri Feb 10 16:19:35 2023 +0300 fix mutex commit 9374cf0c54dccc2fbfc38765b52c64e1c479137c Author: Stanislav Chzhen Date: Fri Feb 10 16:03:17 2023 +0300 fix typo commit 1f4fd1e7ab1b3c2f8e9c3d32ef7e4958f99abb47 Author: Stanislav Chzhen Date: Fri Feb 10 15:55:44 2023 +0300 add mutex commit 2148048ce9ad228381cbb51a806c9b9cc21458fd Author: Stanislav Chzhen Date: Fri Feb 10 12:27:36 2023 +0300 add key check commit a19350977c463f888aea70d0dace26dff0173a65 Author: Stanislav Chzhen Date: Thu Feb 9 18:34:36 2023 +0300 fix changelog commit 23c3b6da162dfd513884b460c265ba4cafeb9727 Merge: 8fccc0b8 b89105e3 Author: Stanislav Chzhen Date: Thu Feb 9 13:28:59 2023 +0300 Merge branch 'master' into 4299-stats-ignore commit 8fccc0b8ec670a37e5209d795f35c43dd64afeb3 Author: Stanislav Chzhen Date: Thu Feb 9 13:27:42 2023 +0300 add changelog commit 0416c71742795b2fb8adb0173dcd6a99d9d9c676 Author: Stanislav Chzhen Date: Wed Feb 8 14:31:55 2023 +0300 all: stats ignore --- CHANGELOG.md | 30 +++++++-- internal/dnsforward/stats.go | 9 ++- internal/dnsforward/stats_test.go | 13 ++-- internal/home/config.go | 40 ++++++++---- internal/home/dns.go | 46 +++++++++++--- internal/home/upgrade.go | 55 +++++++++++++++- internal/home/upgrade_test.go | 59 ++++++++++++++++++ internal/querylog/http.go | 3 + internal/querylog/querylog.go | 4 +- internal/querylog/querylogfile.go | 3 + internal/stats/http.go | 14 +++-- internal/stats/stats.go | 100 ++++++++++++++++++++++-------- internal/stats/stats_test.go | 2 + internal/stats/unit.go | 13 ++-- 14 files changed, 320 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47168a00..6a23d483 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,16 +25,38 @@ NOTE: Add new changes BELOW THIS COMMENT. ### Added -- The ability to exclude domain names from the query log by using the new - `querylog.ignored` field ([#1717], [#4299]). +- The ability to disable statistics by using the new `statistics.enabled` + field. Previously it was necessary to set the `statistics_interval` to 0, + losing the previous value ([#1717], [#4299]). +- The ability to exclude domain names from the query log or statistics by using + the new `querylog.ignored` or `statistics.ignored` fields ([#1717], [#4299]). ### Changed #### Configuration Changes -In this release, the schema version has changed from 14 to 15. +In this release, the schema version has changed from 14 to 16. -- The fields `dns.…` have been moved to the new `querylog` object. +- Property `statistics_interval`, which in schema versions 15 and earlier used + to be a part of the `dns` object, is now a part of the `statistics` object: + + ```yaml + # BEFORE: + 'dns': + # … + 'statistics_interval': 1 + + # AFTER: + 'statistics': + # … + 'interval': 1 + ``` + + To rollback this change, move the property back into the `dns` object and + change the `schema_version` back to `15`. +- The fields `dns.querylog_enabled`, `dns.querylog_file_enabled`, + `dns.querylog_interval`, `dns.querylog_size_memory` have been moved to the + new `querylog` object. ```yaml # BEFORE: diff --git a/internal/dnsforward/stats.go b/internal/dnsforward/stats.go index 1e1f90d0..3fb6e245 100644 --- a/internal/dnsforward/stats.go +++ b/internal/dnsforward/stats.go @@ -48,10 +48,15 @@ func (s *Server) processQueryLogsAndStats(dctx *dnsContext) (rc resultCode) { s.queryLog.ShouldLog(host, q.Qtype, q.Qclass) { s.logQuery(dctx, pctx, elapsed, ip) } else { - log.Debug("request for %s from %s ignored; not logging", host, ip) + log.Debug( + "dnsforward: request %s %s from %s ignored; not logging", + dns.Type(q.Qtype), + host, + ip, + ) } - if s.stats != nil { + if s.stats != nil && s.stats.ShouldCount(host, q.Qtype, q.Qclass) { s.updateStats(dctx, elapsed, *dctx.result, ip) } diff --git a/internal/dnsforward/stats_test.go b/internal/dnsforward/stats_test.go index 17e12b46..6fa82a0e 100644 --- a/internal/dnsforward/stats_test.go +++ b/internal/dnsforward/stats_test.go @@ -35,20 +35,25 @@ func (l *testQueryLog) ShouldLog(string, uint16, uint16) bool { return true } -// testStats is a simple stats.Stats implementation for tests. +// testStats is a simple [stats.Interface] implementation for tests. type testStats struct { - // Stats is embedded here simply to make testStats a stats.Stats without - // actually implementing all methods. + // Stats is embedded here simply to make testStats a [stats.Interface] + // without actually implementing all methods. stats.Interface lastEntry stats.Entry } -// Update implements the stats.Stats interface for *testStats. +// Update implements the [stats.Interface] interface for *testStats. func (l *testStats) Update(e stats.Entry) { l.lastEntry = e } +// ShouldCount implements the [stats.Interface] interface for *testStats. +func (l *testStats) ShouldCount(string, uint16, uint16) bool { + return true +} + func TestProcessQueryLogsAndStats(t *testing.T) { testCases := []struct { name string diff --git a/internal/home/config.go b/internal/home/config.go index 41f5f55c..69baa18c 100644 --- a/internal/home/config.go +++ b/internal/home/config.go @@ -116,6 +116,7 @@ type configuration struct { DNS dnsConfig `yaml:"dns"` TLS tlsConfigSettings `yaml:"tls"` QueryLog queryLogConfig `yaml:"querylog"` + Stats statsConfig `yaml:"statistics"` // Filters reflects the filters from [filtering.Config]. It's cloned to the // config used in the filtering module at the startup. Afterwards it's @@ -149,10 +150,6 @@ type dnsConfig struct { BindHosts []netip.Addr `yaml:"bind_hosts"` Port int `yaml:"port"` - // StatsInterval is the time interval for flushing statistics to the disk in - // days. - StatsInterval uint32 `yaml:"statistics_interval"` - // AnonymizeClientIP defines if clients' IP addresses should be anonymized // in query log and statistics. AnonymizeClientIP bool `yaml:"anonymize_client_ip"` @@ -234,8 +231,20 @@ type queryLogConfig struct { // flushed to disk. MemSize uint32 `yaml:"size_memory"` - // Ignored is the list of host names, which are should not be written - // to log. + // Ignored is the list of host names, which should not be written to + // log. + Ignored []string `yaml:"ignored"` +} + +type statsConfig struct { + // Enabled defines if the statistics are enabled. + Enabled bool `yaml:"enabled"` + + // Interval is the time interval for flushing statistics to the disk in + // days. + Interval uint32 `yaml:"interval"` + + // Ignored is the list of host names, which should not be counted. Ignored []string `yaml:"ignored"` } @@ -249,9 +258,8 @@ var config = &configuration{ AuthBlockMin: 15, WebSessionTTLHours: 30 * 24, DNS: dnsConfig{ - BindHosts: []netip.Addr{netip.IPv4Unspecified()}, - Port: defaultPortDNS, - StatsInterval: 1, + BindHosts: []netip.Addr{netip.IPv4Unspecified()}, + Port: defaultPortDNS, FilteringConfig: dnsforward.FilteringConfig{ ProtectionEnabled: true, // whether or not use any of filtering features BlockingMode: dnsforward.BlockingModeDefault, @@ -296,6 +304,11 @@ var config = &configuration{ MemSize: 1000, Ignored: []string{}, }, + Stats: statsConfig{ + Enabled: true, + Interval: 1, + Ignored: []string{}, + }, // NOTE: Keep these parameters in sync with the one put into // client/src/helpers/filters/filters.js by scripts/vetted-filters. // @@ -472,9 +485,12 @@ func (c *configuration) write() (err error) { } if Context.stats != nil { - sdc := stats.DiskConfig{} - Context.stats.WriteDiskConfig(&sdc) - config.DNS.StatsInterval = sdc.Interval + statsConf := stats.Config{} + Context.stats.WriteDiskConfig(&statsConf) + config.Stats.Interval = statsConf.LimitDays + config.Stats.Enabled = statsConf.Enabled + config.Stats.Ignored = statsConf.Ignored.Values() + sort.Strings(config.Stats.Ignored) } if Context.queryLog != nil { diff --git a/internal/home/dns.go b/internal/home/dns.go index 39bbb7c1..855a4029 100644 --- a/internal/home/dns.go +++ b/internal/home/dns.go @@ -53,10 +53,18 @@ func initDNS() (err error) { statsConf := stats.Config{ Filename: filepath.Join(baseDir, "stats.db"), - LimitDays: config.DNS.StatsInterval, + LimitDays: config.Stats.Interval, ConfigModified: onConfigModified, HTTPRegister: httpRegister, + Enabled: config.Stats.Enabled, } + + set, err := nonDupEmptyHostNames(config.Stats.Ignored) + if err != nil { + return fmt.Errorf("statistics: ignored list: %w", err) + } + + statsConf.Ignored = set Context.stats, err = stats.New(statsConf) if err != nil { return fmt.Errorf("init stats: %w", err) @@ -73,16 +81,14 @@ func initDNS() (err error) { MemSize: config.QueryLog.MemSize, Enabled: config.QueryLog.Enabled, FileEnabled: config.QueryLog.FileEnabled, - Ignored: stringutil.NewSet(), } - for _, v := range config.QueryLog.Ignored { - host := strings.ToLower(strings.TrimSuffix(v, ".")) - if conf.Ignored.Has(host) { - return fmt.Errorf("duplicate ignored host %s", host) - } - conf.Ignored.Add(host) + set, err = nonDupEmptyHostNames(config.QueryLog.Ignored) + if err != nil { + return fmt.Errorf("querylog: ignored list: %w", err) } + + conf.Ignored = set Context.queryLog = querylog.New(conf) Context.filters, err = filtering.New(config.DNS.DnsfilterConf, nil) @@ -526,3 +532,27 @@ func closeDNSServer() { log.Debug("all dns modules are closed") } + +// nonDupEmptyHostNames returns nil and error, if list has duplicate or empty +// host name. Otherwise returns a set, which contains lowercase host names +// without dot at the end, and nil error. +func nonDupEmptyHostNames(list []string) (set *stringutil.Set, err error) { + set = stringutil.NewSet() + + for _, v := range list { + host := strings.ToLower(strings.TrimSuffix(v, ".")) + // TODO(a.garipov): Think about ignoring empty (".") names in + // the future. + if host == "" { + return nil, errors.Error("host name is empty") + } + + if set.Has(host) { + return nil, fmt.Errorf("duplicate host name %q", host) + } + + set.Add(host) + } + + return set, nil +} diff --git a/internal/home/upgrade.go b/internal/home/upgrade.go index 2a0f1e37..0f53bcdc 100644 --- a/internal/home/upgrade.go +++ b/internal/home/upgrade.go @@ -22,7 +22,7 @@ import ( ) // currentSchemaVersion is the current schema version. -const currentSchemaVersion = 15 +const currentSchemaVersion = 16 // These aliases are provided for convenience. type ( @@ -88,6 +88,7 @@ func upgradeConfigSchema(oldVersion int, diskConf yobj) (err error) { upgradeSchema12to13, upgradeSchema13to14, upgradeSchema14to15, + upgradeSchema15to16, } n := 0 @@ -823,8 +824,12 @@ func upgradeSchema14to15(diskConf yobj) (err error) { log.Printf("Upgrade yaml: 14 to 15") diskConf["schema_version"] = 15 - dnsVal := diskConf["dns"] - dns, ok := dnsVal.(map[string]any) + dnsVal, ok := diskConf["dns"] + if !ok { + return nil + } + + dns, ok := dnsVal.(yobj) if !ok { return fmt.Errorf("unexpected type of dns: %T", dnsVal) } @@ -856,6 +861,50 @@ func upgradeSchema14to15(diskConf yobj) (err error) { return nil } +// upgradeSchema15to16 performs the following changes: +// +// # BEFORE: +// 'dns': +// 'statistics_interval': 1 +// +// # AFTER: +// 'statistics': +// 'enabled': true +// 'interval': 1 +// 'ignored': [] +func upgradeSchema15to16(diskConf yobj) (err error) { + log.Printf("Upgrade yaml: 15 to 16") + diskConf["schema_version"] = 16 + + dnsVal, ok := diskConf["dns"] + if !ok { + return nil + } + + dns, ok := dnsVal.(yobj) + if !ok { + return fmt.Errorf("unexpected type of dns: %T", dnsVal) + } + + stats := map[string]any{ + "enabled": true, + "interval": 1, + "ignored": []any{}, + } + + k := "statistics_interval" + v, has := dns[k] + if has { + stats["enabled"] = v != 0 + stats["interval"] = v + } + delete(dns, k) + + diskConf["statistics"] = stats + + return nil +} + // TODO(a.garipov): Replace with log.Output when we port it to our logging // package. func funcName() string { diff --git a/internal/home/upgrade_test.go b/internal/home/upgrade_test.go index 76ac1bc9..14a43f70 100644 --- a/internal/home/upgrade_test.go +++ b/internal/home/upgrade_test.go @@ -688,3 +688,62 @@ func TestUpgradeSchema14to15(t *testing.T) { }) } } + +func TestUpgradeSchema15to16(t *testing.T) { + const newSchemaVer = 16 + + defaultWantObj := yobj{ + "statistics": map[string]any{ + "enabled": true, + "interval": 1, + "ignored": []any{}, + }, + "dns": map[string]any{}, + "schema_version": newSchemaVer, + } + + testCases := []struct { + in yobj + want yobj + name string + }{{ + in: yobj{ + "dns": map[string]any{ + "statistics_interval": 1, + }, + }, + want: defaultWantObj, + name: "basic", + }, { + in: yobj{ + "dns": map[string]any{}, + }, + want: defaultWantObj, + name: "default_values", + }, { + in: yobj{ + "dns": map[string]any{ + "statistics_interval": 0, + }, + }, + want: yobj{ + "statistics": map[string]any{ + "enabled": false, + "interval": 0, + "ignored": []any{}, + }, + "dns": map[string]any{}, + "schema_version": newSchemaVer, + }, + name: "stats_disabled", + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := upgradeSchema15to16(tc.in) + require.NoError(t, err) + + assert.Equal(t, tc.want, tc.in) + }) + } +} diff --git a/internal/querylog/http.go b/internal/querylog/http.go index f5c2960f..e8fbbc30 100644 --- a/internal/querylog/http.go +++ b/internal/querylog/http.go @@ -44,6 +44,9 @@ func (l *queryLog) initWeb() { } func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) { + l.lock.Lock() + defer l.lock.Unlock() + params, err := l.parseSearchParams(r) if err != nil { aghhttp.Error(r, w, http.StatusBadRequest, "failed to parse params: %s", err) diff --git a/internal/querylog/querylog.go b/internal/querylog/querylog.go index ff897bd3..28775774 100644 --- a/internal/querylog/querylog.go +++ b/internal/querylog/querylog.go @@ -76,8 +76,8 @@ type Config struct { // addresses. AnonymizeClientIP bool - // Ignored is the list of host names, which are should not be written - // to log. + // Ignored is the list of host names, which should not be written to + // log. Ignored *stringutil.Set } diff --git a/internal/querylog/querylogfile.go b/internal/querylog/querylogfile.go index f46e15cd..23e750fa 100644 --- a/internal/querylog/querylogfile.go +++ b/internal/querylog/querylogfile.go @@ -155,6 +155,9 @@ func (l *queryLog) periodicRotate() { // checkAndRotate rotates log files if those are older than the specified // rotation interval. func (l *queryLog) checkAndRotate() { + l.lock.Lock() + defer l.lock.Unlock() + oldest, err := l.readFileFirstTimeValue() if err != nil && !errors.Is(err, os.ErrNotExist) { log.Error("querylog: reading oldest record for rotation: %s", err) diff --git a/internal/stats/http.go b/internal/stats/http.go index b06a7cdc..4b041455 100644 --- a/internal/stats/http.go +++ b/internal/stats/http.go @@ -5,7 +5,6 @@ package stats import ( "encoding/json" "net/http" - "sync/atomic" "time" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" @@ -41,10 +40,11 @@ type StatsResp struct { // handleStats handles requests to the GET /control/stats endpoint. func (s *StatsCtx) handleStats(w http.ResponseWriter, r *http.Request) { - limit := atomic.LoadUint32(&s.limitHours) + s.lock.Lock() + defer s.lock.Unlock() start := time.Now() - resp, ok := s.getData(limit) + resp, ok := s.getData(s.limitHours) log.Debug("stats: prepared data in %v", time.Since(start)) if !ok { @@ -65,7 +65,13 @@ type configResp struct { // handleStatsInfo handles requests to the GET /control/stats_info endpoint. func (s *StatsCtx) handleStatsInfo(w http.ResponseWriter, r *http.Request) { - resp := configResp{IntervalDays: atomic.LoadUint32(&s.limitHours) / 24} + s.lock.Lock() + defer s.lock.Unlock() + + resp := configResp{IntervalDays: s.limitHours / 24} + if !s.enabled { + resp.IntervalDays = 0 + } _ = aghhttp.WriteJSONResponse(w, r, resp) } diff --git a/internal/stats/stats.go b/internal/stats/stats.go index 70cbd71c..9b53874d 100644 --- a/internal/stats/stats.go +++ b/internal/stats/stats.go @@ -15,16 +15,10 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/stringutil" "go.etcd.io/bbolt" ) -// DiskConfig is the configuration structure that is stored in file. -type DiskConfig struct { - // Interval is the number of days for which the statistics are collected - // before flushing to the database. - Interval uint32 `yaml:"statistics_interval"` -} - // checkInterval returns true if days is valid to be used as statistics // retention interval. The valid values are 0, 1, 7, 30 and 90. func checkInterval(days uint32) (ok bool) { @@ -51,6 +45,12 @@ type Config struct { // LimitDays is the maximum number of days to collect statistics into the // current unit. LimitDays uint32 + + // Enabled tells if the statistics are enabled. + Enabled bool + + // Ignored is the list of host names, which should not be counted. + Ignored *stringutil.Set } // Interface is the statistics interface to be used by other packages. @@ -68,19 +68,15 @@ type Interface interface { TopClientsIP(limit uint) []netip.Addr // WriteDiskConfig puts the Interface's configuration to the dc. - WriteDiskConfig(dc *DiskConfig) + WriteDiskConfig(dc *Config) + + // ShouldCount returns true if request for the host should be counted. + ShouldCount(host string, qType, qClass uint16) bool } // StatsCtx collects the statistics and flushes it to the database. Its default // flushing interval is one hour. type StatsCtx struct { - // limitHours is the maximum number of hours to collect statistics into the - // current unit. - // - // It is of type uint32 to be accessed by atomic. It's arranged at the - // beginning of the structure to keep 64-bit alignment. - limitHours uint32 - // currMu protects curr. currMu *sync.RWMutex // curr is the actual statistics collection result. @@ -102,6 +98,21 @@ type StatsCtx struct { // filename is the name of database file. filename string + + // lock protects all the fields below. + lock sync.Mutex + + // enabled tells if the statistics are enabled. + enabled bool + + // limitHours is the maximum number of hours to collect statistics into the + // current unit. + // + // TODO(s.chzhen): Rewrite to use time.Duration. + limitHours uint32 + + // ignored is the list of host names, which should not be counted. + ignored *stringutil.Set } // New creates s from conf and properly initializes it. Don't use s before @@ -110,10 +121,12 @@ func New(conf Config) (s *StatsCtx, err error) { defer withRecovered(&err) s = &StatsCtx{ + enabled: conf.Enabled, currMu: &sync.RWMutex{}, filename: conf.Filename, configModified: conf.ConfigModified, httpRegister: conf.HTTPRegister, + ignored: conf.Ignored, } if s.limitHours = conf.LimitDays * 24; !checkInterval(conf.LimitDays) { s.limitHours = 24 @@ -215,7 +228,10 @@ func (s *StatsCtx) Close() (err error) { // Update implements the Interface interface for *StatsCtx. func (s *StatsCtx) Update(e Entry) { - if atomic.LoadUint32(&s.limitHours) == 0 { + s.lock.Lock() + defer s.lock.Unlock() + + if !s.enabled || s.limitHours == 0 { return } @@ -243,14 +259,22 @@ func (s *StatsCtx) Update(e Entry) { } // WriteDiskConfig implements the Interface interface for *StatsCtx. -func (s *StatsCtx) WriteDiskConfig(dc *DiskConfig) { - dc.Interval = atomic.LoadUint32(&s.limitHours) / 24 +func (s *StatsCtx) WriteDiskConfig(dc *Config) { + s.lock.Lock() + defer s.lock.Unlock() + + dc.LimitDays = s.limitHours / 24 + dc.Enabled = s.enabled + dc.Ignored = s.ignored } // TopClientsIP implements the [Interface] interface for *StatsCtx. func (s *StatsCtx) TopClientsIP(maxCount uint) (ips []netip.Addr) { - limit := atomic.LoadUint32(&s.limitHours) - if limit == 0 { + s.lock.Lock() + defer s.lock.Unlock() + + limit := s.limitHours + if !s.enabled || limit == 0 { return nil } @@ -342,6 +366,9 @@ func (s *StatsCtx) openDB() (err error) { func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) { id := s.unitIDGen() + s.lock.Lock() + defer s.lock.Unlock() + s.currMu.Lock() defer s.currMu.Unlock() @@ -350,7 +377,7 @@ func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) { return false, 0 } - limit := atomic.LoadUint32(&s.limitHours) + limit := s.limitHours if limit == 0 || ptr.id == id { return true, time.Second } @@ -410,14 +437,23 @@ func (s *StatsCtx) periodicFlush() { } func (s *StatsCtx) setLimit(limitDays int) { - atomic.StoreUint32(&s.limitHours, uint32(24*limitDays)) - if limitDays == 0 { - if err := s.clear(); err != nil { - log.Error("stats: %s", err) - } + s.lock.Lock() + defer s.lock.Unlock() + + if limitDays != 0 { + s.enabled = true + s.limitHours = uint32(24 * limitDays) + log.Debug("stats: set limit: %d days", limitDays) + + return } - log.Debug("stats: set limit: %d days", limitDays) + s.enabled = false + log.Debug("stats: disabled") + + if err := s.clear(); err != nil { + log.Error("stats: %s", err) + } } // Reset counters and clear database @@ -520,3 +556,13 @@ func (s *StatsCtx) loadUnits(limit uint32) (units []*unitDB, firstID uint32) { return units, firstID } + +// ShouldCount returns true if request for the host should be counted. +func (s *StatsCtx) ShouldCount(host string, _, _ uint16) bool { + return !s.isIgnored(host) +} + +// isIgnored returns true if the host is in the Ignored list. +func (s *StatsCtx) isIgnored(host string) bool { + return s.ignored.Has(host) +} diff --git a/internal/stats/stats_test.go b/internal/stats/stats_test.go index cd88cbaf..f795c2da 100644 --- a/internal/stats/stats_test.go +++ b/internal/stats/stats_test.go @@ -53,6 +53,7 @@ func TestStats(t *testing.T) { conf := stats.Config{ Filename: filepath.Join(t.TempDir(), "stats.db"), LimitDays: 1, + Enabled: true, UnitID: constUnitID, HTTPRegister: func(_, url string, handler http.HandlerFunc) { handlers[url] = handler @@ -158,6 +159,7 @@ func TestLargeNumbers(t *testing.T) { conf := stats.Config{ Filename: filepath.Join(t.TempDir(), "stats.db"), LimitDays: 1, + Enabled: true, UnitID: func() (id uint32) { return atomic.LoadUint32(&curHour) }, HTTPRegister: func(_, url string, handler http.HandlerFunc) { handlers[url] = handler }, } diff --git a/internal/stats/unit.go b/internal/stats/unit.go index 0e39cfea..99e04429 100644 --- a/internal/stats/unit.go +++ b/internal/stats/unit.go @@ -10,6 +10,7 @@ import ( "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/stringutil" "go.etcd.io/bbolt" ) @@ -341,11 +342,13 @@ type pairsGetter func(u *unitDB) (pairs []countPair) // topsCollector collects statistics about highest values from the given *unitDB // slice using pg to retrieve data. -func topsCollector(units []*unitDB, max int, pg pairsGetter) []map[string]uint64 { +func topsCollector(units []*unitDB, max int, ignored *stringutil.Set, pg pairsGetter) []map[string]uint64 { m := map[string]uint64{} for _, u := range units { for _, cp := range pg(u) { - m[cp.Name] += cp.Count + if !ignored.Has(cp.Name) { + m[cp.Name] += cp.Count + } } } a2 := convertMapToSlice(m, max) @@ -408,9 +411,9 @@ func (s *StatsCtx) getData(limit uint32) (StatsResp, bool) { BlockedFiltering: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RFiltered] }), ReplacedSafebrowsing: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RSafeBrowsing] }), ReplacedParental: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RParental] }), - TopQueried: topsCollector(units, maxDomains, func(u *unitDB) (pairs []countPair) { return u.Domains }), - TopBlocked: topsCollector(units, maxDomains, func(u *unitDB) (pairs []countPair) { return u.BlockedDomains }), - TopClients: topsCollector(units, maxClients, func(u *unitDB) (pairs []countPair) { return u.Clients }), + TopQueried: topsCollector(units, maxDomains, s.ignored, func(u *unitDB) (pairs []countPair) { return u.Domains }), + TopBlocked: topsCollector(units, maxDomains, s.ignored, func(u *unitDB) (pairs []countPair) { return u.BlockedDomains }), + TopClients: topsCollector(units, maxClients, nil, func(u *unitDB) (pairs []countPair) { return u.Clients }), } // Total counters: