Pull request 1731: 4299-stats-ignore

Merge in DNS/adguard-home from 4299-stats-ignore to master

Updates #1717.
Updates #4299.

Squashed commit of the following:

commit 1d1212d088c944e995deae2fd599eccb0a075033
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 13 17:53:36 2023 +0300

    fix changelog

commit 5f56852c21d794bd87c13192d3857757be10f9b2
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 13 17:39:02 2023 +0300

    add todo; fix data race

commit 89b8b16ddf5a43ebf68174cbaf9e8a53365f8cbe
Merge: e0a6bb49 ec19a85e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 17:21:38 2023 +0300

    Merge branch 'master' into 4299-stats-ignore

commit e0a6bb490b651d1cf31589a7f17095fff4cb4dbb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 17:21:06 2023 +0300

    interval under mutex

commit c569c7bc237f11b23fe47c98a20a1c5cb36751cb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 16:19:35 2023 +0300

    fix mutex

commit 9374cf0c54
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 16:03:17 2023 +0300

    fix typo

commit 1f4fd1e7ab
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 15:55:44 2023 +0300

    add mutex

commit 2148048ce9
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 12:27:36 2023 +0300

    add key check

commit a19350977c463f888aea70d0dace26dff0173a65
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Feb 9 18:34:36 2023 +0300

    fix changelog

commit 23c3b6da162dfd513884b460c265ba4cafeb9727
Merge: 8fccc0b8 b89105e3
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Feb 9 13:28:59 2023 +0300

    Merge branch 'master' into 4299-stats-ignore

commit 8fccc0b8ec670a37e5209d795f35c43dd64afeb3
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Feb 9 13:27:42 2023 +0300

    add changelog

commit 0416c71742795b2fb8adb0173dcd6a99d9d9c676
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Feb 8 14:31:55 2023 +0300

    all: stats ignore
This commit is contained in:
Stanislav Chzhen 2023-02-13 18:15:33 +03:00
parent ec19a85ed0
commit ff04b2a7d3
14 changed files with 320 additions and 71 deletions

View File

@ -25,16 +25,38 @@ NOTE: Add new changes BELOW THIS COMMENT.
### Added ### Added
- The ability to exclude domain names from the query log by using the new - The ability to disable statistics by using the new `statistics.enabled`
`querylog.ignored` field ([#1717], [#4299]). field. Previously it was necessary to set the `statistics_interval` to 0,
losing the previous value ([#1717], [#4299]).
- The ability to exclude domain names from the query log or statistics by using
the new `querylog.ignored` or `statistics.ignored` fields ([#1717], [#4299]).
### Changed ### Changed
#### Configuration Changes #### Configuration Changes
In this release, the schema version has changed from 14 to 15. In this release, the schema version has changed from 14 to 16.
- The fields `dns.…` have been moved to the new `querylog` object. - Property `statistics_interval`, which in schema versions 15 and earlier used
to be a part of the `dns` object, is now a part of the `statistics` object:
```yaml
# BEFORE:
'dns':
# …
'statistics_interval': 1
# AFTER:
'statistics':
# …
'interval': 1
```
To rollback this change, move the property back into the `dns` object and
change the `schema_version` back to `15`.
- The fields `dns.querylog_enabled`, `dns.querylog_file_enabled`,
`dns.querylog_interval`, `dns.querylog_size_memory` have been moved to the
new `querylog` object.
```yaml ```yaml
# BEFORE: # BEFORE:

View File

@ -48,10 +48,15 @@ func (s *Server) processQueryLogsAndStats(dctx *dnsContext) (rc resultCode) {
s.queryLog.ShouldLog(host, q.Qtype, q.Qclass) { s.queryLog.ShouldLog(host, q.Qtype, q.Qclass) {
s.logQuery(dctx, pctx, elapsed, ip) s.logQuery(dctx, pctx, elapsed, ip)
} else { } else {
log.Debug("request for %s from %s ignored; not logging", host, ip) log.Debug(
"dnsforward: request %s %s from %s ignored; not logging",
dns.Type(q.Qtype),
host,
ip,
)
} }
if s.stats != nil { if s.stats != nil && s.stats.ShouldCount(host, q.Qtype, q.Qclass) {
s.updateStats(dctx, elapsed, *dctx.result, ip) s.updateStats(dctx, elapsed, *dctx.result, ip)
} }

View File

@ -35,20 +35,25 @@ func (l *testQueryLog) ShouldLog(string, uint16, uint16) bool {
return true return true
} }
// testStats is a simple stats.Stats implementation for tests. // testStats is a simple [stats.Interface] implementation for tests.
type testStats struct { type testStats struct {
// Stats is embedded here simply to make testStats a stats.Stats without // Stats is embedded here simply to make testStats a [stats.Interface]
// actually implementing all methods. // without actually implementing all methods.
stats.Interface stats.Interface
lastEntry stats.Entry lastEntry stats.Entry
} }
// Update implements the stats.Stats interface for *testStats. // Update implements the [stats.Interface] interface for *testStats.
func (l *testStats) Update(e stats.Entry) { func (l *testStats) Update(e stats.Entry) {
l.lastEntry = e l.lastEntry = e
} }
// ShouldCount implements the [stats.Interface] interface for *testStats.
func (l *testStats) ShouldCount(string, uint16, uint16) bool {
return true
}
func TestProcessQueryLogsAndStats(t *testing.T) { func TestProcessQueryLogsAndStats(t *testing.T) {
testCases := []struct { testCases := []struct {
name string name string

View File

@ -116,6 +116,7 @@ type configuration struct {
DNS dnsConfig `yaml:"dns"` DNS dnsConfig `yaml:"dns"`
TLS tlsConfigSettings `yaml:"tls"` TLS tlsConfigSettings `yaml:"tls"`
QueryLog queryLogConfig `yaml:"querylog"` QueryLog queryLogConfig `yaml:"querylog"`
Stats statsConfig `yaml:"statistics"`
// Filters reflects the filters from [filtering.Config]. It's cloned to the // Filters reflects the filters from [filtering.Config]. It's cloned to the
// config used in the filtering module at the startup. Afterwards it's // config used in the filtering module at the startup. Afterwards it's
@ -149,10 +150,6 @@ type dnsConfig struct {
BindHosts []netip.Addr `yaml:"bind_hosts"` BindHosts []netip.Addr `yaml:"bind_hosts"`
Port int `yaml:"port"` Port int `yaml:"port"`
// StatsInterval is the time interval for flushing statistics to the disk in
// days.
StatsInterval uint32 `yaml:"statistics_interval"`
// AnonymizeClientIP defines if clients' IP addresses should be anonymized // AnonymizeClientIP defines if clients' IP addresses should be anonymized
// in query log and statistics. // in query log and statistics.
AnonymizeClientIP bool `yaml:"anonymize_client_ip"` AnonymizeClientIP bool `yaml:"anonymize_client_ip"`
@ -234,8 +231,20 @@ type queryLogConfig struct {
// flushed to disk. // flushed to disk.
MemSize uint32 `yaml:"size_memory"` MemSize uint32 `yaml:"size_memory"`
// Ignored is the list of host names, which are should not be written // Ignored is the list of host names, which should not be written to
// to log. // log.
Ignored []string `yaml:"ignored"`
}
type statsConfig struct {
// Enabled defines if the statistics are enabled.
Enabled bool `yaml:"enabled"`
// Interval is the time interval for flushing statistics to the disk in
// days.
Interval uint32 `yaml:"interval"`
// Ignored is the list of host names, which should not be counted.
Ignored []string `yaml:"ignored"` Ignored []string `yaml:"ignored"`
} }
@ -249,9 +258,8 @@ var config = &configuration{
AuthBlockMin: 15, AuthBlockMin: 15,
WebSessionTTLHours: 30 * 24, WebSessionTTLHours: 30 * 24,
DNS: dnsConfig{ DNS: dnsConfig{
BindHosts: []netip.Addr{netip.IPv4Unspecified()}, BindHosts: []netip.Addr{netip.IPv4Unspecified()},
Port: defaultPortDNS, Port: defaultPortDNS,
StatsInterval: 1,
FilteringConfig: dnsforward.FilteringConfig{ FilteringConfig: dnsforward.FilteringConfig{
ProtectionEnabled: true, // whether or not use any of filtering features ProtectionEnabled: true, // whether or not use any of filtering features
BlockingMode: dnsforward.BlockingModeDefault, BlockingMode: dnsforward.BlockingModeDefault,
@ -296,6 +304,11 @@ var config = &configuration{
MemSize: 1000, MemSize: 1000,
Ignored: []string{}, Ignored: []string{},
}, },
Stats: statsConfig{
Enabled: true,
Interval: 1,
Ignored: []string{},
},
// NOTE: Keep these parameters in sync with the one put into // NOTE: Keep these parameters in sync with the one put into
// client/src/helpers/filters/filters.js by scripts/vetted-filters. // client/src/helpers/filters/filters.js by scripts/vetted-filters.
// //
@ -472,9 +485,12 @@ func (c *configuration) write() (err error) {
} }
if Context.stats != nil { if Context.stats != nil {
sdc := stats.DiskConfig{} statsConf := stats.Config{}
Context.stats.WriteDiskConfig(&sdc) Context.stats.WriteDiskConfig(&statsConf)
config.DNS.StatsInterval = sdc.Interval config.Stats.Interval = statsConf.LimitDays
config.Stats.Enabled = statsConf.Enabled
config.Stats.Ignored = statsConf.Ignored.Values()
sort.Strings(config.Stats.Ignored)
} }
if Context.queryLog != nil { if Context.queryLog != nil {

View File

@ -53,10 +53,18 @@ func initDNS() (err error) {
statsConf := stats.Config{ statsConf := stats.Config{
Filename: filepath.Join(baseDir, "stats.db"), Filename: filepath.Join(baseDir, "stats.db"),
LimitDays: config.DNS.StatsInterval, LimitDays: config.Stats.Interval,
ConfigModified: onConfigModified, ConfigModified: onConfigModified,
HTTPRegister: httpRegister, HTTPRegister: httpRegister,
Enabled: config.Stats.Enabled,
} }
set, err := nonDupEmptyHostNames(config.Stats.Ignored)
if err != nil {
return fmt.Errorf("statistics: ignored list: %w", err)
}
statsConf.Ignored = set
Context.stats, err = stats.New(statsConf) Context.stats, err = stats.New(statsConf)
if err != nil { if err != nil {
return fmt.Errorf("init stats: %w", err) return fmt.Errorf("init stats: %w", err)
@ -73,16 +81,14 @@ func initDNS() (err error) {
MemSize: config.QueryLog.MemSize, MemSize: config.QueryLog.MemSize,
Enabled: config.QueryLog.Enabled, Enabled: config.QueryLog.Enabled,
FileEnabled: config.QueryLog.FileEnabled, FileEnabled: config.QueryLog.FileEnabled,
Ignored: stringutil.NewSet(),
} }
for _, v := range config.QueryLog.Ignored {
host := strings.ToLower(strings.TrimSuffix(v, "."))
if conf.Ignored.Has(host) {
return fmt.Errorf("duplicate ignored host %s", host)
}
conf.Ignored.Add(host) set, err = nonDupEmptyHostNames(config.QueryLog.Ignored)
if err != nil {
return fmt.Errorf("querylog: ignored list: %w", err)
} }
conf.Ignored = set
Context.queryLog = querylog.New(conf) Context.queryLog = querylog.New(conf)
Context.filters, err = filtering.New(config.DNS.DnsfilterConf, nil) Context.filters, err = filtering.New(config.DNS.DnsfilterConf, nil)
@ -526,3 +532,27 @@ func closeDNSServer() {
log.Debug("all dns modules are closed") log.Debug("all dns modules are closed")
} }
// nonDupEmptyHostNames returns nil and error, if list has duplicate or empty
// host name. Otherwise returns a set, which contains lowercase host names
// without dot at the end, and nil error.
func nonDupEmptyHostNames(list []string) (set *stringutil.Set, err error) {
set = stringutil.NewSet()
for _, v := range list {
host := strings.ToLower(strings.TrimSuffix(v, "."))
// TODO(a.garipov): Think about ignoring empty (".") names in
// the future.
if host == "" {
return nil, errors.Error("host name is empty")
}
if set.Has(host) {
return nil, fmt.Errorf("duplicate host name %q", host)
}
set.Add(host)
}
return set, nil
}

View File

@ -22,7 +22,7 @@ import (
) )
// currentSchemaVersion is the current schema version. // currentSchemaVersion is the current schema version.
const currentSchemaVersion = 15 const currentSchemaVersion = 16
// These aliases are provided for convenience. // These aliases are provided for convenience.
type ( type (
@ -88,6 +88,7 @@ func upgradeConfigSchema(oldVersion int, diskConf yobj) (err error) {
upgradeSchema12to13, upgradeSchema12to13,
upgradeSchema13to14, upgradeSchema13to14,
upgradeSchema14to15, upgradeSchema14to15,
upgradeSchema15to16,
} }
n := 0 n := 0
@ -823,8 +824,12 @@ func upgradeSchema14to15(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 14 to 15") log.Printf("Upgrade yaml: 14 to 15")
diskConf["schema_version"] = 15 diskConf["schema_version"] = 15
dnsVal := diskConf["dns"] dnsVal, ok := diskConf["dns"]
dns, ok := dnsVal.(map[string]any) if !ok {
return nil
}
dns, ok := dnsVal.(yobj)
if !ok { if !ok {
return fmt.Errorf("unexpected type of dns: %T", dnsVal) return fmt.Errorf("unexpected type of dns: %T", dnsVal)
} }
@ -856,6 +861,50 @@ func upgradeSchema14to15(diskConf yobj) (err error) {
return nil return nil
} }
// upgradeSchema15to16 performs the following changes:
//
// # BEFORE:
// 'dns':
// 'statistics_interval': 1
//
// # AFTER:
// 'statistics':
// 'enabled': true
// 'interval': 1
// 'ignored': []
func upgradeSchema15to16(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 15 to 16")
diskConf["schema_version"] = 16
dnsVal, ok := diskConf["dns"]
if !ok {
return nil
}
dns, ok := dnsVal.(yobj)
if !ok {
return fmt.Errorf("unexpected type of dns: %T", dnsVal)
}
stats := map[string]any{
"enabled": true,
"interval": 1,
"ignored": []any{},
}
k := "statistics_interval"
v, has := dns[k]
if has {
stats["enabled"] = v != 0
stats["interval"] = v
}
delete(dns, k)
diskConf["statistics"] = stats
return nil
}
// TODO(a.garipov): Replace with log.Output when we port it to our logging // TODO(a.garipov): Replace with log.Output when we port it to our logging
// package. // package.
func funcName() string { func funcName() string {

View File

@ -688,3 +688,62 @@ func TestUpgradeSchema14to15(t *testing.T) {
}) })
} }
} }
func TestUpgradeSchema15to16(t *testing.T) {
const newSchemaVer = 16
defaultWantObj := yobj{
"statistics": map[string]any{
"enabled": true,
"interval": 1,
"ignored": []any{},
},
"dns": map[string]any{},
"schema_version": newSchemaVer,
}
testCases := []struct {
in yobj
want yobj
name string
}{{
in: yobj{
"dns": map[string]any{
"statistics_interval": 1,
},
},
want: defaultWantObj,
name: "basic",
}, {
in: yobj{
"dns": map[string]any{},
},
want: defaultWantObj,
name: "default_values",
}, {
in: yobj{
"dns": map[string]any{
"statistics_interval": 0,
},
},
want: yobj{
"statistics": map[string]any{
"enabled": false,
"interval": 0,
"ignored": []any{},
},
"dns": map[string]any{},
"schema_version": newSchemaVer,
},
name: "stats_disabled",
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
err := upgradeSchema15to16(tc.in)
require.NoError(t, err)
assert.Equal(t, tc.want, tc.in)
})
}
}

View File

@ -44,6 +44,9 @@ func (l *queryLog) initWeb() {
} }
func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) { func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
l.lock.Lock()
defer l.lock.Unlock()
params, err := l.parseSearchParams(r) params, err := l.parseSearchParams(r)
if err != nil { if err != nil {
aghhttp.Error(r, w, http.StatusBadRequest, "failed to parse params: %s", err) aghhttp.Error(r, w, http.StatusBadRequest, "failed to parse params: %s", err)

View File

@ -76,8 +76,8 @@ type Config struct {
// addresses. // addresses.
AnonymizeClientIP bool AnonymizeClientIP bool
// Ignored is the list of host names, which are should not be written // Ignored is the list of host names, which should not be written to
// to log. // log.
Ignored *stringutil.Set Ignored *stringutil.Set
} }

View File

@ -155,6 +155,9 @@ func (l *queryLog) periodicRotate() {
// checkAndRotate rotates log files if those are older than the specified // checkAndRotate rotates log files if those are older than the specified
// rotation interval. // rotation interval.
func (l *queryLog) checkAndRotate() { func (l *queryLog) checkAndRotate() {
l.lock.Lock()
defer l.lock.Unlock()
oldest, err := l.readFileFirstTimeValue() oldest, err := l.readFileFirstTimeValue()
if err != nil && !errors.Is(err, os.ErrNotExist) { if err != nil && !errors.Is(err, os.ErrNotExist) {
log.Error("querylog: reading oldest record for rotation: %s", err) log.Error("querylog: reading oldest record for rotation: %s", err)

View File

@ -5,7 +5,6 @@ package stats
import ( import (
"encoding/json" "encoding/json"
"net/http" "net/http"
"sync/atomic"
"time" "time"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
@ -41,10 +40,11 @@ type StatsResp struct {
// handleStats handles requests to the GET /control/stats endpoint. // handleStats handles requests to the GET /control/stats endpoint.
func (s *StatsCtx) handleStats(w http.ResponseWriter, r *http.Request) { func (s *StatsCtx) handleStats(w http.ResponseWriter, r *http.Request) {
limit := atomic.LoadUint32(&s.limitHours) s.lock.Lock()
defer s.lock.Unlock()
start := time.Now() start := time.Now()
resp, ok := s.getData(limit) resp, ok := s.getData(s.limitHours)
log.Debug("stats: prepared data in %v", time.Since(start)) log.Debug("stats: prepared data in %v", time.Since(start))
if !ok { if !ok {
@ -65,7 +65,13 @@ type configResp struct {
// handleStatsInfo handles requests to the GET /control/stats_info endpoint. // handleStatsInfo handles requests to the GET /control/stats_info endpoint.
func (s *StatsCtx) handleStatsInfo(w http.ResponseWriter, r *http.Request) { func (s *StatsCtx) handleStatsInfo(w http.ResponseWriter, r *http.Request) {
resp := configResp{IntervalDays: atomic.LoadUint32(&s.limitHours) / 24} s.lock.Lock()
defer s.lock.Unlock()
resp := configResp{IntervalDays: s.limitHours / 24}
if !s.enabled {
resp.IntervalDays = 0
}
_ = aghhttp.WriteJSONResponse(w, r, resp) _ = aghhttp.WriteJSONResponse(w, r, resp)
} }

View File

@ -15,16 +15,10 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"go.etcd.io/bbolt" "go.etcd.io/bbolt"
) )
// DiskConfig is the configuration structure that is stored in file.
type DiskConfig struct {
// Interval is the number of days for which the statistics are collected
// before flushing to the database.
Interval uint32 `yaml:"statistics_interval"`
}
// checkInterval returns true if days is valid to be used as statistics // checkInterval returns true if days is valid to be used as statistics
// retention interval. The valid values are 0, 1, 7, 30 and 90. // retention interval. The valid values are 0, 1, 7, 30 and 90.
func checkInterval(days uint32) (ok bool) { func checkInterval(days uint32) (ok bool) {
@ -51,6 +45,12 @@ type Config struct {
// LimitDays is the maximum number of days to collect statistics into the // LimitDays is the maximum number of days to collect statistics into the
// current unit. // current unit.
LimitDays uint32 LimitDays uint32
// Enabled tells if the statistics are enabled.
Enabled bool
// Ignored is the list of host names, which should not be counted.
Ignored *stringutil.Set
} }
// Interface is the statistics interface to be used by other packages. // Interface is the statistics interface to be used by other packages.
@ -68,19 +68,15 @@ type Interface interface {
TopClientsIP(limit uint) []netip.Addr TopClientsIP(limit uint) []netip.Addr
// WriteDiskConfig puts the Interface's configuration to the dc. // WriteDiskConfig puts the Interface's configuration to the dc.
WriteDiskConfig(dc *DiskConfig) WriteDiskConfig(dc *Config)
// ShouldCount returns true if request for the host should be counted.
ShouldCount(host string, qType, qClass uint16) bool
} }
// StatsCtx collects the statistics and flushes it to the database. Its default // StatsCtx collects the statistics and flushes it to the database. Its default
// flushing interval is one hour. // flushing interval is one hour.
type StatsCtx struct { type StatsCtx struct {
// limitHours is the maximum number of hours to collect statistics into the
// current unit.
//
// It is of type uint32 to be accessed by atomic. It's arranged at the
// beginning of the structure to keep 64-bit alignment.
limitHours uint32
// currMu protects curr. // currMu protects curr.
currMu *sync.RWMutex currMu *sync.RWMutex
// curr is the actual statistics collection result. // curr is the actual statistics collection result.
@ -102,6 +98,21 @@ type StatsCtx struct {
// filename is the name of database file. // filename is the name of database file.
filename string filename string
// lock protects all the fields below.
lock sync.Mutex
// enabled tells if the statistics are enabled.
enabled bool
// limitHours is the maximum number of hours to collect statistics into the
// current unit.
//
// TODO(s.chzhen): Rewrite to use time.Duration.
limitHours uint32
// ignored is the list of host names, which should not be counted.
ignored *stringutil.Set
} }
// New creates s from conf and properly initializes it. Don't use s before // New creates s from conf and properly initializes it. Don't use s before
@ -110,10 +121,12 @@ func New(conf Config) (s *StatsCtx, err error) {
defer withRecovered(&err) defer withRecovered(&err)
s = &StatsCtx{ s = &StatsCtx{
enabled: conf.Enabled,
currMu: &sync.RWMutex{}, currMu: &sync.RWMutex{},
filename: conf.Filename, filename: conf.Filename,
configModified: conf.ConfigModified, configModified: conf.ConfigModified,
httpRegister: conf.HTTPRegister, httpRegister: conf.HTTPRegister,
ignored: conf.Ignored,
} }
if s.limitHours = conf.LimitDays * 24; !checkInterval(conf.LimitDays) { if s.limitHours = conf.LimitDays * 24; !checkInterval(conf.LimitDays) {
s.limitHours = 24 s.limitHours = 24
@ -215,7 +228,10 @@ func (s *StatsCtx) Close() (err error) {
// Update implements the Interface interface for *StatsCtx. // Update implements the Interface interface for *StatsCtx.
func (s *StatsCtx) Update(e Entry) { func (s *StatsCtx) Update(e Entry) {
if atomic.LoadUint32(&s.limitHours) == 0 { s.lock.Lock()
defer s.lock.Unlock()
if !s.enabled || s.limitHours == 0 {
return return
} }
@ -243,14 +259,22 @@ func (s *StatsCtx) Update(e Entry) {
} }
// WriteDiskConfig implements the Interface interface for *StatsCtx. // WriteDiskConfig implements the Interface interface for *StatsCtx.
func (s *StatsCtx) WriteDiskConfig(dc *DiskConfig) { func (s *StatsCtx) WriteDiskConfig(dc *Config) {
dc.Interval = atomic.LoadUint32(&s.limitHours) / 24 s.lock.Lock()
defer s.lock.Unlock()
dc.LimitDays = s.limitHours / 24
dc.Enabled = s.enabled
dc.Ignored = s.ignored
} }
// TopClientsIP implements the [Interface] interface for *StatsCtx. // TopClientsIP implements the [Interface] interface for *StatsCtx.
func (s *StatsCtx) TopClientsIP(maxCount uint) (ips []netip.Addr) { func (s *StatsCtx) TopClientsIP(maxCount uint) (ips []netip.Addr) {
limit := atomic.LoadUint32(&s.limitHours) s.lock.Lock()
if limit == 0 { defer s.lock.Unlock()
limit := s.limitHours
if !s.enabled || limit == 0 {
return nil return nil
} }
@ -342,6 +366,9 @@ func (s *StatsCtx) openDB() (err error) {
func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) { func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) {
id := s.unitIDGen() id := s.unitIDGen()
s.lock.Lock()
defer s.lock.Unlock()
s.currMu.Lock() s.currMu.Lock()
defer s.currMu.Unlock() defer s.currMu.Unlock()
@ -350,7 +377,7 @@ func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) {
return false, 0 return false, 0
} }
limit := atomic.LoadUint32(&s.limitHours) limit := s.limitHours
if limit == 0 || ptr.id == id { if limit == 0 || ptr.id == id {
return true, time.Second return true, time.Second
} }
@ -410,14 +437,23 @@ func (s *StatsCtx) periodicFlush() {
} }
func (s *StatsCtx) setLimit(limitDays int) { func (s *StatsCtx) setLimit(limitDays int) {
atomic.StoreUint32(&s.limitHours, uint32(24*limitDays)) s.lock.Lock()
if limitDays == 0 { defer s.lock.Unlock()
if err := s.clear(); err != nil {
log.Error("stats: %s", err) if limitDays != 0 {
} s.enabled = true
s.limitHours = uint32(24 * limitDays)
log.Debug("stats: set limit: %d days", limitDays)
return
} }
log.Debug("stats: set limit: %d days", limitDays) s.enabled = false
log.Debug("stats: disabled")
if err := s.clear(); err != nil {
log.Error("stats: %s", err)
}
} }
// Reset counters and clear database // Reset counters and clear database
@ -520,3 +556,13 @@ func (s *StatsCtx) loadUnits(limit uint32) (units []*unitDB, firstID uint32) {
return units, firstID return units, firstID
} }
// ShouldCount returns true if request for the host should be counted.
func (s *StatsCtx) ShouldCount(host string, _, _ uint16) bool {
return !s.isIgnored(host)
}
// isIgnored returns true if the host is in the Ignored list.
func (s *StatsCtx) isIgnored(host string) bool {
return s.ignored.Has(host)
}

View File

@ -53,6 +53,7 @@ func TestStats(t *testing.T) {
conf := stats.Config{ conf := stats.Config{
Filename: filepath.Join(t.TempDir(), "stats.db"), Filename: filepath.Join(t.TempDir(), "stats.db"),
LimitDays: 1, LimitDays: 1,
Enabled: true,
UnitID: constUnitID, UnitID: constUnitID,
HTTPRegister: func(_, url string, handler http.HandlerFunc) { HTTPRegister: func(_, url string, handler http.HandlerFunc) {
handlers[url] = handler handlers[url] = handler
@ -158,6 +159,7 @@ func TestLargeNumbers(t *testing.T) {
conf := stats.Config{ conf := stats.Config{
Filename: filepath.Join(t.TempDir(), "stats.db"), Filename: filepath.Join(t.TempDir(), "stats.db"),
LimitDays: 1, LimitDays: 1,
Enabled: true,
UnitID: func() (id uint32) { return atomic.LoadUint32(&curHour) }, UnitID: func() (id uint32) { return atomic.LoadUint32(&curHour) },
HTTPRegister: func(_, url string, handler http.HandlerFunc) { handlers[url] = handler }, HTTPRegister: func(_, url string, handler http.HandlerFunc) { handlers[url] = handler },
} }

View File

@ -10,6 +10,7 @@ import (
"github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"go.etcd.io/bbolt" "go.etcd.io/bbolt"
) )
@ -341,11 +342,13 @@ type pairsGetter func(u *unitDB) (pairs []countPair)
// topsCollector collects statistics about highest values from the given *unitDB // topsCollector collects statistics about highest values from the given *unitDB
// slice using pg to retrieve data. // slice using pg to retrieve data.
func topsCollector(units []*unitDB, max int, pg pairsGetter) []map[string]uint64 { func topsCollector(units []*unitDB, max int, ignored *stringutil.Set, pg pairsGetter) []map[string]uint64 {
m := map[string]uint64{} m := map[string]uint64{}
for _, u := range units { for _, u := range units {
for _, cp := range pg(u) { for _, cp := range pg(u) {
m[cp.Name] += cp.Count if !ignored.Has(cp.Name) {
m[cp.Name] += cp.Count
}
} }
} }
a2 := convertMapToSlice(m, max) a2 := convertMapToSlice(m, max)
@ -408,9 +411,9 @@ func (s *StatsCtx) getData(limit uint32) (StatsResp, bool) {
BlockedFiltering: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RFiltered] }), BlockedFiltering: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RFiltered] }),
ReplacedSafebrowsing: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RSafeBrowsing] }), ReplacedSafebrowsing: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RSafeBrowsing] }),
ReplacedParental: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RParental] }), ReplacedParental: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RParental] }),
TopQueried: topsCollector(units, maxDomains, func(u *unitDB) (pairs []countPair) { return u.Domains }), TopQueried: topsCollector(units, maxDomains, s.ignored, func(u *unitDB) (pairs []countPair) { return u.Domains }),
TopBlocked: topsCollector(units, maxDomains, func(u *unitDB) (pairs []countPair) { return u.BlockedDomains }), TopBlocked: topsCollector(units, maxDomains, s.ignored, func(u *unitDB) (pairs []countPair) { return u.BlockedDomains }),
TopClients: topsCollector(units, maxClients, func(u *unitDB) (pairs []countPair) { return u.Clients }), TopClients: topsCollector(units, maxClients, nil, func(u *unitDB) (pairs []countPair) { return u.Clients }),
} }
// Total counters: // Total counters: