+ config: new setting "querylog_file_enabled"

Close #876

Squashed commit of the following:

commit f83f60a7340d8a3f6de7ecfebb426e47d19e83d8
Merge: cfb72869 391e6199
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu May 28 15:17:21 2020 +0300

    Merge remote-tracking branch 'origin/master' into 876-logs

commit cfb72869f7cf0bf59a478ab8c7920c273e2fa5f9
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu May 28 12:50:02 2020 +0300

    tests

commit 35376e4f450cf66507d733c931b7ed27eff1f36c
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Wed May 27 18:15:12 2020 +0300

    fix

commit 0cfb802d73db52a4b09c459a68a8a18918447b76
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Wed May 27 16:49:52 2020 +0300

    tests

commit 03ca280b6aed3a4880a9d4f4cd18bf47b1c742f6
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Wed May 27 15:32:27 2020 +0300

    + config: new setting "querylog_file_enabled" - query log will be written to a file
This commit is contained in:
Simon Zolin 2020-05-28 15:29:36 +03:00
parent 391e619979
commit 32d1f385ff
6 changed files with 60 additions and 30 deletions

View File

@ -78,10 +78,11 @@ type dnsConfig struct {
// time interval for statistics (in days) // time interval for statistics (in days)
StatsInterval uint32 `yaml:"statistics_interval"` StatsInterval uint32 `yaml:"statistics_interval"`
QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days) QueryLogFileEnabled bool `yaml:"querylog_file_enabled"` // if true, query log will be written to a file
QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days)
AnonymizeClientIP bool `yaml:"anonymize_client_ip"` // anonymize clients' IP addresses in logs and stats QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
AnonymizeClientIP bool `yaml:"anonymize_client_ip"` // anonymize clients' IP addresses in logs and stats
dnsforward.FilteringConfig `yaml:",inline"` dnsforward.FilteringConfig `yaml:",inline"`
@ -138,6 +139,7 @@ func initConfig() {
config.WebSessionTTLHours = 30 * 24 config.WebSessionTTLHours = 30 * 24
config.DNS.QueryLogEnabled = true config.DNS.QueryLogEnabled = true
config.DNS.QueryLogFileEnabled = true
config.DNS.QueryLogInterval = 90 config.DNS.QueryLogInterval = 90
config.DNS.QueryLogMemSize = 1000 config.DNS.QueryLogMemSize = 1000
@ -239,9 +241,10 @@ func (c *configuration) write() error {
} }
if Context.queryLog != nil { if Context.queryLog != nil {
dc := querylog.DiskConfig{} dc := querylog.Config{}
Context.queryLog.WriteDiskConfig(&dc) Context.queryLog.WriteDiskConfig(&dc)
config.DNS.QueryLogEnabled = dc.Enabled config.DNS.QueryLogEnabled = dc.Enabled
config.DNS.QueryLogFileEnabled = dc.FileEnabled
config.DNS.QueryLogInterval = dc.Interval config.DNS.QueryLogInterval = dc.Interval
config.DNS.QueryLogMemSize = dc.MemSize config.DNS.QueryLogMemSize = dc.MemSize
config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP

View File

@ -40,6 +40,7 @@ func initDNSServer() error {
} }
conf := querylog.Config{ conf := querylog.Config{
Enabled: config.DNS.QueryLogEnabled, Enabled: config.DNS.QueryLogEnabled,
FileEnabled: config.DNS.QueryLogFileEnabled,
BaseDir: baseDir, BaseDir: baseDir,
Interval: config.DNS.QueryLogInterval, Interval: config.DNS.QueryLogInterval,
MemSize: config.DNS.QueryLogMemSize, MemSize: config.DNS.QueryLogMemSize,

View File

@ -73,11 +73,8 @@ func checkInterval(days uint32) bool {
return days == 1 || days == 7 || days == 30 || days == 90 return days == 1 || days == 7 || days == 30 || days == 90
} }
func (l *queryLog) WriteDiskConfig(dc *DiskConfig) { func (l *queryLog) WriteDiskConfig(c *Config) {
dc.Enabled = l.conf.Enabled *c = *l.conf
dc.Interval = l.conf.Interval
dc.MemSize = l.conf.MemSize
dc.AnonymizeClientIP = l.conf.AnonymizeClientIP
} }
// Clear memory buffer and remove log files // Clear memory buffer and remove log files
@ -152,7 +149,14 @@ func (l *queryLog) Add(params AddParams) {
l.bufferLock.Lock() l.bufferLock.Lock()
l.buffer = append(l.buffer, &entry) l.buffer = append(l.buffer, &entry)
needFlush := false needFlush := false
if !l.flushPending {
if !l.conf.FileEnabled {
if len(l.buffer) > int(l.conf.MemSize) {
// writing to file is disabled - just remove the oldest entry from array
l.buffer = l.buffer[1:]
}
} else if !l.flushPending {
needFlush = len(l.buffer) >= int(l.conf.MemSize) needFlush = len(l.buffer) >= int(l.conf.MemSize)
if needFlush { if needFlush {
l.flushPending = true l.flushPending = true
@ -162,8 +166,8 @@ func (l *queryLog) Add(params AddParams) {
// if buffer needs to be flushed to disk, do it now // if buffer needs to be flushed to disk, do it now
if needFlush { if needFlush {
// write to file go func() {
// do it in separate goroutine -- we are stalling DNS response this whole time _ = l.flushLogBuffer(false)
go l.flushLogBuffer(false) // nolint }()
} }
} }

View File

@ -22,9 +22,10 @@ func prepareTestDir() string {
// Check adding and loading (with filtering) entries from disk and memory // Check adding and loading (with filtering) entries from disk and memory
func TestQueryLog(t *testing.T) { func TestQueryLog(t *testing.T) {
conf := Config{ conf := Config{
Enabled: true, Enabled: true,
Interval: 1, FileEnabled: true,
MemSize: 100, Interval: 1,
MemSize: 100,
} }
conf.BaseDir = prepareTestDir() conf.BaseDir = prepareTestDir()
defer func() { _ = os.RemoveAll(conf.BaseDir) }() defer func() { _ = os.RemoveAll(conf.BaseDir) }()
@ -158,9 +159,10 @@ func TestQueryLogOffsetLimit(t *testing.T) {
func TestQueryLogMaxFileScanEntries(t *testing.T) { func TestQueryLogMaxFileScanEntries(t *testing.T) {
conf := Config{ conf := Config{
Enabled: true, Enabled: true,
Interval: 1, FileEnabled: true,
MemSize: 100, Interval: 1,
MemSize: 100,
} }
conf.BaseDir = prepareTestDir() conf.BaseDir = prepareTestDir()
defer func() { _ = os.RemoveAll(conf.BaseDir) }() defer func() { _ = os.RemoveAll(conf.BaseDir) }()
@ -183,6 +185,29 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) {
assert.Equal(t, 10, len(entries)) assert.Equal(t, 10, len(entries))
} }
func TestQueryLogFileDisabled(t *testing.T) {
conf := Config{
Enabled: true,
FileEnabled: false,
Interval: 1,
MemSize: 2,
}
conf.BaseDir = prepareTestDir()
defer func() { _ = os.RemoveAll(conf.BaseDir) }()
l := newQueryLog(conf)
addEntry(l, "example1.org", "1.1.1.1", "2.2.2.1")
addEntry(l, "example2.org", "1.1.1.1", "2.2.2.1")
addEntry(l, "example3.org", "1.1.1.1", "2.2.2.1")
// the oldest entry is now removed from mem buffer
params := newSearchParams()
ll, _ := l.search(params)
assert.Equal(t, 2, len(ll))
assert.Equal(t, "example3.org", ll[0].QHost)
assert.Equal(t, "example2.org", ll[1].QHost)
}
func addEntry(l *queryLog, host, answerStr, client string) { func addEntry(l *queryLog, host, answerStr, client string) {
q := dns.Msg{} q := dns.Msg{}
q.Question = append(q.Question, dns.Question{ q.Question = append(q.Question, dns.Question{

View File

@ -9,14 +9,6 @@ import (
"github.com/miekg/dns" "github.com/miekg/dns"
) )
// DiskConfig - configuration settings that are stored on disk
type DiskConfig struct {
Enabled bool
Interval uint32
MemSize uint32
AnonymizeClientIP bool
}
// QueryLog - main interface // QueryLog - main interface
type QueryLog interface { type QueryLog interface {
Start() Start()
@ -28,12 +20,13 @@ type QueryLog interface {
Add(params AddParams) Add(params AddParams)
// WriteDiskConfig - write configuration // WriteDiskConfig - write configuration
WriteDiskConfig(dc *DiskConfig) WriteDiskConfig(c *Config)
} }
// Config - configuration object // Config - configuration object
type Config struct { type Config struct {
Enabled bool Enabled bool // enable the module
FileEnabled bool // write logs to file
BaseDir string // directory where log file is stored BaseDir string // directory where log file is stored
Interval uint32 // interval to rotate logs (in days) Interval uint32 // interval to rotate logs (in days)
MemSize uint32 // number of entries kept in memory before they are flushed to disk MemSize uint32 // number of entries kept in memory before they are flushed to disk

View File

@ -11,6 +11,10 @@ import (
// flushLogBuffer flushes the current buffer to file and resets the current buffer // flushLogBuffer flushes the current buffer to file and resets the current buffer
func (l *queryLog) flushLogBuffer(fullFlush bool) error { func (l *queryLog) flushLogBuffer(fullFlush bool) error {
if !l.conf.FileEnabled {
return nil
}
l.fileFlushLock.Lock() l.fileFlushLock.Lock()
defer l.fileFlushLock.Unlock() defer l.fileFlushLock.Unlock()