Pull request 1796: 5661-opt-more-lock

Updates #5661.

Squashed commit of the following:

commit 0a1425df0ea1278c73ac88cbee053897d3daaf1b
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Mar 31 18:31:19 2023 +0300

    querylog: opt locks more
This commit is contained in:
Ainar Garipov 2023-03-31 18:44:51 +03:00
parent f191cb07a5
commit 3575aa0570
6 changed files with 54 additions and 50 deletions

View File

@ -82,15 +82,17 @@ func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
return return
} }
var resp map[string]any var entries []*logEntry
var oldest time.Time
func() { func() {
l.lock.Lock() l.confMu.RLock()
defer l.lock.Unlock() defer l.confMu.RUnlock()
entries, oldest := l.search(params) entries, oldest = l.search(params)
resp = l.entriesToJSON(entries, oldest)
}() }()
resp := entriesToJSON(entries, oldest, l.anonymizer.Load())
_ = aghhttp.WriteJSONResponse(w, r, resp) _ = aghhttp.WriteJSONResponse(w, r, resp)
} }
@ -105,8 +107,8 @@ func (l *queryLog) handleQueryLogClear(_ http.ResponseWriter, _ *http.Request) {
// //
// Deprecated: Remove it when migration to the new API is over. // Deprecated: Remove it when migration to the new API is over.
func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) { func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
l.lock.Lock() l.confMu.RLock()
defer l.lock.Unlock() defer l.confMu.RUnlock()
ivl := l.conf.RotationIvl ivl := l.conf.RotationIvl
@ -128,8 +130,8 @@ func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
func (l *queryLog) handleGetQueryLogConfig(w http.ResponseWriter, r *http.Request) { func (l *queryLog) handleGetQueryLogConfig(w http.ResponseWriter, r *http.Request) {
var resp *getConfigResp var resp *getConfigResp
func() { func() {
l.lock.Lock() l.confMu.RLock()
defer l.lock.Unlock() defer l.confMu.RUnlock()
resp = &getConfigResp{ resp = &getConfigResp{
Interval: float64(l.conf.RotationIvl.Milliseconds()), Interval: float64(l.conf.RotationIvl.Milliseconds()),
@ -137,9 +139,9 @@ func (l *queryLog) handleGetQueryLogConfig(w http.ResponseWriter, r *http.Reques
AnonymizeClientIP: aghalg.BoolToNullBool(l.conf.AnonymizeClientIP), AnonymizeClientIP: aghalg.BoolToNullBool(l.conf.AnonymizeClientIP),
Ignored: l.conf.Ignored.Values(), Ignored: l.conf.Ignored.Values(),
} }
}()
slices.Sort(resp.Ignored) slices.Sort(resp.Ignored)
}()
_ = aghhttp.WriteJSONResponse(w, r, resp) _ = aghhttp.WriteJSONResponse(w, r, resp)
} }
@ -187,8 +189,8 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
defer l.conf.ConfigModified() defer l.conf.ConfigModified()
l.lock.Lock() l.confMu.Lock()
defer l.lock.Unlock() defer l.confMu.Unlock()
conf := *l.conf conf := *l.conf
if newConf.Enabled != aghalg.NBNull { if newConf.Enabled != aghalg.NBNull {
@ -251,8 +253,8 @@ func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Reques
defer l.conf.ConfigModified() defer l.conf.ConfigModified()
l.lock.Lock() l.confMu.Lock()
defer l.lock.Unlock() defer l.confMu.Unlock()
conf := *l.conf conf := *l.conf

View File

@ -19,12 +19,16 @@ import (
type jobject = map[string]any type jobject = map[string]any
// entriesToJSON converts query log entries to JSON. // entriesToJSON converts query log entries to JSON.
func (l *queryLog) entriesToJSON(entries []*logEntry, oldest time.Time) (res jobject) { func entriesToJSON(
entries []*logEntry,
oldest time.Time,
anonFunc aghnet.IPMutFunc,
) (res jobject) {
data := make([]jobject, 0, len(entries)) data := make([]jobject, 0, len(entries))
// The elements order is already reversed to be from newer to older. // The elements order is already reversed to be from newer to older.
for _, entry := range entries { for _, entry := range entries {
jsonEntry := l.entryToJSON(entry, l.anonymizer.Load()) jsonEntry := entryToJSON(entry, anonFunc)
data = append(data, jsonEntry) data = append(data, jsonEntry)
} }
@ -40,7 +44,7 @@ func (l *queryLog) entriesToJSON(entries []*logEntry, oldest time.Time) (res job
} }
// entryToJSON converts a log entry's data into an entry for the JSON API. // entryToJSON converts a log entry's data into an entry for the JSON API.
func (l *queryLog) entryToJSON(entry *logEntry, anonFunc aghnet.IPMutFunc) (jsonEntry jobject) { func entryToJSON(entry *logEntry, anonFunc aghnet.IPMutFunc) (jsonEntry jobject) {
hostname := entry.QHost hostname := entry.QHost
question := jobject{ question := jobject{
"type": entry.QType, "type": entry.QType,
@ -92,14 +96,14 @@ func (l *queryLog) entryToJSON(entry *logEntry, anonFunc aghnet.IPMutFunc) (json
jsonEntry["service_name"] = entry.Result.ServiceName jsonEntry["service_name"] = entry.Result.ServiceName
} }
l.setMsgData(entry, jsonEntry) setMsgData(entry, jsonEntry)
l.setOrigAns(entry, jsonEntry) setOrigAns(entry, jsonEntry)
return jsonEntry return jsonEntry
} }
// setMsgData sets the message data in jsonEntry. // setMsgData sets the message data in jsonEntry.
func (l *queryLog) setMsgData(entry *logEntry, jsonEntry jobject) { func setMsgData(entry *logEntry, jsonEntry jobject) {
if len(entry.Answer) == 0 { if len(entry.Answer) == 0 {
return return
} }
@ -122,7 +126,7 @@ func (l *queryLog) setMsgData(entry *logEntry, jsonEntry jobject) {
} }
// setOrigAns sets the original answer data in jsonEntry. // setOrigAns sets the original answer data in jsonEntry.
func (l *queryLog) setOrigAns(entry *logEntry, jsonEntry jobject) { func setOrigAns(entry *logEntry, jsonEntry jobject) {
if len(entry.OrigAnswer) == 0 { if len(entry.OrigAnswer) == 0 {
return return
} }

View File

@ -25,9 +25,12 @@ const (
type queryLog struct { type queryLog struct {
findClient func(ids []string) (c *Client, err error) findClient func(ids []string) (c *Client, err error)
// confMu protects conf.
confMu *sync.RWMutex
conf *Config conf *Config
lock sync.Mutex
logFile string // path to the log file // logFile is the path to the log file.
logFile string
// bufferLock protects buffer. // bufferLock protects buffer.
bufferLock sync.RWMutex bufferLock sync.RWMutex
@ -279,8 +282,8 @@ func (l *queryLog) Add(params *AddParams) {
// ShouldLog returns true if request for the host should be logged. // ShouldLog returns true if request for the host should be logged.
func (l *queryLog) ShouldLog(host string, _, _ uint16) bool { func (l *queryLog) ShouldLog(host string, _, _ uint16) bool {
l.lock.Lock() l.confMu.RLock()
defer l.lock.Unlock() defer l.confMu.RUnlock()
return !l.isIgnored(host) return !l.isIgnored(host)
} }

View File

@ -259,11 +259,11 @@ func TestQueryLogShouldLog(t *testing.T) {
set := stringutil.NewSet(ignored1, ignored2) set := stringutil.NewSet(ignored1, ignored2)
l, err := newQueryLog(Config{ l, err := newQueryLog(Config{
Ignored: set,
Enabled: true, Enabled: true,
RotationIvl: timeutil.Day, RotationIvl: timeutil.Day,
MemSize: 100, MemSize: 100,
BaseDir: t.TempDir(), BaseDir: t.TempDir(),
Ignored: set,
}) })
require.NoError(t, err) require.NoError(t, err)

View File

@ -4,6 +4,7 @@ import (
"fmt" "fmt"
"net" "net"
"path/filepath" "path/filepath"
"sync"
"time" "time"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
@ -33,11 +34,14 @@ type QueryLog interface {
// Config is the query log configuration structure. // Config is the query log configuration structure.
type Config struct { type Config struct {
// Ignored is the list of host names, which should not be written to log.
Ignored *stringutil.Set
// Anonymizer processes the IP addresses to anonymize those if needed. // Anonymizer processes the IP addresses to anonymize those if needed.
Anonymizer *aghnet.IPMut Anonymizer *aghnet.IPMut
// ConfigModified is called when the configuration is changed, for // ConfigModified is called when the configuration is changed, for example
// example by HTTP requests. // by HTTP requests.
ConfigModified func() ConfigModified func()
// HTTPRegister registers an HTTP handler. // HTTPRegister registers an HTTP handler.
@ -49,20 +53,13 @@ type Config struct {
// BaseDir is the base directory for log files. // BaseDir is the base directory for log files.
BaseDir string BaseDir string
// RotationIvl is the interval for log rotation. After that period, the // RotationIvl is the interval for log rotation. After that period, the old
// old log file will be renamed, NOT deleted, so the actual log // log file will be renamed, NOT deleted, so the actual log retention time
// retention time is twice the interval. The value must be one of: // is twice the interval.
//
// 6 * time.Hour
// 1 * timeutil.Day
// 7 * timeutil.Day
// 30 * timeutil.Day
// 90 * timeutil.Day
//
RotationIvl time.Duration RotationIvl time.Duration
// MemSize is the number of entries kept in a memory buffer before they // MemSize is the number of entries kept in a memory buffer before they are
// are flushed to disk. // flushed to disk.
MemSize uint32 MemSize uint32
// Enabled tells if the query log is enabled. // Enabled tells if the query log is enabled.
@ -74,10 +71,6 @@ type Config struct {
// AnonymizeClientIP tells if the query log should anonymize clients' IP // AnonymizeClientIP tells if the query log should anonymize clients' IP
// addresses. // addresses.
AnonymizeClientIP bool AnonymizeClientIP bool
// Ignored is the list of host names, which should not be written to
// log.
Ignored *stringutil.Set
} }
// AddParams is the parameters for adding an entry. // AddParams is the parameters for adding an entry.
@ -150,11 +143,13 @@ func newQueryLog(conf Config) (l *queryLog, err error) {
l = &queryLog{ l = &queryLog{
findClient: findClient, findClient: findClient,
conf: &Config{},
confMu: &sync.RWMutex{},
logFile: filepath.Join(conf.BaseDir, queryLogFileName), logFile: filepath.Join(conf.BaseDir, queryLogFileName),
anonymizer: conf.Anonymizer, anonymizer: conf.Anonymizer,
} }
l.conf = &Config{}
*l.conf = conf *l.conf = conf
err = validateIvl(conf.RotationIvl) err = validateIvl(conf.RotationIvl)

View File

@ -167,8 +167,8 @@ func (l *queryLog) periodicRotate() {
// checkAndRotate rotates log files if those are older than the specified // checkAndRotate rotates log files if those are older than the specified
// rotation interval. // rotation interval.
func (l *queryLog) checkAndRotate() { func (l *queryLog) checkAndRotate() {
l.lock.Lock() l.confMu.RLock()
defer l.lock.Unlock() defer l.confMu.RUnlock()
oldest, err := l.readFileFirstTimeValue() oldest, err := l.readFileFirstTimeValue()
if err != nil && !errors.Is(err, os.ErrNotExist) { if err != nil && !errors.Is(err, os.ErrNotExist) {