mirrors
/
AdGuardHome
mirror of https://github.com/AdguardTeam/AdGuardHome.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

all: add new querylog stats api

This commit is contained in:
Stanislav Chzhen 2023-02-15 13:13:32 +03:00
parent 3378d1c103
commit 480b42c596
6 changed files with 229 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
internal/aghnet/hostgen.go
View File

@ -1,8 +1,13 @@
package aghnet
import (
"fmt"
"net"
"strconv"
"strings"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/stringutil"
)
// The maximum lengths of generated hostnames for different IP versions.
@ -59,3 +64,27 @@ func GenerateHostname(ip net.IP) (hostname string) {
return generateIPv6Hostname(ip)
}
// NewDomainNameSet returns nil and error, if list has duplicate or empty host
// name. Otherwise returns a set, which contains lowercase host names without
// dot at the end, and nil error.
func NewDomainNameSet(list []string) (set *stringutil.Set, err error) {
set = stringutil.NewSet()
for _, v := range list {
host := strings.ToLower(strings.TrimSuffix(v, "."))
// TODO(a.garipov): Think about ignoring empty (".") names in
// the future.
if host == "" {
return nil, errors.Error("host name is empty")
}
if set.Has(host) {
return nil, fmt.Errorf("duplicate host name %q", host)
}
set.Add(host)
}
return set, nil
}

30
internal/home/dns.go
View File

@ -7,7 +7,6 @@ import (
"net/url"
"os"
"path/filepath"
"strings"
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
@ -21,7 +20,6 @@ import (
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/ameshkov/dnscrypt/v2"
yaml "gopkg.in/yaml.v3"
)
@ -59,7 +57,7 @@ func initDNS() (err error) {
Enabled: config.Stats.Enabled,
}
set, err := nonDupEmptyHostNames(config.Stats.Ignored)
set, err := aghnet.NewDomainNameSet(config.Stats.Ignored)
if err != nil {
return fmt.Errorf("statistics: ignored list: %w", err)
}
@ -83,7 +81,7 @@ func initDNS() (err error) {
FileEnabled: config.QueryLog.FileEnabled,
}
set, err = nonDupEmptyHostNames(config.QueryLog.Ignored)
set, err = aghnet.NewDomainNameSet(config.QueryLog.Ignored)
if err != nil {
return fmt.Errorf("querylog: ignored list: %w", err)
}
@ -532,27 +530,3 @@ func closeDNSServer() {
log.Debug("all dns modules are closed")
}
// nonDupEmptyHostNames returns nil and error, if list has duplicate or empty
// host name. Otherwise returns a set, which contains lowercase host names
// without dot at the end, and nil error.
func nonDupEmptyHostNames(list []string) (set *stringutil.Set, err error) {
set = stringutil.NewSet()
for _, v := range list {
host := strings.ToLower(strings.TrimSuffix(v, "."))
// TODO(a.garipov): Think about ignoring empty (".") names in
// the future.
if host == "" {
return nil, errors.Error("host name is empty")
}
if set.Has(host) {
return nil, fmt.Errorf("duplicate host name %q", host)
}
set.Add(host)
}
return set, nil
}

109
internal/querylog/http.go
View File

@ -13,6 +13,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/timeutil"
@ -35,12 +36,39 @@ type configJSON struct {
AnonymizeClientIP aghalg.NullBool `json:"anonymize_client_ip"`
}
// configJSONv2 is the JSON structure for the querylog configuration.
type configJSONv2 struct {
// Enabled shows if the querylog is enabled. It is an [aghalg.NullBool]
// to be able to tell when it's set without using pointers.
Enabled aghalg.NullBool `json:"enabled"`
// AnonymizeClientIP shows if the clients' IP addresses must be anonymized.
// It is an [aghalg.NullBool] to be able to tell when it's set without using
// pointers.
AnonymizeClientIP aghalg.NullBool `json:"anonymize_client_ip"`
// Interval is the querylog rotation interval.
Interval float64 `json:"interval"`
// Ignored is the list of host names, which should not be written to
// log.
Ignored []string `json:"ignored,omitempty"`
}
// Register web handlers
func (l *queryLog) initWeb() {
l.conf.HTTPRegister(http.MethodGet, "/control/querylog", l.handleQueryLog)
l.conf.HTTPRegister(http.MethodGet, "/control/querylog_info", l.handleQueryLogInfo)
l.conf.HTTPRegister(http.MethodPost, "/control/querylog_clear", l.handleQueryLogClear)
l.conf.HTTPRegister(http.MethodPost, "/control/querylog_config", l.handleQueryLogConfig)
// API v2.
l.conf.HTTPRegister(http.MethodGet, "/control/querylog/config", l.handleQueryLogInfoV2)
l.conf.HTTPRegister(
http.MethodPut,
"/control/querylog/config/update",
l.handleQueryLogConfigV2,
)
}
func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
@ -67,8 +95,12 @@ func (l *queryLog) handleQueryLogClear(_ http.ResponseWriter, _ *http.Request) {
l.clear()
}
// Get configuration
// handleQueryLogInfo handles requests to the GET /control/querylog_info
// endpoint.
func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
l.lock.Lock()
defer l.lock.Unlock()
_ = aghhttp.WriteJSONResponse(w, r, configJSON{
Enabled: aghalg.BoolToNullBool(l.conf.Enabled),
Interval: l.conf.RotationIvl.Hours() / 24,
@ -76,6 +108,22 @@ func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
})
}
// handleQueryLogInfoV2 handles requests to the GET
// /control/querylog/config endpoint.
func (l *queryLog) handleQueryLogInfoV2(w http.ResponseWriter, r *http.Request) {
l.lock.Lock()
defer l.lock.Unlock()
ignored := l.conf.Ignored.Values()
_ = aghhttp.WriteJSONResponse(w, r, configJSONv2{
Enabled: aghalg.BoolToNullBool(l.conf.Enabled),
// 1 hour = 3,600,000 ms
Interval: l.conf.RotationIvl.Hours() * 3_600_000,
AnonymizeClientIP: aghalg.BoolToNullBool(l.conf.AnonymizeClientIP),
Ignored: ignored,
})
}
// AnonymizeIP masks ip to anonymize the client if the ip is a valid one.
func AnonymizeIP(ip net.IP) {
// zeroes is a slice of zero bytes from which the IP address tail is copied.
@ -141,6 +189,65 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
l.conf = &conf
}
// handleQueryLogConfigV2 handles the PUT /control/querylog/config/update
// queries.
func (l *queryLog) handleQueryLogConfigV2(w http.ResponseWriter, r *http.Request) {
// Set NaN as initial value to be able to know if it changed later by
// comparing it to NaN.
newConf := &configJSONv2{}
err := json.NewDecoder(r.Body).Decode(newConf)
if err != nil {
aghhttp.Error(r, w, http.StatusBadRequest, "%s", err)
return
}
ivl := time.Duration(float64(time.Millisecond) * newConf.Interval)
if ivl < time.Hour || ivl > timeutil.Day*366 || !checkInterval(ivl) {
aghhttp.Error(r, w, http.StatusBadRequest, "unsupported interval")
return
}
defer l.conf.ConfigModified()
l.lock.Lock()
defer l.lock.Unlock()
// Copy data, modify it, then activate. Other threads (readers) don't need
// to use this lock.
conf := *l.conf
if newConf.Enabled != aghalg.NBNull {
conf.Enabled = newConf.Enabled == aghalg.NBTrue
}
conf.RotationIvl = ivl
if newConf.AnonymizeClientIP != aghalg.NBNull {
conf.AnonymizeClientIP = newConf.AnonymizeClientIP == aghalg.NBTrue
if conf.AnonymizeClientIP {
l.anonymizer.Store(AnonymizeIP)
} else {
l.anonymizer.Store(nil)
}
}
if len(newConf.Ignored) > 0 {
set, serr := aghnet.NewDomainNameSet(newConf.Ignored)
if serr != nil {
aghhttp.Error(r, w, http.StatusBadRequest, "ignored: %s", serr)
return
} else {
conf.Ignored = set
}
}
l.conf = &conf
}
// "value" -> value, return TRUE
func getDoubleQuotesEnclosedValue(s *string) bool {
t := *s

3
internal/querylog/qlog.go
View File

@ -250,6 +250,9 @@ func (l *queryLog) Add(params *AddParams) {
// ShouldLog returns true if request for the host should be logged.
func (l *queryLog) ShouldLog(host string, _, _ uint16) bool {
l.lock.Lock()
defer l.lock.Unlock()
return !l.isIgnored(host)
}

91
internal/stats/http.go
View File

@ -7,8 +7,11 @@ import (
"net/http"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/timeutil"
)
// topAddrs is an alias for the types of the TopFoo fields of statsResponse.
@ -63,6 +66,19 @@ type configResp struct {
IntervalDays uint32 `json:"interval"`
}
// configRespV2 is the response to the GET /control/stats_info.
type configRespV2 struct {
// Enabled shows if statistics are enabled. It is an [aghalg.NullBool]
// to be able to tell when it's set without using pointers.
Enabled aghalg.NullBool `json:"enabled"`
// Interval is the statistics rotation interval.
Interval float64 `json:"interval"`
// Ignored is the list of host names, which should not be counted.
Ignored []string `json:"ignored,omitempty"`
}
// handleStatsInfo handles requests to the GET /control/stats_info endpoint.
func (s *StatsCtx) handleStatsInfo(w http.ResponseWriter, r *http.Request) {
s.lock.Lock()
@ -75,25 +91,86 @@ func (s *StatsCtx) handleStatsInfo(w http.ResponseWriter, r *http.Request) {
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
// handleStatsConfig handles requests to the POST /control/stats_config
// handleStatsInfoV2 handles requests to the GET /control/stats_info endpoint.
func (s *StatsCtx) handleStatsInfoV2(w http.ResponseWriter, r *http.Request) {
s.lock.Lock()
defer s.lock.Unlock()
resp := configRespV2{
Enabled: aghalg.BoolToNullBool(s.enabled),
// 1 hour = 3,600,600 ms
Interval: float64(s.limitHours) * 3_600_000,
Ignored: s.ignored.Values(),
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
// handleStatsConfig handles requests to the POST /v2/control/stats_config
// endpoint.
func (s *StatsCtx) handleStatsConfig(w http.ResponseWriter, r *http.Request) {
reqData := configResp{}
err := json.NewDecoder(r.Body).Decode(&reqData)
if err != nil {
aghhttp.Error(r, w, http.StatusBadRequest, "json decode: %s", err)
s.lock.Unlock()
return
}
if !checkInterval(reqData.IntervalDays) {
aghhttp.Error(r, w, http.StatusBadRequest, "Unsupported interval")
s.lock.Unlock()
return
}
defer s.configModified()
s.lock.Lock()
defer s.lock.Unlock()
s.setLimit(int(reqData.IntervalDays))
s.configModified()
}
// handleStatsConfigV2 handles requests to the POST /v2/control/stats_config
// endpoint.
func (s *StatsCtx) handleStatsConfigV2(w http.ResponseWriter, r *http.Request) {
reqData := configRespV2{}
err := json.NewDecoder(r.Body).Decode(&reqData)
if err != nil {
aghhttp.Error(r, w, http.StatusBadRequest, "json decode: %s", err)
return
}
if !checkInterval(reqData.IntervalDays) {
aghhttp.Error(r, w, http.StatusBadRequest, "Unsupported interval")
ivl := time.Duration(float64(time.Millisecond) * reqData.Interval)
days := ivl.Hours() / 24
if ivl < time.Hour || ivl > timeutil.Day*366 || !checkInterval(uint32(days)) {
aghhttp.Error(r, w, http.StatusBadRequest, "unsupported interval")
return
}
s.setLimit(int(reqData.IntervalDays))
s.configModified()
defer s.configModified()
s.lock.Lock()
defer s.lock.Unlock()
s.setLimit(int(days))
if len(reqData.Ignored) > 0 {
set, serr := aghnet.NewDomainNameSet(reqData.Ignored)
if serr != nil {
aghhttp.Error(r, w, http.StatusBadRequest, "ignored: %s", serr)
return
} else {
s.ignored = set
}
}
}
// handleStatsReset handles requests to the POST /control/stats_reset endpoint.
@ -114,4 +191,8 @@ func (s *StatsCtx) initWeb() {
s.httpRegister(http.MethodPost, "/control/stats_reset", s.handleStatsReset)
s.httpRegister(http.MethodPost, "/control/stats_config", s.handleStatsConfig)
s.httpRegister(http.MethodGet, "/control/stats_info", s.handleStatsInfo)
// API v2.
s.httpRegister(http.MethodGet, "/control/stats/config", s.handleStatsInfoV2)
s.httpRegister(http.MethodPut, "/control/stats/config/update", s.handleStatsConfigV2)
}

4
internal/stats/stats.go
View File

@ -436,10 +436,8 @@ func (s *StatsCtx) periodicFlush() {
log.Debug("periodic flushing finished")
}
// s.lock is expected to be locked.
func (s *StatsCtx) setLimit(limitDays int) {
s.lock.Lock()
defer s.lock.Unlock()
if limitDays != 0 {
s.enabled = true
s.limitHours = uint32(24 * limitDays)