2019-09-27 16:58:57 +01:00
|
|
|
package querylog
|
|
|
|
|
|
|
|
import (
|
2022-10-28 17:15:27 +01:00
|
|
|
"encoding/json"
|
2019-09-27 16:58:57 +01:00
|
|
|
"fmt"
|
2022-10-28 17:15:27 +01:00
|
|
|
"math"
|
2021-12-06 14:26:43 +00:00
|
|
|
"net"
|
2019-09-27 16:58:57 +01:00
|
|
|
"net/http"
|
2020-05-26 13:37:37 +01:00
|
|
|
"net/url"
|
2024-04-03 11:44:51 +01:00
|
|
|
"slices"
|
2020-05-26 13:37:37 +01:00
|
|
|
"strconv"
|
2021-06-30 09:04:48 +01:00
|
|
|
"strings"
|
2019-09-27 16:58:57 +01:00
|
|
|
"time"
|
|
|
|
|
2022-10-28 17:15:27 +01:00
|
|
|
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
|
2021-12-16 17:54:59 +00:00
|
|
|
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
2023-03-23 10:46:57 +00:00
|
|
|
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
2019-09-27 16:58:57 +01:00
|
|
|
"github.com/AdguardTeam/golibs/log"
|
2021-12-06 14:26:43 +00:00
|
|
|
"github.com/AdguardTeam/golibs/timeutil"
|
2021-06-30 09:04:48 +01:00
|
|
|
"golang.org/x/net/idna"
|
2019-09-27 16:58:57 +01:00
|
|
|
)
|
|
|
|
|
2022-10-28 17:15:27 +01:00
|
|
|
// configJSON is the JSON structure for the querylog configuration.
|
|
|
|
type configJSON struct {
|
|
|
|
// Interval is the querylog rotation interval. Use float64 here to support
|
|
|
|
// fractional numbers and not mess the API users by changing the units.
|
|
|
|
Interval float64 `json:"interval"`
|
|
|
|
|
2023-03-23 10:46:57 +00:00
|
|
|
// Enabled shows if the querylog is enabled. It is an aghalg.NullBool to
|
|
|
|
// be able to tell when it's set without using pointers.
|
2022-10-28 17:15:27 +01:00
|
|
|
Enabled aghalg.NullBool `json:"enabled"`
|
|
|
|
|
|
|
|
// AnonymizeClientIP shows if the clients' IP addresses must be anonymized.
|
|
|
|
// It is an [aghalg.NullBool] to be able to tell when it's set without using
|
|
|
|
// pointers.
|
|
|
|
AnonymizeClientIP aghalg.NullBool `json:"anonymize_client_ip"`
|
2020-05-26 13:37:37 +01:00
|
|
|
}
|
|
|
|
|
2023-03-23 10:46:57 +00:00
|
|
|
// getConfigResp is the JSON structure for the querylog configuration.
|
|
|
|
type getConfigResp struct {
|
|
|
|
// Ignored is the list of host names, which should not be written to log.
|
|
|
|
Ignored []string `json:"ignored"`
|
|
|
|
|
|
|
|
// Interval is the querylog rotation interval in milliseconds.
|
|
|
|
Interval float64 `json:"interval"`
|
|
|
|
|
|
|
|
// Enabled shows if the querylog is enabled. It is an aghalg.NullBool to
|
|
|
|
// be able to tell when it's set without using pointers.
|
|
|
|
Enabled aghalg.NullBool `json:"enabled"`
|
|
|
|
|
|
|
|
// AnonymizeClientIP shows if the clients' IP addresses must be anonymized.
|
|
|
|
// It is an aghalg.NullBool to be able to tell when it's set without using
|
|
|
|
// pointers.
|
|
|
|
//
|
|
|
|
// TODO(a.garipov): Consider using separate setting for statistics.
|
|
|
|
AnonymizeClientIP aghalg.NullBool `json:"anonymize_client_ip"`
|
|
|
|
}
|
|
|
|
|
2020-05-26 13:37:37 +01:00
|
|
|
// Register web handlers
|
|
|
|
func (l *queryLog) initWeb() {
|
2021-02-04 11:15:34 +00:00
|
|
|
l.conf.HTTPRegister(http.MethodGet, "/control/querylog", l.handleQueryLog)
|
|
|
|
l.conf.HTTPRegister(http.MethodPost, "/control/querylog_clear", l.handleQueryLogClear)
|
2023-03-23 10:46:57 +00:00
|
|
|
l.conf.HTTPRegister(http.MethodGet, "/control/querylog/config", l.handleGetQueryLogConfig)
|
|
|
|
l.conf.HTTPRegister(
|
|
|
|
http.MethodPut,
|
|
|
|
"/control/querylog/config/update",
|
|
|
|
l.handlePutQueryLogConfig,
|
|
|
|
)
|
2023-03-31 15:39:04 +01:00
|
|
|
|
|
|
|
// Deprecated handlers.
|
|
|
|
l.conf.HTTPRegister(http.MethodGet, "/control/querylog_info", l.handleQueryLogInfo)
|
|
|
|
l.conf.HTTPRegister(http.MethodPost, "/control/querylog_config", l.handleQueryLogConfig)
|
2020-05-26 13:37:37 +01:00
|
|
|
}
|
|
|
|
|
2023-03-31 15:39:04 +01:00
|
|
|
// handleQueryLog is the handler for the GET /control/querylog HTTP API.
|
2019-09-27 16:58:57 +01:00
|
|
|
func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
|
2023-03-31 15:39:04 +01:00
|
|
|
params, err := parseSearchParams(r)
|
2020-05-26 13:37:37 +01:00
|
|
|
if err != nil {
|
2023-03-31 15:39:04 +01:00
|
|
|
aghhttp.Error(r, w, http.StatusBadRequest, "parsing params: %s", err)
|
2021-12-16 17:54:59 +00:00
|
|
|
|
2020-05-26 13:37:37 +01:00
|
|
|
return
|
2019-09-27 16:58:57 +01:00
|
|
|
}
|
|
|
|
|
2023-03-31 16:44:51 +01:00
|
|
|
var entries []*logEntry
|
|
|
|
var oldest time.Time
|
2023-03-31 15:39:04 +01:00
|
|
|
func() {
|
2023-03-31 16:44:51 +01:00
|
|
|
l.confMu.RLock()
|
|
|
|
defer l.confMu.RUnlock()
|
2019-09-27 16:58:57 +01:00
|
|
|
|
2023-03-31 16:44:51 +01:00
|
|
|
entries, oldest = l.search(params)
|
2023-03-31 15:39:04 +01:00
|
|
|
}()
|
|
|
|
|
2023-03-31 16:44:51 +01:00
|
|
|
resp := entriesToJSON(entries, oldest, l.anonymizer.Load())
|
|
|
|
|
2023-08-29 18:03:40 +01:00
|
|
|
aghhttp.WriteJSONResponseOK(w, r, resp)
|
2019-09-27 16:58:57 +01:00
|
|
|
}
|
|
|
|
|
2023-03-31 15:39:04 +01:00
|
|
|
// handleQueryLogClear is the handler for the POST /control/querylog/clear HTTP
|
|
|
|
// API.
|
2020-05-26 13:37:37 +01:00
|
|
|
func (l *queryLog) handleQueryLogClear(_ http.ResponseWriter, _ *http.Request) {
|
2019-09-27 16:58:57 +01:00
|
|
|
l.clear()
|
|
|
|
}
|
|
|
|
|
2023-03-31 15:39:04 +01:00
|
|
|
// handleQueryLogInfo is the handler for the GET /control/querylog_info HTTP
|
|
|
|
// API.
|
2023-03-23 10:46:57 +00:00
|
|
|
//
|
|
|
|
// Deprecated: Remove it when migration to the new API is over.
|
2019-09-27 16:58:57 +01:00
|
|
|
func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
|
2023-03-31 16:44:51 +01:00
|
|
|
l.confMu.RLock()
|
|
|
|
defer l.confMu.RUnlock()
|
2023-03-23 10:46:57 +00:00
|
|
|
|
|
|
|
ivl := l.conf.RotationIvl
|
|
|
|
|
|
|
|
if !checkInterval(ivl) {
|
|
|
|
// NOTE: If interval is custom we set it to 90 days for compatibility
|
|
|
|
// with old API.
|
|
|
|
ivl = timeutil.Day * 90
|
|
|
|
}
|
|
|
|
|
2023-08-29 18:03:40 +01:00
|
|
|
aghhttp.WriteJSONResponseOK(w, r, configJSON{
|
2022-10-28 17:15:27 +01:00
|
|
|
Enabled: aghalg.BoolToNullBool(l.conf.Enabled),
|
2023-03-23 10:46:57 +00:00
|
|
|
Interval: ivl.Hours() / 24,
|
|
|
|
AnonymizeClientIP: aghalg.BoolToNullBool(l.conf.AnonymizeClientIP),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2023-03-31 15:39:04 +01:00
|
|
|
// handleGetQueryLogConfig is the handler for the GET /control/querylog/config
|
|
|
|
// HTTP API.
|
2023-03-23 10:46:57 +00:00
|
|
|
func (l *queryLog) handleGetQueryLogConfig(w http.ResponseWriter, r *http.Request) {
|
2023-03-31 15:39:04 +01:00
|
|
|
var resp *getConfigResp
|
|
|
|
func() {
|
2023-03-31 16:44:51 +01:00
|
|
|
l.confMu.RLock()
|
|
|
|
defer l.confMu.RUnlock()
|
2023-03-31 15:39:04 +01:00
|
|
|
|
|
|
|
resp = &getConfigResp{
|
|
|
|
Interval: float64(l.conf.RotationIvl.Milliseconds()),
|
|
|
|
Enabled: aghalg.BoolToNullBool(l.conf.Enabled),
|
|
|
|
AnonymizeClientIP: aghalg.BoolToNullBool(l.conf.AnonymizeClientIP),
|
|
|
|
Ignored: l.conf.Ignored.Values(),
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
2023-08-29 18:03:40 +01:00
|
|
|
aghhttp.WriteJSONResponseOK(w, r, resp)
|
2019-09-27 16:58:57 +01:00
|
|
|
}
|
|
|
|
|
2021-12-06 14:26:43 +00:00
|
|
|
// AnonymizeIP masks ip to anonymize the client if the ip is a valid one.
|
|
|
|
func AnonymizeIP(ip net.IP) {
|
2021-12-07 11:12:59 +00:00
|
|
|
// zeroes is a slice of zero bytes from which the IP address tail is copied.
|
|
|
|
// Using constant string as source of copying is more efficient than byte
|
|
|
|
// slice, see https://github.com/golang/go/issues/49997.
|
|
|
|
const zeroes = "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
|
|
|
|
2021-12-06 14:26:43 +00:00
|
|
|
if ip4 := ip.To4(); ip4 != nil {
|
2021-12-07 11:12:59 +00:00
|
|
|
copy(ip4[net.IPv4len-2:net.IPv4len], zeroes)
|
2021-12-06 14:26:43 +00:00
|
|
|
} else if len(ip) == net.IPv6len {
|
2021-12-07 11:12:59 +00:00
|
|
|
copy(ip[net.IPv6len-10:net.IPv6len], zeroes)
|
2021-12-06 14:26:43 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-31 15:39:04 +01:00
|
|
|
// handleQueryLogConfig is the handler for the POST /control/querylog_config
|
|
|
|
// HTTP API.
|
2023-03-23 10:46:57 +00:00
|
|
|
//
|
|
|
|
// Deprecated: Remove it when migration to the new API is over.
|
2019-09-27 16:58:57 +01:00
|
|
|
func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request) {
|
2022-10-28 17:15:27 +01:00
|
|
|
// Set NaN as initial value to be able to know if it changed later by
|
|
|
|
// comparing it to NaN.
|
|
|
|
newConf := &configJSON{
|
|
|
|
Interval: math.NaN(),
|
|
|
|
}
|
|
|
|
|
|
|
|
err := json.NewDecoder(r.Body).Decode(newConf)
|
2019-09-27 16:58:57 +01:00
|
|
|
if err != nil {
|
2021-12-16 17:54:59 +00:00
|
|
|
aghhttp.Error(r, w, http.StatusBadRequest, "%s", err)
|
|
|
|
|
2019-09-27 16:58:57 +01:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2022-10-28 17:15:27 +01:00
|
|
|
ivl := time.Duration(float64(timeutil.Day) * newConf.Interval)
|
2023-03-23 10:46:57 +00:00
|
|
|
|
2022-10-28 17:15:27 +01:00
|
|
|
hasIvl := !math.IsNaN(newConf.Interval)
|
|
|
|
if hasIvl && !checkInterval(ivl) {
|
|
|
|
aghhttp.Error(r, w, http.StatusBadRequest, "unsupported interval")
|
2021-12-16 17:54:59 +00:00
|
|
|
|
2019-09-27 16:58:57 +01:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2021-12-06 14:26:43 +00:00
|
|
|
defer l.conf.ConfigModified()
|
|
|
|
|
2023-03-31 16:44:51 +01:00
|
|
|
l.confMu.Lock()
|
|
|
|
defer l.confMu.Unlock()
|
2021-12-06 14:26:43 +00:00
|
|
|
|
2019-11-12 12:36:17 +00:00
|
|
|
conf := *l.conf
|
2022-10-28 17:15:27 +01:00
|
|
|
if newConf.Enabled != aghalg.NBNull {
|
|
|
|
conf.Enabled = newConf.Enabled == aghalg.NBTrue
|
2019-11-12 12:36:17 +00:00
|
|
|
}
|
2022-10-28 17:15:27 +01:00
|
|
|
|
|
|
|
if hasIvl {
|
2021-07-01 16:50:28 +01:00
|
|
|
conf.RotationIvl = ivl
|
2019-09-27 16:58:57 +01:00
|
|
|
}
|
2022-10-28 17:15:27 +01:00
|
|
|
|
|
|
|
if newConf.AnonymizeClientIP != aghalg.NBNull {
|
|
|
|
conf.AnonymizeClientIP = newConf.AnonymizeClientIP == aghalg.NBTrue
|
|
|
|
if conf.AnonymizeClientIP {
|
2021-12-06 14:26:43 +00:00
|
|
|
l.anonymizer.Store(AnonymizeIP)
|
|
|
|
} else {
|
|
|
|
l.anonymizer.Store(nil)
|
|
|
|
}
|
2020-03-03 17:21:53 +00:00
|
|
|
}
|
2022-10-28 17:15:27 +01:00
|
|
|
|
2019-11-12 12:36:17 +00:00
|
|
|
l.conf = &conf
|
2019-09-27 16:58:57 +01:00
|
|
|
}
|
|
|
|
|
2023-03-31 15:39:04 +01:00
|
|
|
// handlePutQueryLogConfig is the handler for the PUT
|
|
|
|
// /control/querylog/config/update HTTP API.
|
2023-03-23 10:46:57 +00:00
|
|
|
func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Request) {
|
|
|
|
newConf := &getConfigResp{}
|
|
|
|
err := json.NewDecoder(r.Body).Decode(newConf)
|
|
|
|
if err != nil {
|
|
|
|
aghhttp.Error(r, w, http.StatusBadRequest, "%s", err)
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-09-05 13:13:35 +01:00
|
|
|
engine, err := aghnet.NewIgnoreEngine(newConf.Ignored)
|
2023-03-23 10:46:57 +00:00
|
|
|
if err != nil {
|
|
|
|
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err)
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
ivl := time.Duration(newConf.Interval) * time.Millisecond
|
|
|
|
err = validateIvl(ivl)
|
|
|
|
if err != nil {
|
|
|
|
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "unsupported interval: %s", err)
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if newConf.Enabled == aghalg.NBNull {
|
|
|
|
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "enabled is null")
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if newConf.AnonymizeClientIP == aghalg.NBNull {
|
|
|
|
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "anonymize_client_ip is null")
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
defer l.conf.ConfigModified()
|
|
|
|
|
2023-03-31 16:44:51 +01:00
|
|
|
l.confMu.Lock()
|
|
|
|
defer l.confMu.Unlock()
|
2023-03-23 10:46:57 +00:00
|
|
|
|
|
|
|
conf := *l.conf
|
|
|
|
|
2023-09-05 13:13:35 +01:00
|
|
|
conf.Ignored = engine
|
2023-03-23 10:46:57 +00:00
|
|
|
conf.RotationIvl = ivl
|
|
|
|
conf.Enabled = newConf.Enabled == aghalg.NBTrue
|
|
|
|
|
|
|
|
conf.AnonymizeClientIP = newConf.AnonymizeClientIP == aghalg.NBTrue
|
|
|
|
if conf.AnonymizeClientIP {
|
|
|
|
l.anonymizer.Store(AnonymizeIP)
|
|
|
|
} else {
|
|
|
|
l.anonymizer.Store(nil)
|
|
|
|
}
|
|
|
|
|
|
|
|
l.conf = &conf
|
|
|
|
}
|
|
|
|
|
2020-05-26 13:37:37 +01:00
|
|
|
// "value" -> value, return TRUE
|
|
|
|
func getDoubleQuotesEnclosedValue(s *string) bool {
|
|
|
|
t := *s
|
|
|
|
if len(t) >= 2 && t[0] == '"' && t[len(t)-1] == '"' {
|
|
|
|
*s = t[1 : len(t)-1]
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2021-04-12 16:22:11 +01:00
|
|
|
// parseSearchCriterion parses a search criterion from the query parameter.
|
2023-03-31 15:39:04 +01:00
|
|
|
func parseSearchCriterion(q url.Values, name string, ct criterionType) (
|
2021-06-30 09:04:48 +01:00
|
|
|
ok bool,
|
|
|
|
sc searchCriterion,
|
|
|
|
err error,
|
|
|
|
) {
|
2020-05-26 13:37:37 +01:00
|
|
|
val := q.Get(name)
|
2021-06-30 09:04:48 +01:00
|
|
|
if val == "" {
|
|
|
|
return false, sc, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
strict := getDoubleQuotesEnclosedValue(&val)
|
|
|
|
|
|
|
|
var asciiVal string
|
|
|
|
switch ct {
|
|
|
|
case ctTerm:
|
|
|
|
// Decode lowercased value from punycode to make EqualFold and
|
|
|
|
// friends work properly with IDNAs.
|
|
|
|
//
|
|
|
|
// TODO(e.burkov): Make it work with parts of IDNAs somehow.
|
|
|
|
loweredVal := strings.ToLower(val)
|
|
|
|
if asciiVal, err = idna.ToASCII(loweredVal); err != nil {
|
|
|
|
log.Debug("can't convert %q to ascii: %s", val, err)
|
|
|
|
} else if asciiVal == loweredVal {
|
|
|
|
// Purge asciiVal to prevent checking the same value
|
|
|
|
// twice.
|
|
|
|
asciiVal = ""
|
|
|
|
}
|
|
|
|
case ctFilteringStatus:
|
2024-04-03 11:44:51 +01:00
|
|
|
if !slices.Contains(filteringStatusValues, val) {
|
2021-06-30 09:04:48 +01:00
|
|
|
return false, sc, fmt.Errorf("invalid value %s", val)
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
return false, sc, fmt.Errorf(
|
|
|
|
"invalid criterion type %v: should be one of %v",
|
|
|
|
ct,
|
|
|
|
[]criterionType{ctTerm, ctFilteringStatus},
|
|
|
|
)
|
2020-05-26 13:37:37 +01:00
|
|
|
}
|
|
|
|
|
2021-06-30 09:04:48 +01:00
|
|
|
sc = searchCriterion{
|
2021-04-12 16:22:11 +01:00
|
|
|
criterionType: ct,
|
|
|
|
value: val,
|
2021-06-30 09:04:48 +01:00
|
|
|
asciiVal: asciiVal,
|
|
|
|
strict: strict,
|
2020-05-26 13:37:37 +01:00
|
|
|
}
|
|
|
|
|
2021-06-30 09:04:48 +01:00
|
|
|
return true, sc, nil
|
2020-05-26 13:37:37 +01:00
|
|
|
}
|
|
|
|
|
2023-03-31 15:39:04 +01:00
|
|
|
// parseSearchParams parses search parameters from the HTTP request's query
|
|
|
|
// string.
|
|
|
|
func parseSearchParams(r *http.Request) (p *searchParams, err error) {
|
2021-03-11 17:36:54 +00:00
|
|
|
p = newSearchParams()
|
2020-05-26 13:37:37 +01:00
|
|
|
|
|
|
|
q := r.URL.Query()
|
|
|
|
olderThan := q.Get("older_than")
|
|
|
|
if len(olderThan) != 0 {
|
|
|
|
p.olderThan, err = time.Parse(time.RFC3339Nano, olderThan)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-11 17:36:54 +00:00
|
|
|
var limit64 int64
|
|
|
|
if limit64, err = strconv.ParseInt(q.Get("limit"), 10, 64); err == nil {
|
|
|
|
p.limit = int(limit64)
|
2020-05-26 13:37:37 +01:00
|
|
|
}
|
2021-03-11 17:36:54 +00:00
|
|
|
|
|
|
|
var offset64 int64
|
|
|
|
if offset64, err = strconv.ParseInt(q.Get("offset"), 10, 64); err == nil {
|
|
|
|
p.offset = int(offset64)
|
2020-09-01 14:30:30 +01:00
|
|
|
|
|
|
|
// If we don't use "olderThan" and use offset/limit instead, we should change the default behavior
|
|
|
|
// and scan all log records until we found enough log entries
|
2020-05-26 13:37:37 +01:00
|
|
|
p.maxFileScanEntries = 0
|
|
|
|
}
|
|
|
|
|
2021-06-30 09:04:48 +01:00
|
|
|
for _, v := range []struct {
|
|
|
|
urlField string
|
|
|
|
ct criterionType
|
|
|
|
}{{
|
|
|
|
urlField: "search",
|
|
|
|
ct: ctTerm,
|
|
|
|
}, {
|
|
|
|
urlField: "response_status",
|
|
|
|
ct: ctFilteringStatus,
|
|
|
|
}} {
|
2021-03-11 17:36:54 +00:00
|
|
|
var ok bool
|
2021-04-12 16:22:11 +01:00
|
|
|
var c searchCriterion
|
2023-03-31 15:39:04 +01:00
|
|
|
ok, c, err = parseSearchCriterion(q, v.urlField, v.ct)
|
2020-05-26 13:37:37 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if ok {
|
|
|
|
p.searchCriteria = append(p.searchCriteria, c)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return p, nil
|
2019-09-27 16:58:57 +01:00
|
|
|
}
|