AdGuardHome/internal/querylog/json.go

208 lines
4.7 KiB
Go
Raw Normal View History

package querylog
import (
"context"
"slices"
"strconv"
"strings"
"time"
Pull request: 3890 fix anonymization Merge in DNS/adguard-home from 3890-fix-stats to master Updates #3890. Squashed commit of the following: commit a77a6204bc8a58f62a4fac70efdcae4267a64810 Merge: 834493a2 90e65b66 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 17:22:16 2021 +0300 Merge branch 'master' into 3890-fix-stats commit 834493a22ae79199efcc44e0715e2ac6f6272963 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 17:09:30 2021 +0300 querylog: load once commit b8000e7ba7a998fcd4553230ec5e5f9c90106e31 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 16:54:41 2021 +0300 querylog: fix docs commit 7db99ccfa19b58100950c11d67b23bca7af3e5cb Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 16:51:31 2021 +0300 querylog: imp docs commit 2a84650bd7ac5195730a7ab47b9562a83f721499 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 15:48:09 2021 +0300 querylog: imp anonyization commit 0f63feb1ff5f006fc528c3b681ef3b9d2199581e Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:44:37 2021 +0300 all: imp code & docs commit c4ccdcbb7248897edd178fd5cb77127e39ada73d Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:24:30 2021 +0300 all: log changes commit 60bb777a5aff36bba129a078fa11ae566298178a Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:08:41 2021 +0300 all: use atomic value commit c45886bd20eee2212b42686ff369830d8c08fe36 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Nov 30 18:50:02 2021 +0300 all: anonymize separately
2021-12-06 14:26:43 +00:00
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/golibs/logutil/slogutil"
"github.com/miekg/dns"
"golang.org/x/net/idna"
)
// TODO(a.garipov): Use a proper structured approach here.
// jobject is a JSON object alias.
type jobject = map[string]any
// entriesToJSON converts query log entries to JSON.
func (l *queryLog) entriesToJSON(
ctx context.Context,
entries []*logEntry,
oldest time.Time,
anonFunc aghnet.IPMutFunc,
) (res jobject) {
Pull request: 3890 fix anonymization Merge in DNS/adguard-home from 3890-fix-stats to master Updates #3890. Squashed commit of the following: commit a77a6204bc8a58f62a4fac70efdcae4267a64810 Merge: 834493a2 90e65b66 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 17:22:16 2021 +0300 Merge branch 'master' into 3890-fix-stats commit 834493a22ae79199efcc44e0715e2ac6f6272963 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 17:09:30 2021 +0300 querylog: load once commit b8000e7ba7a998fcd4553230ec5e5f9c90106e31 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 16:54:41 2021 +0300 querylog: fix docs commit 7db99ccfa19b58100950c11d67b23bca7af3e5cb Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 16:51:31 2021 +0300 querylog: imp docs commit 2a84650bd7ac5195730a7ab47b9562a83f721499 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 15:48:09 2021 +0300 querylog: imp anonyization commit 0f63feb1ff5f006fc528c3b681ef3b9d2199581e Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:44:37 2021 +0300 all: imp code & docs commit c4ccdcbb7248897edd178fd5cb77127e39ada73d Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:24:30 2021 +0300 all: log changes commit 60bb777a5aff36bba129a078fa11ae566298178a Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:08:41 2021 +0300 all: use atomic value commit c45886bd20eee2212b42686ff369830d8c08fe36 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Nov 30 18:50:02 2021 +0300 all: anonymize separately
2021-12-06 14:26:43 +00:00
data := make([]jobject, 0, len(entries))
Pull request: 3890 fix anonymization Merge in DNS/adguard-home from 3890-fix-stats to master Updates #3890. Squashed commit of the following: commit a77a6204bc8a58f62a4fac70efdcae4267a64810 Merge: 834493a2 90e65b66 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 17:22:16 2021 +0300 Merge branch 'master' into 3890-fix-stats commit 834493a22ae79199efcc44e0715e2ac6f6272963 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 17:09:30 2021 +0300 querylog: load once commit b8000e7ba7a998fcd4553230ec5e5f9c90106e31 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 16:54:41 2021 +0300 querylog: fix docs commit 7db99ccfa19b58100950c11d67b23bca7af3e5cb Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 16:51:31 2021 +0300 querylog: imp docs commit 2a84650bd7ac5195730a7ab47b9562a83f721499 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 15:48:09 2021 +0300 querylog: imp anonyization commit 0f63feb1ff5f006fc528c3b681ef3b9d2199581e Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:44:37 2021 +0300 all: imp code & docs commit c4ccdcbb7248897edd178fd5cb77127e39ada73d Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:24:30 2021 +0300 all: log changes commit 60bb777a5aff36bba129a078fa11ae566298178a Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:08:41 2021 +0300 all: use atomic value commit c45886bd20eee2212b42686ff369830d8c08fe36 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Nov 30 18:50:02 2021 +0300 all: anonymize separately
2021-12-06 14:26:43 +00:00
// The elements order is already reversed to be from newer to older.
for _, entry := range entries {
jsonEntry := l.entryToJSON(ctx, entry, anonFunc)
data = append(data, jsonEntry)
}
res = jobject{
"data": data,
"oldest": "",
}
if !oldest.IsZero() {
res["oldest"] = oldest.Format(time.RFC3339Nano)
}
return res
}
// entryToJSON converts a log entry's data into an entry for the JSON API.
func (l *queryLog) entryToJSON(
ctx context.Context,
entry *logEntry,
anonFunc aghnet.IPMutFunc,
) (jsonEntry jobject) {
hostname := entry.QHost
question := jobject{
"type": entry.QType,
"class": entry.QClass,
"name": hostname,
}
if qhost, err := idna.ToUnicode(hostname); err != nil {
l.logger.DebugContext(
ctx,
"translating into unicode",
"hostname", hostname,
slogutil.KeyError, err,
)
} else if qhost != hostname && qhost != "" {
question["unicode_name"] = qhost
}
entIP := slices.Clone(entry.IP)
anonFunc(entIP)
Pull request: 3890 fix anonymization Merge in DNS/adguard-home from 3890-fix-stats to master Updates #3890. Squashed commit of the following: commit a77a6204bc8a58f62a4fac70efdcae4267a64810 Merge: 834493a2 90e65b66 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 17:22:16 2021 +0300 Merge branch 'master' into 3890-fix-stats commit 834493a22ae79199efcc44e0715e2ac6f6272963 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 17:09:30 2021 +0300 querylog: load once commit b8000e7ba7a998fcd4553230ec5e5f9c90106e31 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 16:54:41 2021 +0300 querylog: fix docs commit 7db99ccfa19b58100950c11d67b23bca7af3e5cb Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 16:51:31 2021 +0300 querylog: imp docs commit 2a84650bd7ac5195730a7ab47b9562a83f721499 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 15:48:09 2021 +0300 querylog: imp anonyization commit 0f63feb1ff5f006fc528c3b681ef3b9d2199581e Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:44:37 2021 +0300 all: imp code & docs commit c4ccdcbb7248897edd178fd5cb77127e39ada73d Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:24:30 2021 +0300 all: log changes commit 60bb777a5aff36bba129a078fa11ae566298178a Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:08:41 2021 +0300 all: use atomic value commit c45886bd20eee2212b42686ff369830d8c08fe36 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Nov 30 18:50:02 2021 +0300 all: anonymize separately
2021-12-06 14:26:43 +00:00
jsonEntry = jobject{
2020-05-29 09:15:22 +01:00
"reason": entry.Result.Reason.String(),
"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
"time": entry.Time.Format(time.RFC3339Nano),
"client": entIP,
2020-05-29 09:15:22 +01:00
"client_proto": entry.ClientProto,
"cached": entry.Cached,
"upstream": entry.Upstream,
"question": question,
"rules": resultRulesToJSONRules(entry.Result.Rules),
}
if entIP.Equal(entry.IP) {
Pull request: 3890 fix anonymization Merge in DNS/adguard-home from 3890-fix-stats to master Updates #3890. Squashed commit of the following: commit a77a6204bc8a58f62a4fac70efdcae4267a64810 Merge: 834493a2 90e65b66 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 17:22:16 2021 +0300 Merge branch 'master' into 3890-fix-stats commit 834493a22ae79199efcc44e0715e2ac6f6272963 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 17:09:30 2021 +0300 querylog: load once commit b8000e7ba7a998fcd4553230ec5e5f9c90106e31 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 16:54:41 2021 +0300 querylog: fix docs commit 7db99ccfa19b58100950c11d67b23bca7af3e5cb Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 16:51:31 2021 +0300 querylog: imp docs commit 2a84650bd7ac5195730a7ab47b9562a83f721499 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 15:48:09 2021 +0300 querylog: imp anonyization commit 0f63feb1ff5f006fc528c3b681ef3b9d2199581e Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:44:37 2021 +0300 all: imp code & docs commit c4ccdcbb7248897edd178fd5cb77127e39ada73d Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:24:30 2021 +0300 all: log changes commit 60bb777a5aff36bba129a078fa11ae566298178a Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Dec 6 14:08:41 2021 +0300 all: use atomic value commit c45886bd20eee2212b42686ff369830d8c08fe36 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Nov 30 18:50:02 2021 +0300 all: anonymize separately
2021-12-06 14:26:43 +00:00
jsonEntry["client_info"] = entry.client
}
Pull request: all: client id support Merge in DNS/adguard-home from 1383-client-id to master Updates #1383. Squashed commit of the following: commit ebe2678bfa9bf651a2cb1e64499b38edcf19a7ad Author: Ildar Kamalov <ik@adguard.com> Date: Wed Jan 27 17:51:59 2021 +0300 - client: check if IP is valid commit 0c330585a170ea149ee75e43dfa65211e057299c Author: Ildar Kamalov <ik@adguard.com> Date: Wed Jan 27 17:07:50 2021 +0300 - client: find clients by client_id commit 71c9593ee35d996846f061e114b7867c3aa3c978 Merge: 9104f161 3e9edd9e Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Wed Jan 27 16:09:45 2021 +0300 Merge branch 'master' into 1383-client-id commit 9104f1615d2d462606c52017df25a422df872cea Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Wed Jan 27 13:28:50 2021 +0300 dnsforward: imp tests commit ed47f26e611ade625a2cc2c2f71a291b796bbf8f Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Wed Jan 27 12:39:52 2021 +0300 dnsforward: fix address commit 98b222ba69a5d265f620c180c960d01c84a1fb3b Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Tue Jan 26 19:50:31 2021 +0300 home: imp code commit 4f3966548a2d8437d0b68207dd108dd1a6cb7d20 Merge: 199fdc05 c215b820 Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Tue Jan 26 19:45:13 2021 +0300 Merge branch 'master' into 1383-client-id commit 199fdc056f8a8be5500584f3aaee32865188aedc Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Tue Jan 26 19:20:37 2021 +0300 all: imp tests, logging, etc commit 35ff14f4d534251aecb2ea60baba225f3eed8a3e Author: Ildar Kamalov <ik@adguard.com> Date: Tue Jan 26 18:55:19 2021 +0300 + client: remove block button from clients with client_id commit 32991a0b4c56583a02fb5e00bba95d96000bce20 Author: Ildar Kamalov <ik@adguard.com> Date: Tue Jan 26 18:54:25 2021 +0300 + client: add requests count for client_id commit 2d68df4d2eac4a296d7469923e601dad4575c1a1 Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Tue Jan 26 15:49:50 2021 +0300 stats: handle client ids commit 4e14ab3590328f93a8cd6e9cbe1665baf74f220b Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Tue Jan 26 13:45:25 2021 +0300 openapi: fix example commit ca9cf3f744fe197cace2c28ddc5bc68f71dad1f3 Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Tue Jan 26 13:37:10 2021 +0300 openapi: improve clients find api docs commit f79876e550c424558b704bc316a4cd04f25db011 Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Tue Jan 26 13:18:52 2021 +0300 home: accept ids in clients find commit 5b72595122aa0bd64debadfd753ed8a0e0840629 Merge: 607e241f abf8f65f Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Mon Jan 25 18:34:56 2021 +0300 Merge branch 'master' into 1383-client-id commit 607e241f1c339dd6397218f70b8301e3de6a1ee0 Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Mon Jan 25 18:30:39 2021 +0300 dnsforward: fix quic commit f046352fef93e46234c2bbe8ae316d21034260e5 Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Mon Jan 25 16:53:09 2021 +0300 all: remove wildcard requirement commit 3b679489bae82c54177372be453fe184d8f0bab6 Author: Andrey Meshkov <am@adguard.com> Date: Mon Jan 25 16:02:28 2021 +0300 workDir now supports symlinks commit 0647ab4f113de2223f6949df001f42ecab05c995 Author: Ildar Kamalov <ik@adguard.com> Date: Mon Jan 25 14:59:46 2021 +0300 - client: remove wildcard from domain validation commit b1aec04a4ecadc9d65648ed6d284188fecce01c3 Author: Ildar Kamalov <ik@adguard.com> Date: Mon Jan 25 14:55:39 2021 +0300 + client: add form to download mobileconfig ... and 12 more commits
2021-01-27 15:32:13 +00:00
if entry.ClientID != "" {
jsonEntry["client_id"] = entry.ClientID
}
Pull request: 3978 Query Log ECS Merge in DNS/adguard-home from 3978-ecs-ip to master Updates #3978. Squashed commit of the following: commit 915b94afa4b6d90169f73d4fa171bc81bcc267a7 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Mar 3 17:46:40 2022 +0300 all: rm dot commit 2dd2ed081b199de7e5d8269dae5d08d53b5eea6d Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Mar 3 17:42:45 2022 +0300 client: imp txt commit 8d5a23df739f0b650f9f3870141fd83e8fa0c1e0 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Mar 3 14:36:04 2022 +0300 client: imp text commit 69c856749a20144822ef3f1f67c5f3e3c24f5374 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Mar 3 14:24:56 2022 +0300 client: imp description commit cd0150128ad29d1874492735a5d621c0803ad0bd Merge: 28181fbc e0b557ed Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Wed Mar 2 21:02:16 2022 +0300 Merge branch 'master' into 3978-ecs-ip commit 28181fbc79eb22e7fd13cbd1d5a3c040af9fa2a4 Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Wed Mar 2 20:45:50 2022 +0300 client: show ecs commit cdc5e7f8c4155b798426d815eed0da547ef6efb7 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Feb 17 20:15:56 2022 +0300 openapi: fix milestone commit 404d6d822fa1ba4ed4cd41d92d4c1b805342fe55 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Feb 17 20:08:21 2022 +0300 all: fix deps, docs commit 8fb80526f1e251d3b7b193c53a4a6dee0e22c145 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Thu Feb 17 19:39:34 2022 +0300 all: add querylog ecs backend
2022-03-03 14:52:11 +00:00
if entry.ReqECS != "" {
jsonEntry["ecs"] = entry.ReqECS
}
if len(entry.Result.Rules) > 0 {
if r := entry.Result.Rules[0]; len(r.Text) > 0 {
jsonEntry["rule"] = r.Text
jsonEntry["filterId"] = r.FilterListID
}
}
if len(entry.Result.ServiceName) != 0 {
jsonEntry["service_name"] = entry.Result.ServiceName
}
l.setMsgData(ctx, entry, jsonEntry)
l.setOrigAns(ctx, entry, jsonEntry)
return jsonEntry
}
// setMsgData sets the message data in jsonEntry.
func (l *queryLog) setMsgData(ctx context.Context, entry *logEntry, jsonEntry jobject) {
if len(entry.Answer) == 0 {
return
}
msg := &dns.Msg{}
if err := msg.Unpack(entry.Answer); err != nil {
l.logger.DebugContext(
ctx,
"unpacking dns message",
"answer", entry.Answer,
slogutil.KeyError, err,
)
return
}
jsonEntry["status"] = dns.RcodeToString[msg.Rcode]
// Old query logs may still keep AD flag value in the message. Try to get
// it from there as well.
jsonEntry["answer_dnssec"] = entry.AuthenticatedData || msg.AuthenticatedData
if a := answerToJSON(msg); a != nil {
jsonEntry["answer"] = a
}
}
// setOrigAns sets the original answer data in jsonEntry.
func (l *queryLog) setOrigAns(ctx context.Context, entry *logEntry, jsonEntry jobject) {
if len(entry.OrigAnswer) == 0 {
return
}
orig := &dns.Msg{}
err := orig.Unpack(entry.OrigAnswer)
if err != nil {
l.logger.DebugContext(
ctx,
"setting original answer",
"answer", entry.OrigAnswer,
slogutil.KeyError, err,
)
return
}
if a := answerToJSON(orig); a != nil {
jsonEntry["original_answer"] = a
}
}
func resultRulesToJSONRules(rules []*filtering.ResultRule) (jsonRules []jobject) {
jsonRules = make([]jobject, len(rules))
for i, r := range rules {
jsonRules[i] = jobject{
"filter_list_id": r.FilterListID,
"text": r.Text,
}
}
return jsonRules
}
type dnsAnswer struct {
Type string `json:"type"`
Value string `json:"value"`
TTL uint32 `json:"ttl"`
}
// answerToJSON converts the answer records of msg, if any, to their JSON form.
func answerToJSON(msg *dns.Msg) (answers []*dnsAnswer) {
if msg == nil || len(msg.Answer) == 0 {
return nil
}
answers = make([]*dnsAnswer, 0, len(msg.Answer))
for _, rr := range msg.Answer {
header := rr.Header()
a := &dnsAnswer{
Type: dns.TypeToString[header.Rrtype],
// Remove the header string from the answer value since it's mostly
// unnecessary in the log.
Value: strings.TrimPrefix(rr.String(), header.String()),
TTL: header.Ttl,
}
answers = append(answers, a)
}
return answers
}