Pull request 1751: imp-querylog

Merge in DNS/adguard-home from imp-querylog to master

Squashed commit of the following:

commit 40b88f9dac46576399cd4d1fad52ecffd8f5945d
Merge: fcfe40b3 bb80a7c2
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Feb 27 17:14:34 2023 +0300

    Merge branch 'master' into imp-querylog

commit fcfe40b33143f82fe4ef03fd883c3159dfb06026
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Feb 27 17:13:45 2023 +0300

    querylog: imp docs, names

commit 21722c6d853465c97e6f18693095a0bc86308ea3
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Feb 22 20:28:49 2023 +0300

    querylog: fix race; refactor

internal/querylog/decode.go

@@ -247,55 +247,20 @@ var resultHandlers = map[string]logEntryHandler{
 }
 
 func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) {
+	var vToken json.Token
 	switch key {
 	case "FilterListID":
-		vToken, err := dec.Token()
+		ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
-		if err != nil {
-			if err != io.EOF {
-				log.Debug("decodeResultRuleKey %s err: %s", key, err)
-			}
-
-			return
-		}
-
-		if len(ent.Result.Rules) < i+1 {
-			ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
-		}
-
 		if n, ok := vToken.(json.Number); ok {
 			ent.Result.Rules[i].FilterListID, _ = n.Int64()
 		}
 	case "IP":
-		vToken, err := dec.Token()
+		ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
-		if err != nil {
-			if err != io.EOF {
-				log.Debug("decodeResultRuleKey %s err: %s", key, err)
-			}
-
-			return
-		}
-
-		if len(ent.Result.Rules) < i+1 {
-			ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
-		}
-
 		if ipStr, ok := vToken.(string); ok {
 			ent.Result.Rules[i].IP = net.ParseIP(ipStr)
 		}
 	case "Text":
-		vToken, err := dec.Token()
+		ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
-		if err != nil {
-			if err != io.EOF {
-				log.Debug("decodeResultRuleKey %s err: %s", key, err)
-			}
-
-			return
-		}
-
-		if len(ent.Result.Rules) < i+1 {
-			ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
-		}
-
 		if s, ok := vToken.(string); ok {
 			ent.Result.Rules[i].Text = s
 		}
@@ -304,6 +269,30 @@ func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) {
 	}
 }
 
+func decodeVTokenAndAddRule(
+	key string,
+	i int,
+	dec *json.Decoder,
+	rules []*filtering.ResultRule,
+) (newRules []*filtering.ResultRule, vToken json.Token) {
+	newRules = rules
+
+	vToken, err := dec.Token()
+	if err != nil {
+		if err != io.EOF {
+			log.Debug("decodeResultRuleKey %s err: %s", key, err)
+		}
+
+		return newRules, nil
+	}
+
+	if len(rules) < i+1 {
+		newRules = append(newRules, &filtering.ResultRule{})
+	}
+
+	return newRules, vToken
+}
+
 func decodeResultRules(dec *json.Decoder, ent *logEntry) {
 	for {
 		delimToken, err := dec.Token()

internal/querylog/http.go

@@ -54,10 +54,7 @@ func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// search for the log entries
 	entries, oldest := l.search(params)
-
-	// convert log entries to JSON
 	data := l.entriesToJSON(entries, oldest)
 
 	_ = aghhttp.WriteJSONResponse(w, r, data)

internal/querylog/json.go

@@ -1,7 +1,6 @@
 package querylog
 
 import (
-	"fmt"
 	"strconv"
 	"strings"
 	"time"
@@ -117,7 +116,7 @@ func (l *queryLog) setMsgData(entry *logEntry, jsonEntry jobject) {
 	// it from there as well.
 	jsonEntry["answer_dnssec"] = entry.AuthenticatedData || msg.AuthenticatedData
 
-	if a := answerToMap(msg); a != nil {
+	if a := answerToJSON(msg); a != nil {
 		jsonEntry["answer"] = a
 	}
 }
@@ -136,7 +135,7 @@ func (l *queryLog) setOrigAns(entry *logEntry, jsonEntry jobject) {
 		return
 	}
 
-	if a := answerToMap(orig); a != nil {
+	if a := answerToJSON(orig); a != nil {
 		jsonEntry["original_answer"] = a
 	}
 }
@@ -159,55 +158,24 @@ type dnsAnswer struct {
 	TTL   uint32 `json:"ttl"`
 }
 
-func answerToMap(a *dns.Msg) (answers []*dnsAnswer) {
+// answerToJSON converts the answer records of msg, if any, to their JSON form.
-	if a == nil || len(a.Answer) == 0 {
+func answerToJSON(msg *dns.Msg) (answers []*dnsAnswer) {
+	if msg == nil || len(msg.Answer) == 0 {
 		return nil
 	}
 
-	answers = make([]*dnsAnswer, 0, len(a.Answer))
+	answers = make([]*dnsAnswer, 0, len(msg.Answer))
-	for _, k := range a.Answer {
+	for _, rr := range msg.Answer {
-		header := k.Header()
+		header := rr.Header()
-		answer := &dnsAnswer{
+		a := &dnsAnswer{
 			Type: dns.TypeToString[header.Rrtype],
-			TTL:  header.Ttl,
+			// Remove the header string from the answer value since it's mostly
+			// unnecessary in the log.
+			Value: strings.TrimPrefix(rr.String(), header.String()),
+			TTL:   header.Ttl,
 		}
 
-		// Some special treatment for some well-known types.
+		answers = append(answers, a)
-		//
-		// TODO(a.garipov): Consider just calling String() for everyone
-		// instead.
-		switch v := k.(type) {
-		case nil:
-			// Probably unlikely, but go on.
-		case *dns.A:
-			answer.Value = v.A.String()
-		case *dns.AAAA:
-			answer.Value = v.AAAA.String()
-		case *dns.MX:
-			answer.Value = fmt.Sprintf("%v %v", v.Preference, v.Mx)
-		case *dns.CNAME:
-			answer.Value = v.Target
-		case *dns.NS:
-			answer.Value = v.Ns
-		case *dns.SPF:
-			answer.Value = strings.Join(v.Txt, "\n")
-		case *dns.TXT:
-			answer.Value = strings.Join(v.Txt, "\n")
-		case *dns.PTR:
-			answer.Value = v.Ptr
-		case *dns.SOA:
-			answer.Value = fmt.Sprintf("%v %v %v %v %v %v %v", v.Ns, v.Mbox, v.Serial, v.Refresh, v.Retry, v.Expire, v.Minttl)
-		case *dns.CAA:
-			answer.Value = fmt.Sprintf("%v %v \"%v\"", v.Flag, v.Tag, v.Value)
-		case *dns.HINFO:
-			answer.Value = fmt.Sprintf("\"%v\" \"%v\"", v.Cpu, v.Os)
-		case *dns.RRSIG:
-			answer.Value = fmt.Sprintf("%v %v %v %v %v %v %v %v %v", dns.TypeToString[v.TypeCovered], v.Algorithm, v.Labels, v.OrigTtl, v.Expiration, v.Inception, v.KeyTag, v.SignerName, v.Signature)
-		default:
-			answer.Value = v.String()
-		}
-
-		answers = append(answers, answer)
 	}
 
 	return answers

internal/querylog/qlog.go

@@ -31,7 +31,8 @@ type queryLog struct {
 
 	// bufferLock protects buffer.
 	bufferLock sync.RWMutex
-	// buffer contains recent log entries.
+	// buffer contains recent log entries.  The entries in this buffer must not
+	// be modified.
 	buffer []*logEntry
 
 	fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread
@@ -100,6 +101,13 @@ type logEntry struct {
 	AuthenticatedData bool `json:"AD,omitempty"`
 }
 
+// shallowClone returns a shallow clone of e.
+func (e *logEntry) shallowClone() (clone *logEntry) {
+	cloneVal := *e
+
+	return &cloneVal
+}
+
 func (l *queryLog) Start() {
 	if l.conf.HTTPRegister != nil {
 		l.initWeb()

internal/querylog/search.go

@@ -52,7 +52,9 @@ func (l *queryLog) searchMemory(params *searchParams, cache clientCache) (entrie
 	// Go through the buffer in the reverse order, from newer to older.
 	var err error
 	for i := len(l.buffer) - 1; i >= 0; i-- {
-		e := l.buffer[i]
+		// A shallow clone is enough, since the only thing that this loop
+		// modifies is the client field.
+		e := l.buffer[i].shallowClone()
 
 		e.client, err = l.client(e.ClientID, e.IP.String(), cache)
 		if err != nil {
@@ -130,7 +132,7 @@ func (l *queryLog) search(params *searchParams) (entries []*logEntry, oldest tim
 // searchFiles looks up log records from all log files.  It optionally uses the
 // client cache, if provided.  searchFiles does not scan more than
 // maxFileScanEntries so callers may need to call it several times to get all
-// results.  oldset and total are the time of the oldest processed entry and the
+// results.  oldest and total are the time of the oldest processed entry and the
 // total number of processed entries, including discarded ones, correspondingly.
 func (l *queryLog) searchFiles(
 	params *searchParams,

internal/querylog/searchcriterion.go

@@ -1,6 +1,7 @@
 package querylog
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering"
@@ -118,7 +119,7 @@ func (c *searchCriterion) match(entry *logEntry) bool {
 	case ctTerm:
 		return c.ctDomainOrClientCase(entry)
 	case ctFilteringStatus:
-		return c.ctFilteringStatusCase(entry.Result)
+		return c.ctFilteringStatusCase(entry.Result.Reason, entry.Result.IsFiltered)
 	}
 
 	return false
@@ -141,54 +142,70 @@ func (c *searchCriterion) ctDomainOrClientCase(e *logEntry) bool {
 	return ctDomainOrClientCaseNonStrict(c.value, c.asciiVal, clientID, name, host, ip)
 }
 
-func (c *searchCriterion) ctFilteringStatusCase(res filtering.Result) bool {
+// ctFilteringStatusCase returns true if the result matches the value.
+func (c *searchCriterion) ctFilteringStatusCase(
+	reason filtering.Reason,
+	isFiltered bool,
+) (matched bool) {
 	switch c.value {
 	case filteringStatusAll:
 		return true
-
+	case
-	case filteringStatusFiltered:
+		filteringStatusBlocked,
-		return res.IsFiltered ||
+		filteringStatusBlockedParental,
-			res.Reason.In(
+		filteringStatusBlockedSafebrowsing,
-				filtering.NotFilteredAllowList,
+		filteringStatusBlockedService,
-				filtering.Rewritten,
+		filteringStatusFiltered,
-				filtering.RewrittenAutoHosts,
+		filteringStatusSafeSearch:
-				filtering.RewrittenRule,
+		return isFiltered && c.isFilteredWithReason(reason)
-			)
-
-	case filteringStatusBlocked:
-		return res.IsFiltered &&
-			res.Reason.In(filtering.FilteredBlockList, filtering.FilteredBlockedService)
-
-	case filteringStatusBlockedService:
-		return res.IsFiltered && res.Reason == filtering.FilteredBlockedService
-
-	case filteringStatusBlockedParental:
-		return res.IsFiltered && res.Reason == filtering.FilteredParental
-
-	case filteringStatusBlockedSafebrowsing:
-		return res.IsFiltered && res.Reason == filtering.FilteredSafeBrowsing
-
 	case filteringStatusWhitelisted:
-		return res.Reason == filtering.NotFilteredAllowList
+		return reason == filtering.NotFilteredAllowList
-
 	case filteringStatusRewritten:
-		return res.Reason.In(
+		return reason.In(
 			filtering.Rewritten,
 			filtering.RewrittenAutoHosts,
 			filtering.RewrittenRule,
 		)
-
-	case filteringStatusSafeSearch:
-		return res.IsFiltered && res.Reason == filtering.FilteredSafeSearch
-
 	case filteringStatusProcessed:
-		return !res.Reason.In(
+		return !reason.In(
 			filtering.FilteredBlockList,
 			filtering.FilteredBlockedService,
 			filtering.NotFilteredAllowList,
 		)
-
 	default:
 		return false
 	}
 }
+
+// isFilteredWithReason returns true if reason matches the criterion value.
+// c.value must be one of:
+//
+//   - filteringStatusBlocked
+//   - filteringStatusBlockedParental
+//   - filteringStatusBlockedSafebrowsing
+//   - filteringStatusBlockedService
+//   - filteringStatusFiltered
+//   - filteringStatusSafeSearch
+func (c *searchCriterion) isFilteredWithReason(reason filtering.Reason) (matched bool) {
+	switch c.value {
+	case filteringStatusBlocked:
+		return reason.In(filtering.FilteredBlockList, filtering.FilteredBlockedService)
+	case filteringStatusBlockedParental:
+		return reason == filtering.FilteredParental
+	case filteringStatusBlockedSafebrowsing:
+		return reason == filtering.FilteredSafeBrowsing
+	case filteringStatusBlockedService:
+		return reason == filtering.FilteredBlockedService
+	case filteringStatusFiltered:
+		return reason.In(
+			filtering.NotFilteredAllowList,
+			filtering.Rewritten,
+			filtering.RewrittenAutoHosts,
+			filtering.RewrittenRule,
+		)
+	case filteringStatusSafeSearch:
+		return reason == filtering.FilteredSafeSearch
+	default:
+		panic(fmt.Errorf("unexpected value %q", c.value))
+	}
+}

scripts/make/go-lint.sh

@@ -161,7 +161,7 @@ run_linter "$GO" vet ./...
 run_linter govulncheck ./...
 
 # Apply more lax standards to the code we haven't properly refactored yet.
-run_linter gocyclo --over 17 ./internal/querylog/
+run_linter gocyclo --over 14 ./internal/querylog/
 run_linter gocyclo --over 13\
 	./internal/dhcpd\
 	./internal/filtering/\