+ new query logs API

+ "upstream"
+ filteringStatusProcessed
+ ctDomainOrClient
remove ctQuestionType, ctDomain, ctClient

AGHTechDoc.md

@@ -1200,8 +1200,8 @@ When a new DNS request is received and processed, we store information about thi
 	"QH":"...", // target host name without the last dot
 	"QT":"...", // question type
 	"QC":"...", // question class
-	"Answer":"...",
+	"Answer":"base64 data",
-	"OrigAnswer":"...",
+	"OrigAnswer":"base64 data",
 	"Result":{
 		"IsFiltered":true,
 		"Reason":3,
@@ -1234,16 +1234,22 @@ Request:
 
 	GET /control/querylog
 	?older_than=2006-01-02T15:04:05.999999999Z07:00
-	&filter_domain=...
+	&search=...
-	&filter_client=...
+	&response_status=""|blocked|whitelisted|processed
-	&filter_question_type=A | AAAA
-	&filter_response_status= | filtered
 
 `older_than` setting is used for paging.  UI uses an empty value for `older_than` on the first request and gets the latest log entries.  To get the older entries, UI sets `older_than` to the `oldest` value from the server's response.
 
-If "filter" settings are set, server returns only entries that match the specified request.
+If search settings are set, server returns only entries that match the specified request.
 
-For `filter.domain` and `filter.client` the server matches substrings by default: `adguard.com` matches `www.adguard.com`.  Strict matching can be enabled by enclosing the value in double quotes: `"adguard.com"` matches `adguard.com` but doesn't match `www.adguard.com`.
+`search`:
+match by domain name or client IP address.
+The server matches substrings by default: e.g. `adguard.com` matches `www.adguard.com`.
+Strict matching can be enabled by enclosing the value in double quotes: e.g. `"adguard.com"` matches `adguard.com` but doesn't match `www.adguard.com`.
+
+`response_status`:
+* blocked: only blocked entries
+* whitelisted: only white-listed entries
+* processed: all not blocked, not white-listed entries
 
 Response:
 
@@ -1266,6 +1272,7 @@ Response:
 			}
 			...
 		],
+		"upstream":"...", // Upstream URL starting with tcp://, tls://, https://, or with an IP address
 		"answer_dnssec": true,
 		"client":"127.0.0.1",
 		"elapsedMs":"0.098403",

querylog/json.go

@@ -112,6 +112,8 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
 		}
 	}
 
+	jsonEntry["upstream"] = entry.Upstream
+
 	return jsonEntry
 }
 

querylog/qlog_http.go

@@ -142,10 +142,6 @@ func (l *queryLog) parseSearchCriteria(q url.Values, name string, ct criteriaTyp
 		c.strict = true
 	}
 
-	if ct == ctClient && l.conf.AnonymizeClientIP {
-		c.value = l.getClientIP(c.value)
-	}
-
 	if ct == ctFilteringStatus && !util.ContainsString(filteringStatusValues, c.value) {
 		return false, c, fmt.Errorf("invalid value %s", c.value)
 	}
@@ -180,10 +176,8 @@ func (l *queryLog) parseSearchParams(r *http.Request) (*searchParams, error) {
 	}
 
 	paramNames := map[string]criteriaType{
-		"filter_domain":          ctDomain,
+		"search":          ctDomainOrClient,
-		"filter_client":          ctClient,
+		"response_status": ctFilteringStatus,
-		"filter_question_type":   ctQuestionType,
-		"filter_response_status": ctFilteringStatus,
 	}
 
 	for k, v := range paramNames {

querylog/qlog_test.go

@@ -57,7 +57,7 @@ func TestQueryLog(t *testing.T) {
 	// search by domain (strict)
 	params = newSearchParams()
 	params.searchCriteria = append(params.searchCriteria, searchCriteria{
-		criteriaType: ctDomain,
+		criteriaType: ctDomainOrClient,
 		strict:       true,
 		value:        "test.example.org",
 	})
@@ -68,7 +68,7 @@ func TestQueryLog(t *testing.T) {
 	// search by domain (not strict)
 	params = newSearchParams()
 	params.searchCriteria = append(params.searchCriteria, searchCriteria{
-		criteriaType: ctDomain,
+		criteriaType: ctDomainOrClient,
 		strict:       false,
 		value:        "example.org",
 	})
@@ -81,7 +81,7 @@ func TestQueryLog(t *testing.T) {
 	// search by client IP (strict)
 	params = newSearchParams()
 	params.searchCriteria = append(params.searchCriteria, searchCriteria{
-		criteriaType: ctClient,
+		criteriaType: ctDomainOrClient,
 		strict:       true,
 		value:        "2.2.2.2",
 	})
@@ -92,7 +92,7 @@ func TestQueryLog(t *testing.T) {
 	// search by client IP (part of)
 	params = newSearchParams()
 	params.searchCriteria = append(params.searchCriteria, searchCriteria{
-		criteriaType: ctClient,
+		criteriaType: ctDomainOrClient,
 		strict:       false,
 		value:        "2.2.2",
 	})

querylog/search_criteria.go

@@ -9,9 +9,7 @@ import (
 type criteriaType int
 
 const (
-	ctDomain          criteriaType = iota // domain name
+	ctDomainOrClient  criteriaType = iota // domain name or client IP address
-	ctClient                              // client IP address
-	ctQuestionType                        // question type
 	ctFilteringStatus                     // filtering status
 )
 
@@ -25,6 +23,7 @@ const (
 	filteringStatusWhitelisted         = "whitelisted"          // whitelisted
 	filteringStatusRewritten           = "rewritten"            // all kinds of rewrites
 	filteringStatusSafeSearch          = "safe_search"          // enforced safe search
+	filteringStatusProcessed           = "processed"            // not blocked, not white-listed entries
 )
 
 // filteringStatusValues -- array with all possible filteringStatus values
@@ -32,6 +31,7 @@ var filteringStatusValues = []string{
 	filteringStatusAll, filteringStatusFiltered, filteringStatusBlocked,
 	filteringStatusBlockedSafebrowsing, filteringStatusBlockedParental,
 	filteringStatusWhitelisted, filteringStatusRewritten, filteringStatusSafeSearch,
+	filteringStatusProcessed,
 }
 
 // searchCriteria - every search request may contain a list of different search criteria
@@ -48,12 +48,9 @@ func (c *searchCriteria) quickMatch(line string) bool {
 	// note that we do this only for a limited set of criteria
 
 	switch c.criteriaType {
-	case ctDomain:
+	case ctDomainOrClient:
-		return c.quickMatchJSONValue(line, "QH")
+		return c.quickMatchJSONValue(line, "QH") ||
-	case ctClient:
+			c.quickMatchJSONValue(line, "IP")
-		return c.quickMatchJSONValue(line, "IP")
-	case ctQuestionType:
-		return c.quickMatchJSONValue(line, "QT")
 	default:
 		return true
 	}
@@ -80,29 +77,23 @@ func (c *searchCriteria) quickMatchJSONValue(line string, propertyName string) b
 // nolint (gocyclo)
 func (c *searchCriteria) match(entry *logEntry) bool {
 	switch c.criteriaType {
-	case ctDomain:
+	case ctDomainOrClient:
 		if c.strict && entry.QHost == c.value {
 			return true
 		}
 		if !c.strict && strings.Contains(entry.QHost, c.value) {
 			return true
 		}
-		return false
+
-	case ctClient:
 		if c.strict && entry.IP == c.value {
 			return true
 		}
 		if !c.strict && strings.Contains(entry.IP, c.value) {
 			return true
 		}
+
 		return false
-	case ctQuestionType:
+
-		if c.strict && entry.QType == c.value {
-			return true
-		}
-		if !c.strict && strings.Contains(entry.QType, c.value) {
-			return true
-		}
 	case ctFilteringStatus:
 		res := entry.Result
 
@@ -127,13 +118,16 @@ func (c *searchCriteria) match(entry *logEntry) bool {
 					res.Reason == dnsfilter.RewriteEtcHosts)
 		case filteringStatusSafeSearch:
 			return res.IsFiltered && res.Reason == dnsfilter.FilteredSafeSearch
-		default:
+
-			return false
+		case filteringStatusProcessed:
-		}
+			return !(res.Reason == dnsfilter.FilteredBlackList ||
+				res.Reason == dnsfilter.FilteredBlockedService ||
+				res.Reason == dnsfilter.NotFilteredWhiteList)
 
 		default:
 			return false
 		}
+	}
 
 	return false
 }