From 0500aa9591d1fb52f24182734859c398203d6962 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Thu, 28 May 2020 17:14:50 +0300 Subject: [PATCH] + new query logs API + "upstream" + filteringStatusProcessed + ctDomainOrClient remove ctQuestionType, ctDomain, ctClient --- AGHTechDoc.md | 23 ++++++++++++++-------- querylog/json.go | 2 ++ querylog/qlog_http.go | 10 ++-------- querylog/qlog_test.go | 8 ++++---- querylog/search_criteria.go | 38 ++++++++++++++++--------------------- 5 files changed, 39 insertions(+), 42 deletions(-) diff --git a/AGHTechDoc.md b/AGHTechDoc.md index 5a3b001c..ee9e7ee8 100644 --- a/AGHTechDoc.md +++ b/AGHTechDoc.md @@ -1200,8 +1200,8 @@ When a new DNS request is received and processed, we store information about thi "QH":"...", // target host name without the last dot "QT":"...", // question type "QC":"...", // question class - "Answer":"...", - "OrigAnswer":"...", + "Answer":"base64 data", + "OrigAnswer":"base64 data", "Result":{ "IsFiltered":true, "Reason":3, @@ -1234,16 +1234,22 @@ Request: GET /control/querylog ?older_than=2006-01-02T15:04:05.999999999Z07:00 - &filter_domain=... - &filter_client=... - &filter_question_type=A | AAAA - &filter_response_status= | filtered + &search=... + &response_status=""|blocked|whitelisted|processed `older_than` setting is used for paging. UI uses an empty value for `older_than` on the first request and gets the latest log entries. To get the older entries, UI sets `older_than` to the `oldest` value from the server's response. -If "filter" settings are set, server returns only entries that match the specified request. +If search settings are set, server returns only entries that match the specified request. -For `filter.domain` and `filter.client` the server matches substrings by default: `adguard.com` matches `www.adguard.com`. Strict matching can be enabled by enclosing the value in double quotes: `"adguard.com"` matches `adguard.com` but doesn't match `www.adguard.com`. +`search`: +match by domain name or client IP address. +The server matches substrings by default: e.g. `adguard.com` matches `www.adguard.com`. +Strict matching can be enabled by enclosing the value in double quotes: e.g. `"adguard.com"` matches `adguard.com` but doesn't match `www.adguard.com`. + +`response_status`: +* blocked: only blocked entries +* whitelisted: only white-listed entries +* processed: all not blocked, not white-listed entries Response: @@ -1266,6 +1272,7 @@ Response: } ... ], + "upstream":"...", // Upstream URL starting with tcp://, tls://, https://, or with an IP address "answer_dnssec": true, "client":"127.0.0.1", "elapsedMs":"0.098403", diff --git a/querylog/json.go b/querylog/json.go index 86f35e11..cc7e6c39 100644 --- a/querylog/json.go +++ b/querylog/json.go @@ -112,6 +112,8 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} { } } + jsonEntry["upstream"] = entry.Upstream + return jsonEntry } diff --git a/querylog/qlog_http.go b/querylog/qlog_http.go index 19caa35c..cf804c47 100644 --- a/querylog/qlog_http.go +++ b/querylog/qlog_http.go @@ -142,10 +142,6 @@ func (l *queryLog) parseSearchCriteria(q url.Values, name string, ct criteriaTyp c.strict = true } - if ct == ctClient && l.conf.AnonymizeClientIP { - c.value = l.getClientIP(c.value) - } - if ct == ctFilteringStatus && !util.ContainsString(filteringStatusValues, c.value) { return false, c, fmt.Errorf("invalid value %s", c.value) } @@ -180,10 +176,8 @@ func (l *queryLog) parseSearchParams(r *http.Request) (*searchParams, error) { } paramNames := map[string]criteriaType{ - "filter_domain": ctDomain, - "filter_client": ctClient, - "filter_question_type": ctQuestionType, - "filter_response_status": ctFilteringStatus, + "search": ctDomainOrClient, + "response_status": ctFilteringStatus, } for k, v := range paramNames { diff --git a/querylog/qlog_test.go b/querylog/qlog_test.go index 921d2870..c087fa4d 100644 --- a/querylog/qlog_test.go +++ b/querylog/qlog_test.go @@ -57,7 +57,7 @@ func TestQueryLog(t *testing.T) { // search by domain (strict) params = newSearchParams() params.searchCriteria = append(params.searchCriteria, searchCriteria{ - criteriaType: ctDomain, + criteriaType: ctDomainOrClient, strict: true, value: "test.example.org", }) @@ -68,7 +68,7 @@ func TestQueryLog(t *testing.T) { // search by domain (not strict) params = newSearchParams() params.searchCriteria = append(params.searchCriteria, searchCriteria{ - criteriaType: ctDomain, + criteriaType: ctDomainOrClient, strict: false, value: "example.org", }) @@ -81,7 +81,7 @@ func TestQueryLog(t *testing.T) { // search by client IP (strict) params = newSearchParams() params.searchCriteria = append(params.searchCriteria, searchCriteria{ - criteriaType: ctClient, + criteriaType: ctDomainOrClient, strict: true, value: "2.2.2.2", }) @@ -92,7 +92,7 @@ func TestQueryLog(t *testing.T) { // search by client IP (part of) params = newSearchParams() params.searchCriteria = append(params.searchCriteria, searchCriteria{ - criteriaType: ctClient, + criteriaType: ctDomainOrClient, strict: false, value: "2.2.2", }) diff --git a/querylog/search_criteria.go b/querylog/search_criteria.go index b2ba63f6..bc2aa435 100644 --- a/querylog/search_criteria.go +++ b/querylog/search_criteria.go @@ -9,9 +9,7 @@ import ( type criteriaType int const ( - ctDomain criteriaType = iota // domain name - ctClient // client IP address - ctQuestionType // question type + ctDomainOrClient criteriaType = iota // domain name or client IP address ctFilteringStatus // filtering status ) @@ -25,6 +23,7 @@ const ( filteringStatusWhitelisted = "whitelisted" // whitelisted filteringStatusRewritten = "rewritten" // all kinds of rewrites filteringStatusSafeSearch = "safe_search" // enforced safe search + filteringStatusProcessed = "processed" // not blocked, not white-listed entries ) // filteringStatusValues -- array with all possible filteringStatus values @@ -32,6 +31,7 @@ var filteringStatusValues = []string{ filteringStatusAll, filteringStatusFiltered, filteringStatusBlocked, filteringStatusBlockedSafebrowsing, filteringStatusBlockedParental, filteringStatusWhitelisted, filteringStatusRewritten, filteringStatusSafeSearch, + filteringStatusProcessed, } // searchCriteria - every search request may contain a list of different search criteria @@ -48,12 +48,9 @@ func (c *searchCriteria) quickMatch(line string) bool { // note that we do this only for a limited set of criteria switch c.criteriaType { - case ctDomain: - return c.quickMatchJSONValue(line, "QH") - case ctClient: - return c.quickMatchJSONValue(line, "IP") - case ctQuestionType: - return c.quickMatchJSONValue(line, "QT") + case ctDomainOrClient: + return c.quickMatchJSONValue(line, "QH") || + c.quickMatchJSONValue(line, "IP") default: return true } @@ -80,29 +77,23 @@ func (c *searchCriteria) quickMatchJSONValue(line string, propertyName string) b // nolint (gocyclo) func (c *searchCriteria) match(entry *logEntry) bool { switch c.criteriaType { - case ctDomain: + case ctDomainOrClient: if c.strict && entry.QHost == c.value { return true } if !c.strict && strings.Contains(entry.QHost, c.value) { return true } - return false - case ctClient: + if c.strict && entry.IP == c.value { return true } if !c.strict && strings.Contains(entry.IP, c.value) { return true } + return false - case ctQuestionType: - if c.strict && entry.QType == c.value { - return true - } - if !c.strict && strings.Contains(entry.QType, c.value) { - return true - } + case ctFilteringStatus: res := entry.Result @@ -127,12 +118,15 @@ func (c *searchCriteria) match(entry *logEntry) bool { res.Reason == dnsfilter.RewriteEtcHosts) case filteringStatusSafeSearch: return res.IsFiltered && res.Reason == dnsfilter.FilteredSafeSearch + + case filteringStatusProcessed: + return !(res.Reason == dnsfilter.FilteredBlackList || + res.Reason == dnsfilter.FilteredBlockedService || + res.Reason == dnsfilter.NotFilteredWhiteList) + default: return false } - - default: - return false } return false