Pull request 1809: 4299-querylog-stats-clients-api

Merge in DNS/adguard-home from 4299-querylog-stats-clients-api to master

Squashed commit of the following:

commit 066100a7869d7572c4ae65b3c7b1487ac50baf15
Merge: 95bc00c0 5da77514
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Apr 14 13:57:30 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-clients-api

commit 95bc00c0b3d05b262ee0b90be9757e61cac0778c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Apr 13 11:48:39 2023 +0300

    all: fix typo

commit 4b868da48f0c976d204346e40ba948803be6397f
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Apr 13 11:42:52 2023 +0300

    all: fix text label

commit 7a3ba5c7f688bd53cf761b5e8e614fbe251bd006
Merge: 315256e3 6c8d89a4
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Apr 13 11:34:59 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-clients-api

commit 315256e3f3861b5116962f7c47384b7c72e41813
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Apr 11 19:07:18 2023 +0300

    all: ignore search, unit

commit 28c6ffec9558e7c38d7bd12055eabddb8f5675c2
Author: Artem Krisanov <a.krisanov@adguard.com>
Date:   Tue Apr 11 15:08:35 2023 +0300

    Added 'Protection' and 'Query Log and statistics' sections to client settings. Added checkboxes to ignore client in (query log/statistics)

commit 2657bd2b820d8b2b3d71d23e4545c867b9ae6cdf
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 10 17:28:59 2023 +0300

    all: add todo

commit e151fcbc0c36d8e6a5c091fbf374bf0e35804699
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 10 15:15:46 2023 +0300

    openapi: imp docs

commit 31875cbbd1bd09a73baa3636d0cc242b5ac35059
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 10 13:02:31 2023 +0300

    all: add querylog stats client ignore api

CHANGELOG.md

@@ -23,6 +23,12 @@ See also the [v0.107.29 GitHub milestone][ms-v0.107.29].
 NOTE: Add new changes BELOW THIS COMMENT.
 -->
 
+### Added
+
+- The ability to exclude client activity from the query log or statistics by
+  editing client's settings on the Clients settings page in the UI  ([#1717],
+  [#4299]).
+
 ### Fixed
 
 - Incorrect recording of blocked results as “Blocked by CNAME or IP” in the
@@ -30,6 +36,8 @@ NOTE: Add new changes BELOW THIS COMMENT.
 - All Safe Search services being unchecked by default.
 - Panic when a DNSCrypt stamp is invalid ([#5721]).
 
+[#1717]: https://github.com/AdguardTeam/AdGuardHome/issues/1717
+[#4299]: https://github.com/AdguardTeam/AdGuardHome/issues/4299
 [#5721]: https://github.com/AdguardTeam/AdGuardHome/issues/5721
 [#5725]: https://github.com/AdguardTeam/AdGuardHome/issues/5725
 

client/src/__locales/en.json

@@ -668,5 +668,9 @@
     "disable_notify_for_hours": "Disable protection for {{count}} hour",
     "disable_notify_for_hours_plural": "Disable protection for {{count}} hours",
     "disable_notify_until_tomorrow": "Disable protection until tomorrow",
-    "enable_protection_timer": "Protection will be enabled in {{time}}"
+    "enable_protection_timer": "Protection will be enabled in {{time}}",
+    "protection_section_label": "Protection",
+    "log_and_stats_section_label": "Query log and statistics",
+    "ignore_query_log": "Ignore this client in query log",
+    "ignore_statistics": "Ignore this client in statistics"
 }

client/src/components/Settings/Clients/Form.js

@@ -41,6 +41,17 @@ const settingsCheckboxes = [
         placeholder: 'use_adguard_parental',
     },
 ];
+
+const logAndStatsCheckboxes = [
+    {
+        name: 'ignore_querylog',
+        placeholder: 'ignore_query_log',
+    },
+    {
+        name: 'ignore_statistics',
+        placeholder: 'ignore_statistics',
+    },
+];
 const validate = (values) => {
     const errors = {};
     const { name, ids } = values;
@@ -148,6 +159,9 @@ let Form = (props) => {
         settings: {
             title: 'settings',
             component: <div label="settings" title={props.t('main_settings')}>
+                <div className="form__label--bot form__label--bold">
+                    {t('protection_section_label')}
+                </div>
                 {settingsCheckboxes.map((setting) => (
                     <div className="form__group" key={setting.name}>
                         <Field
@@ -185,6 +199,19 @@ let Form = (props) => {
                         </div>
                     ))}
                 </div>
+                <div className="form__label--bold form__label--top form__label--bot">
+                    {t('log_and_stats_section_label')}
+                </div>
+                {logAndStatsCheckboxes.map((setting) => (
+                    <div className="form__group" key={setting.name}>
+                        <Field
+                            name={setting.name}
+                            type="checkbox"
+                            component={CheckboxField}
+                            placeholder={t(setting.placeholder)}
+                        />
+                    </div>
+                ))}
             </div>,
         },
         block_services: {

client/src/components/Settings/Settings.css

@@ -100,6 +100,14 @@
     margin-bottom: 0;
 }
 
+.form__label--bot {
+    margin-bottom: 10px;
+}
+
+.form__label--top {
+    margin-top: 10px;
+}
+
 .form__status {
     margin-top: 10px;
     font-size: 14px;

internal/home/clientshttp.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/netip"
 
+	"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
 	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering"
 )
@@ -44,6 +45,9 @@ type clientJSON struct {
 	SafeSearchEnabled        bool `json:"safesearch_enabled"`
 	UseGlobalBlockedServices bool `json:"use_global_blocked_services"`
 	UseGlobalSettings        bool `json:"use_global_settings"`
+
+	IgnoreQueryLog   aghalg.NullBool `json:"ignore_querylog"`
+	IgnoreStatistics aghalg.NullBool `json:"ignore_statistics"`
 }
 
 type runtimeClientJSON struct {
@@ -90,7 +94,7 @@ func (clients *clientsContainer) handleGetClients(w http.ResponseWriter, r *http
 }
 
 // jsonToClient converts JSON object to Client object.
-func (clients *clientsContainer) jsonToClient(cj clientJSON) (c *Client, err error) {
+func (clients *clientsContainer) jsonToClient(cj clientJSON, prev *Client) (c *Client, err error) {
 	var safeSearchConf filtering.SafeSearchConfig
 	if cj.SafeSearchConf != nil {
 		safeSearchConf = *cj.SafeSearchConf
@@ -129,6 +133,18 @@ func (clients *clientsContainer) jsonToClient(cj clientJSON) (c *Client, err err
 		UseOwnBlockedServices: !cj.UseGlobalBlockedServices,
 	}
 
+	if cj.IgnoreQueryLog != aghalg.NBNull {
+		c.IgnoreQueryLog = cj.IgnoreQueryLog == aghalg.NBTrue
+	} else if prev != nil {
+		c.IgnoreQueryLog = prev.IgnoreQueryLog
+	}
+
+	if cj.IgnoreStatistics != aghalg.NBNull {
+		c.IgnoreStatistics = cj.IgnoreStatistics == aghalg.NBTrue
+	} else if prev != nil {
+		c.IgnoreStatistics = prev.IgnoreStatistics
+	}
+
 	if safeSearchConf.Enabled {
 		err = c.setSafeSearch(
 			safeSearchConf,
@@ -165,6 +181,9 @@ func clientToJSON(c *Client) (cj *clientJSON) {
 		BlockedServices:          c.BlockedServices,
 
 		Upstreams: c.Upstreams,
+
+		IgnoreQueryLog:   aghalg.BoolToNullBool(c.IgnoreQueryLog),
+		IgnoreStatistics: aghalg.BoolToNullBool(c.IgnoreStatistics),
 	}
 }
 
@@ -178,7 +197,7 @@ func (clients *clientsContainer) handleAddClient(w http.ResponseWriter, r *http.
 		return
 	}
 
-	c, err := clients.jsonToClient(cj)
+	c, err := clients.jsonToClient(cj, nil)
 	if err != nil {
 		aghhttp.Error(r, w, http.StatusBadRequest, "%s", err)
 
@@ -232,6 +251,8 @@ type updateJSON struct {
 }
 
 // handleUpdateClient is the handler for POST /control/clients/update HTTP API.
+//
+// TODO(s.chzhen):  Accept updated parameters instead of whole structure.
 func (clients *clientsContainer) handleUpdateClient(w http.ResponseWriter, r *http.Request) {
 	dj := updateJSON{}
 	err := json.NewDecoder(r.Body).Decode(&dj)
@@ -247,7 +268,21 @@ func (clients *clientsContainer) handleUpdateClient(w http.ResponseWriter, r *ht
 		return
 	}
 
-	c, err := clients.jsonToClient(dj.Data)
+	var prev *Client
+	var ok bool
+
+	func() {
+		clients.lock.Lock()
+		defer clients.lock.Unlock()
+
+		prev, ok = clients.list[dj.Name]
+	}()
+
+	if !ok {
+		aghhttp.Error(r, w, http.StatusBadRequest, "client not found")
+	}
+
+	c, err := clients.jsonToClient(dj.Data, prev)
 	if err != nil {
 		aghhttp.Error(r, w, http.StatusBadRequest, "%s", err)
 

internal/querylog/search.go

@@ -288,6 +288,10 @@ func (l *queryLog) readNextEntry(
 		// Go on and try to match anyway.
 	}
 
+	if e.client != nil && e.client.IgnoreQueryLog {
+		return nil, ts, nil
+	}
+
 	ts = e.Time.UnixNano()
 	if !params.match(e) {
 		return nil, ts, nil

internal/stats/unit.go

@@ -423,7 +423,7 @@ func (s *StatsCtx) getData(limit uint32) (StatsResp, bool) {
 		ReplacedParental:     statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RParental] }),
 		TopQueried:           topsCollector(units, maxDomains, s.ignored, func(u *unitDB) (pairs []countPair) { return u.Domains }),
 		TopBlocked:           topsCollector(units, maxDomains, s.ignored, func(u *unitDB) (pairs []countPair) { return u.BlockedDomains }),
-		TopClients:           topsCollector(units, maxClients, nil, func(u *unitDB) (pairs []countPair) { return u.Clients }),
+		TopClients:           topsCollector(units, maxClients, nil, topClientPairs(s)),
 	}
 
 	// Total counters:
@@ -460,3 +460,17 @@ func (s *StatsCtx) getData(limit uint32) (StatsResp, bool) {
 
 	return data, true
 }
+
+func topClientPairs(s *StatsCtx) (pg pairsGetter) {
+	return func(u *unitDB) (clients []countPair) {
+		for _, c := range u.Clients {
+			if c.Name != "" && !s.shouldCountClient([]string{c.Name}) {
+				continue
+			}
+
+			clients = append(clients, c)
+		}
+
+		return clients
+	}
+}

openapi/CHANGELOG.md

@@ -4,6 +4,18 @@
 
 ## v0.108.0: API changes
 
+## v0.107.29: API changes
+
+### `GET /control/clients` And `GET /control/clients/find`
+* The new optional fields `"ignore_querylog"` and `"ignore_statistics"` are set
+  if AdGuard Home excludes client activity from query log or statistics.
+
+### `POST /control/clients/add` And `POST /control/clients/update`
+* The new optional fields `"ignore_querylog"` and `"ignore_statistics"` make
+  AdGuard Home exclude client activity from query log or statistics.  If not
+  set AdGuard Home will use default value (false).  It can be changed in the
+  future versions.
+
 ## v0.107.27: API changes
 
 ### The new optional fields `"edns_cs_use_custom"` and `"edns_cs_custom_ip"` in `DNSConfig`

openapi/openapi.yaml

@@ -2494,6 +2494,26 @@
           'items':
             'type': 'string'
           'type': 'array'
+        'ignore_querylog':
+          'description': |
+            NOTE: If `ignore_querylog` is not set in HTTP API `GET /clients/add`
+            request then default value (false) will be used.
+
+            If `ignore_querylog` is not set in HTTP API `GET /clients/update`
+            request then the existing value will not be changed.
+
+            This behaviour can be changed in the future versions.
+          'type': 'boolean'
+        'ignore_statistics':
+          'description': |
+            NOTE: If `ignore_statistics` is not set in HTTP API `GET
+            /clients/add` request then default value (false) will be used.
+
+            If `ignore_statistics` is not set in HTTP API `GET /clients/update`
+            request then the existing value will not be changed.
+
+            This behaviour can be changed in the future versions.
+          'type': 'boolean'
     'ClientAuto':
       'type': 'object'
       'description': 'Auto-Client information'
@@ -2547,6 +2567,8 @@
         'whois_info': {}
         'disallowed': false
         'disallowed_rule': ''
+        'ignore_querylog': false
+        'ignore_statistics': false
       - '1.2.3.4':
         'name': 'Client 1-2-3-4'
         'ids': ['1.2.3.4']
@@ -2562,6 +2584,8 @@
         'whois_info': {}
         'disallowed': false
         'disallowed_rule': ''
+        'ignore_querylog': false
+        'ignore_statistics': false
     'AccessListResponse':
       '$ref': '#/components/schemas/AccessList'
     'AccessSetRequest':
@@ -2643,7 +2667,10 @@
             set to true, and this string is empty, then the client IP is
             disallowed by the "allowed IP list", that is it is not included in
             the allowed list.
-
+        'ignore_querylog':
+          'type': 'boolean'
+        'ignore_statistics':
+          'type': 'boolean'
     'WhoisInfo':
       'type': 'object'
       'additionalProperties':