From 143616ca6e302aff0e5374d34c1d85fbd11ab544 Mon Sep 17 00:00:00 2001 From: Stanislav Chzhen Date: Thu, 23 Mar 2023 13:46:57 +0300 Subject: [PATCH] Pull request 1736: 4299-querylog-stats-api Merge in DNS/adguard-home from 4299-querylog-stats-api to master Updates #1717. Updates #4299. Squashed commit of the following: commit 5b706b7997a536bc4fd2c532fb89ca5ab3536848 Merge: 48b62b0f 306c1983 Author: Stanislav Chzhen Date: Wed Mar 22 13:53:09 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit 48b62b0f1882f1ad120c6cdd90cd7dd8cb8a7738 Author: Vladislav Abdulmyanov Date: Wed Mar 22 12:25:04 2023 +0200 client: fix styles, add titles and descrs commit 97e31cff70d05b51bd0e5ea2d20e8e7a251a7e41 Author: Vladislav Abdulmyanov Date: Tue Mar 21 18:38:12 2023 +0200 client: add ignored domains for querylog commit 24d75c4376382205ae6b8f731b1cd23d517772c9 Author: Vladislav Abdulmyanov Date: Tue Mar 21 18:21:13 2023 +0200 client: add ignore domains for stats commit eefc3891d01f90af79fdac9ba8eea06d4d54a0bc Merge: 978675ea 1daabb97 Author: Stanislav Chzhen Date: Tue Mar 21 10:53:35 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit 978675ea2c07bf248b4c8f26ebdf78cf59a12ef5 Author: Stanislav Chzhen Date: Tue Mar 21 10:53:11 2023 +0300 openapi: fix chlog commit 2ed33007aade115d38b0ca582206cc10678b084c Author: Stanislav Chzhen Date: Mon Mar 20 17:49:07 2023 +0300 home: fix tests commit 6af11520c164553ee9fce8f214ea169672188d7e Author: Stanislav Chzhen Date: Mon Mar 20 17:40:16 2023 +0300 home: fix typo commit 56acdfde5b1ee8d16b232c1293b91affbe319ad1 Merge: 319da34d 48431f8b Author: Stanislav Chzhen Date: Mon Mar 20 17:32:58 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit 319da34de41ec84310b23bba2ad79c8a3a4c14ff Author: Stanislav Chzhen Date: Fri Mar 3 17:34:38 2023 +0300 querylog: fix docs commit d5a8f24d5b336e7bdbbca18069f6ede8c96bcc2c Author: Stanislav Chzhen Date: Fri Mar 3 11:42:00 2023 +0300 stats: fix docs commit e0cbfc1c4078180a05835ce7587e9f45484adc81 Merge: 4743c810 012e5beb Author: Stanislav Chzhen Date: Wed Mar 1 18:45:17 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit 4743c81038052b9e0ca29ae5f1565021d36ca1ef Author: Stanislav Chzhen Date: Wed Mar 1 18:14:16 2023 +0300 all: imp code; fix time conversion commit 34310cffd7e331d098c535590245387051674fa8 Author: Stanislav Chzhen Date: Wed Mar 1 12:34:11 2023 +0300 chlog: restore order commit cadd864a66655242948f1cb16e6d4945c0235d7e Merge: 2f3e25be bb226434 Author: Stanislav Chzhen Date: Wed Mar 1 12:26:06 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit 2f3e25bee56d2c6ddcf4aa2fc6a1dc51ed9b06e1 Author: Stanislav Chzhen Date: Wed Mar 1 12:25:14 2023 +0300 all: fix fmt commit d54022baa6c8a3d0d3c308a9b6b1a6a9dc6ac7b6 Author: Stanislav Chzhen Date: Tue Feb 28 16:16:40 2023 +0300 all: imp code; fix chlog commit df22de91f59a51194c55e7bcbe5bc3fcc60cb8e3 Merge: e1ea4797 a772212d Author: Stanislav Chzhen Date: Mon Feb 27 17:24:09 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit e1ea4797af974c36f06683ffc6eaaae917921a43 Merge: d7db0a5a bb80a7c2 Author: Stanislav Chzhen Date: Mon Feb 27 17:23:20 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit d7db0a5af1e1f49f6174c1c42e6d9306f2381d16 Author: Stanislav Chzhen Date: Mon Feb 27 17:12:20 2023 +0300 all: imp docs ... and 15 more commits --- CHANGELOG.md | 45 +++++- client/src/__locales/en.json | 4 + client/src/actions/queryLogs.js | 2 +- client/src/actions/stats.js | 2 +- client/src/api/Api.js | 20 +-- .../components/Settings/LogsConfig/Form.js | 36 ++++- .../components/Settings/LogsConfig/index.js | 10 +- .../components/Settings/StatsConfig/Form.js | 48 +++--- .../components/Settings/StatsConfig/index.js | 15 +- client/src/components/Settings/index.js | 6 + client/src/helpers/constants.js | 9 +- client/src/reducers/stats.js | 4 +- internal/aghnet/hostgen.go | 29 ++++ internal/home/config.go | 38 +++-- internal/home/dns.go | 37 +---- internal/home/upgrade.go | 44 ++++- internal/home/upgrade_test.go | 95 +++++++++++ internal/querylog/http.go | 131 ++++++++++++++- internal/querylog/qlog.go | 17 ++ internal/querylog/qlog_test.go | 15 +- internal/querylog/querylog.go | 18 +-- internal/querylog/search_test.go | 3 +- internal/stats/http.go | 104 +++++++++++- internal/stats/http_test.go | 152 ++++++++++++++++++ internal/stats/stats.go | 60 ++++--- internal/stats/stats_internal_test.go | 7 +- internal/stats/stats_test.go | 11 +- openapi/CHANGELOG.md | 65 ++++++++ openapi/openapi.yaml | 125 ++++++++++++++ 29 files changed, 995 insertions(+), 157 deletions(-) create mode 100644 internal/stats/http_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 61f8706a..39f44fc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,12 @@ NOTE: Add new changes BELOW THIS COMMENT. ### Added +- Two new HTTP APIs, `PUT /control/stats/config/update` and `GET + control/stats/config`, which can be used to set and receive the query log + configuration. See openapi/openapi.yaml for the full description. +- Two new HTTP APIs, `PUT /control/querylog/config/update` and `GET + control/querylog/config`, which can be used to set and receive the statistics + configuration. See openapi/openapi.yaml for the full description. - The ability to set custom IP for EDNS Client Subnet by using the DNS-server configuration section on the DNS settings page in the UI ([#1472]). - The ability to manage safesearch for each service by using the new @@ -37,8 +43,26 @@ NOTE: Add new changes BELOW THIS COMMENT. #### Configuration Changes -In this release, the schema version has changed from 17 to 19. +In this release, the schema version has changed from 17 to 20. +- Property `statistics.interval`, which in schema versions 19 and earlier used + to be an integer number of days, is now a string with a human-readable + duration: + + ```yaml + # BEFORE: + 'statistics': + # … + 'interval': 1 + + # AFTER: + 'statistics': + # … + 'interval': '24h' + ``` + + To rollback this change, convert the property back into days and change the + `schema_version` back to `19`. - The `dns.safesearch_enabled` field has been replaced with `safe_search` object containing per-service settings. - The `clients.persistent.safesearch_enabled` field has been replaced with @@ -64,6 +88,23 @@ In this release, the schema version has changed from 17 to 19. client's specific `clients.persistent.safesearch` and then change the `schema_version` back to `17`. +### Deprecated + +- The `GET /control/stats_info` HTTP API; use the new `GET + /control/stats/config` API instead. + + **NOTE:** If interval is custom then it will be equal to `90` days for + compatibility reasons. See openapi/openapi.yaml and `openapi/CHANGELOG.md`. +- The `POST /control/stats_config` HTTP API; use the new `PUT + /control/stats/config/update` API instead. +- The `GET /control/querylog_info` HTTP API; use the new `GET + /control/querylog/config` API instead. + + **NOTE:** If interval is custom then it will be equal to `90` days for + compatibility reasons. See openapi/openapi.yaml and `openapi/CHANGELOG.md`. +- The `POST /control/querylog_config` HTTP API; use the new `PUT + /control/querylog/config/update` API instead. + ### Fixed - Panic caused by empty top-level domain name label in `/etc/hosts` files @@ -103,8 +144,6 @@ See also the [v0.107.26 GitHub milestone][ms-v0.107.26]. #### Configuration Changes -In this release, the schema version has changed from 16 to 17. - - Property `edns_client_subnet`, which in schema versions 16 and earlier used to be a part of the `dns` object, is now part of the `dns.edns_client_subnet` object: diff --git a/client/src/__locales/en.json b/client/src/__locales/en.json index 5ccd771b..902fc4d3 100644 --- a/client/src/__locales/en.json +++ b/client/src/__locales/en.json @@ -525,6 +525,10 @@ "statistics_retention_confirm": "Are you sure you want to change statistics retention? If you decrease the interval value, some data will be lost", "statistics_cleared": "Statistics successfully cleared", "statistics_enable": "Enable statistics", + "ignore_domains": "Ignored domains (separated by newline)", + "ignore_domains_title": "Ignored domains", + "ignore_domains_desc_stats": "Queries for these domains are not written to the statistics", + "ignore_domains_desc_query": "Queries for these domains are not written to the query log", "interval_hours": "{{count}} hour", "interval_hours_plural": "{{count}} hours", "filters_configuration": "Filters configuration", diff --git a/client/src/actions/queryLogs.js b/client/src/actions/queryLogs.js index 99da2cb0..e07c6fae 100644 --- a/client/src/actions/queryLogs.js +++ b/client/src/actions/queryLogs.js @@ -177,7 +177,7 @@ export const getLogsConfigSuccess = createAction('GET_LOGS_CONFIG_SUCCESS'); export const getLogsConfig = () => async (dispatch) => { dispatch(getLogsConfigRequest()); try { - const data = await apiClient.getQueryLogInfo(); + const data = await apiClient.getQueryLogConfig(); dispatch(getLogsConfigSuccess(data)); } catch (error) { dispatch(addErrorToast({ error })); diff --git a/client/src/actions/stats.js b/client/src/actions/stats.js index d3948efa..0e5b416e 100644 --- a/client/src/actions/stats.js +++ b/client/src/actions/stats.js @@ -13,7 +13,7 @@ export const getStatsConfigSuccess = createAction('GET_STATS_CONFIG_SUCCESS'); export const getStatsConfig = () => async (dispatch) => { dispatch(getStatsConfigRequest()); try { - const data = await apiClient.getStatsInfo(); + const data = await apiClient.getStatsConfig(); dispatch(getStatsConfigSuccess(data)); } catch (error) { dispatch(addErrorToast({ error })); diff --git a/client/src/api/Api.js b/client/src/api/Api.js index d984bbb8..60f1faad 100644 --- a/client/src/api/Api.js +++ b/client/src/api/Api.js @@ -497,9 +497,9 @@ class Api { // Settings for statistics GET_STATS = { path: 'stats', method: 'GET' }; - STATS_INFO = { path: 'stats_info', method: 'GET' }; + GET_STATS_CONFIG = { path: 'stats/config', method: 'GET' }; - STATS_CONFIG = { path: 'stats_config', method: 'POST' }; + UPDATE_STATS_CONFIG = { path: 'stats/config/update', method: 'PUT' }; STATS_RESET = { path: 'stats_reset', method: 'POST' }; @@ -508,13 +508,13 @@ class Api { return this.makeRequest(path, method); } - getStatsInfo() { - const { path, method } = this.STATS_INFO; + getStatsConfig() { + const { path, method } = this.GET_STATS_CONFIG; return this.makeRequest(path, method); } setStatsConfig(data) { - const { path, method } = this.STATS_CONFIG; + const { path, method } = this.UPDATE_STATS_CONFIG; const config = { data, }; @@ -529,9 +529,9 @@ class Api { // Query log GET_QUERY_LOG = { path: 'querylog', method: 'GET' }; - QUERY_LOG_CONFIG = { path: 'querylog_config', method: 'POST' }; + UPDATE_QUERY_LOG_CONFIG = { path: 'querylog/config/update', method: 'PUT' }; - QUERY_LOG_INFO = { path: 'querylog_info', method: 'GET' }; + GET_QUERY_LOG_CONFIG = { path: 'querylog/config', method: 'GET' }; QUERY_LOG_CLEAR = { path: 'querylog_clear', method: 'POST' }; @@ -543,13 +543,13 @@ class Api { return this.makeRequest(url, method); } - getQueryLogInfo() { - const { path, method } = this.QUERY_LOG_INFO; + getQueryLogConfig() { + const { path, method } = this.GET_QUERY_LOG_CONFIG; return this.makeRequest(path, method); } setQueryLogConfig(data) { - const { path, method } = this.QUERY_LOG_CONFIG; + const { path, method } = this.UPDATE_QUERY_LOG_CONFIG; const config = { data, }; diff --git a/client/src/components/Settings/LogsConfig/Form.js b/client/src/components/Settings/LogsConfig/Form.js index 8db3f18d..b29b974e 100644 --- a/client/src/components/Settings/LogsConfig/Form.js +++ b/client/src/components/Settings/LogsConfig/Form.js @@ -4,18 +4,28 @@ import { Field, reduxForm } from 'redux-form'; import { Trans, withTranslation } from 'react-i18next'; import flow from 'lodash/flow'; -import { CheckboxField, renderRadioField, toFloatNumber } from '../../../helpers/form'; -import { FORM_NAME, QUERY_LOG_INTERVALS_DAYS } from '../../../helpers/constants'; +import { + CheckboxField, + renderRadioField, + toFloatNumber, + renderTextareaField, +} from '../../../helpers/form'; +import { + FORM_NAME, + QUERY_LOG_INTERVALS_DAYS, + HOUR, + DAY, +} from '../../../helpers/constants'; import '../FormButton.css'; const getIntervalTitle = (interval, t) => { switch (interval) { - case 0.25: + case 6 * HOUR: return t('interval_6_hour'); - case 1: + case DAY: return t('interval_24_hour'); default: - return t('interval_days', { count: interval }); + return t('interval_days', { count: interval / DAY }); } }; @@ -66,6 +76,22 @@ const Form = (props) => { {getIntervalFields(processing, t, toFloatNumber)} + +
+ ignore_domains_desc_query +
+
+ +