Pull request 1983: 5720-wildcard-ignored-domains

Updates #5720.

Squashed commit of the following:

commit e8093c990f15e2efc496f1a04f87360825e34e96
Merge: df5413eef 28fefaff1
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 5 15:06:33 2023 +0300

    Merge branch 'master' into 5720-wildcard-ignored-domains

commit df5413eefeac2c7e34eb725db9e2908b5b2d08cb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 5 14:49:05 2023 +0300

    confmigrate: imp docs

commit 1644d99b730cc7f22c9d75b8e990149d3ce5b32a
Merge: 9542ee161 1e4517898
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 5 14:23:42 2023 +0300

    Merge branch 'master' into 5720-wildcard-ignored-domains

commit 9542ee1616c1dd4bdb6ec9a2af79a2af3858a7e3
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 5 12:48:48 2023 +0300

    all: upd chlog

commit 183f84a7f73c7bd33669bd108076f60514ca101e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Sep 1 17:11:31 2023 +0300

    all: imp chlog

commit a704325352a577a9b6652f011b82180ec3a6e095
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Aug 31 18:59:52 2023 +0300

    all: imp code

commit fe99c3b883500850399b1feb72c914ab878b3107
Merge: 7f11e9460 0182b9ec1
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Aug 31 18:43:09 2023 +0300

    Merge branch 'master' into 5720-wildcard-ignored-domains

commit 7f11e94609027ed821a125d27a1ffde03f37334a
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Aug 30 19:57:51 2023 +0300

    aghnet: add tests

commit f10f9190ce1064a5d03155e8b6bba61db977897b
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Aug 30 18:32:07 2023 +0300

    all: add conf migration

commit a53c14df129765366966c5230dd53aa29bdd25c5
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Aug 30 13:37:30 2023 +0300

    all: add ignore engine

CHANGELOG.md

@@ -25,6 +25,8 @@ NOTE: Add new changes BELOW THIS COMMENT.
 
 ### Added
 
+- AdBlock-style syntax support for ignored domains in logs and statistics
+  ([#5720]).
 - [`Strict-Transport-Security`][hsts] header in the HTTP API and DNS-over-HTTPS
   responses when HTTPS is forced ([#2998]).  See [RFC 6979][rfc6797].
 - UI for the schedule of the service-blocking pause ([#951]).
@@ -50,7 +52,11 @@ NOTE: Add new changes BELOW THIS COMMENT.
 
 #### Configuration Changes
 
-In this release, the schema version has changed from 24 to 26.
+In this release, the schema version has changed from 24 to 27.
+
+- Ignore rules blocking `.` in `querylog.…` and `stats.…` have been migrated to
+  AdBlock syntax (`|.^`).  To rollback this change, restore the rules  and
+  change the `schema_version` back to `26`.
 
 - Filtering-related settings have been moved from `dns` section of the YAML
   configuration file to the new section `filtering`:
@@ -156,6 +162,7 @@ In this release, the schema version has changed from 24 to 26.
 [#1453]: https://github.com/AdguardTeam/AdGuardHome/issues/1453
 [#2998]: https://github.com/AdguardTeam/AdGuardHome/issues/2998
 [#3701]: https://github.com/AdguardTeam/AdGuardHome/issues/3701
+[#5720]: https://github.com/AdguardTeam/AdGuardHome/issues/5720
 [#5793]: https://github.com/AdguardTeam/AdGuardHome/issues/5793
 [#5948]: https://github.com/AdguardTeam/AdGuardHome/issues/5948
 [#6020]: https://github.com/AdguardTeam/AdGuardHome/issues/6020

client/src/__locales/en.json

@@ -537,8 +537,8 @@
     "statistics_enable": "Enable statistics",
     "ignore_domains": "Ignored domains (separated by newline)",
     "ignore_domains_title": "Ignored domains",
-    "ignore_domains_desc_stats": "Queries for these domains are not written to the statistics",
+    "ignore_domains_desc_stats": "Queries matching these rules are not written to the statistics",
-    "ignore_domains_desc_query": "Queries for these domains are not written to the query log",
+    "ignore_domains_desc_query": "Queries matching these rules are not written to the query log",
     "interval_hours": "{{count}} hour",
     "interval_hours_plural": "{{count}} hours",
     "filters_configuration": "Filters configuration",

internal/aghnet/ignore.go

@@ -0,0 +1,56 @@
+package aghnet
+
+import (
+	"strings"
+
+	"github.com/AdguardTeam/urlfilter"
+	"github.com/AdguardTeam/urlfilter/filterlist"
+	"golang.org/x/exp/slices"
+)
+
+// IgnoreEngine contains the list of rules for ignoring hostnames and matches
+// them.
+//
+// TODO(s.chzhen):  Move all urlfilter stuff to aghfilter.
+type IgnoreEngine struct {
+	// engine is the filtering engine that can match rules for ignoring
+	// hostnames.
+	engine *urlfilter.DNSEngine
+
+	// ignored is the list of rules for ignoring hostnames.
+	ignored []string
+}
+
+// NewIgnoreEngine creates a new instance of the IgnoreEngine and stores the
+// list of rules for ignoring hostnames.
+func NewIgnoreEngine(ignored []string) (e *IgnoreEngine, err error) {
+	ruleList := &filterlist.StringRuleList{
+		RulesText:      strings.ToLower(strings.Join(ignored, "\n")),
+		IgnoreCosmetic: true,
+	}
+	ruleStorage, err := filterlist.NewRuleStorage([]filterlist.RuleList{ruleList})
+	if err != nil {
+		return nil, err
+	}
+
+	return &IgnoreEngine{
+		engine:  urlfilter.NewDNSEngine(ruleStorage),
+		ignored: ignored,
+	}, nil
+}
+
+// Has returns true if IgnoreEngine matches the host.
+func (e *IgnoreEngine) Has(host string) (ignore bool) {
+	if e == nil {
+		return false
+	}
+
+	_, ignore = e.engine.Match(host)
+
+	return ignore
+}
+
+// Values returns a copy of list of rules for ignoring hostnames.
+func (e *IgnoreEngine) Values() (ignored []string) {
+	return slices.Clone(e.ignored)
+}

internal/aghnet/ignore_test.go

@@ -0,0 +1,46 @@
+package aghnet_test
+
+import (
+	"testing"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIgnoreEngine_Has(t *testing.T) {
+	hostnames := []string{
+		"*.example.com",
+		"example.com",
+		"|.^",
+	}
+
+	engine, err := aghnet.NewIgnoreEngine(hostnames)
+	require.NotNil(t, engine)
+	require.NoError(t, err)
+
+	testCases := []struct {
+		name   string
+		host   string
+		ignore bool
+	}{{
+		name:   "basic",
+		host:   "example.com",
+		ignore: true,
+	}, {
+		name:   "root",
+		host:   ".",
+		ignore: true,
+	}, {
+		name:   "wildcard",
+		host:   "www.example.com",
+		ignore: true,
+	}, {
+		name:   "not_ignored",
+		host:   "something.com",
+		ignore: false,
+	}}
+
+	for _, tc := range testCases {
+		require.Equal(t, tc.ignore, engine.Has(tc.host))
+	}
+}

internal/confmigrate/migrations_test.go

@@ -1563,3 +1563,86 @@ func TestUpgradeSchema25to26(t *testing.T) {
 		})
 	}
 }
+
+func TestUpgradeSchema26to27(t *testing.T) {
+	const newSchemaVer = 27
+
+	testCases := []struct {
+		in   yobj
+		want yobj
+		name string
+	}{{
+		name: "empty",
+		in:   yobj{},
+		want: yobj{
+			"schema_version": newSchemaVer,
+		},
+	}, {
+		name: "single_dot",
+		in: yobj{
+			"querylog": yobj{
+				"ignored": yarr{
+					".",
+				},
+			},
+			"statistics": yobj{
+				"ignored": yarr{
+					".",
+				},
+			},
+		},
+		want: yobj{
+			"querylog": yobj{
+				"ignored": yarr{
+					"|.^",
+				},
+			},
+			"statistics": yobj{
+				"ignored": yarr{
+					"|.^",
+				},
+			},
+			"schema_version": newSchemaVer,
+		},
+	}, {
+		name: "mixed",
+		in: yobj{
+			"querylog": yobj{
+				"ignored": yarr{
+					".",
+					"example.com",
+				},
+			},
+			"statistics": yobj{
+				"ignored": yarr{
+					".",
+					"example.org",
+				},
+			},
+		},
+		want: yobj{
+			"querylog": yobj{
+				"ignored": yarr{
+					"|.^",
+					"example.com",
+				},
+			},
+			"statistics": yobj{
+				"ignored": yarr{
+					"|.^",
+					"example.org",
+				},
+			},
+			"schema_version": newSchemaVer,
+		},
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := migrateTo27(tc.in)
+			require.NoError(t, err)
+
+			assert.Equal(t, tc.want, tc.in)
+		})
+	}
+}

internal/confmigrate/migrator.go

@@ -10,7 +10,7 @@ import (
 )
 
 // LastSchemaVersion is the most recent schema version.
-const LastSchemaVersion uint = 26
+const LastSchemaVersion uint = 27
 
 // Config is a the configuration for initializing a [Migrator].
 type Config struct {
@@ -122,6 +122,7 @@ func (m *Migrator) upgradeConfigSchema(current, target uint, diskConf yobj) (err
 		23: migrateTo24,
 		24: migrateTo25,
 		25: migrateTo26,
+		26: migrateTo27,
 	}
 
 	for i, migrate := range upgrades[current:target] {

internal/confmigrate/migrator_test.go

@@ -181,6 +181,10 @@ func TestMigrateConfig_Migrate(t *testing.T) {
 		yamlEqFunc:    require.YAMLEq,
 		name:          "v26",
 		targetVersion: 26,
+	}, {
+		yamlEqFunc:    require.YAMLEq,
+		name:          "v27",
+		targetVersion: 27,
 	}}
 
 	for _, tc := range testCases {

internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/input.yml

@@ -0,0 +1,123 @@
+http:
+  address: 127.0.0.1:3000
+  session_ttl: 3h
+  pprof:
+    enabled: true
+    port: 6060
+users:
+- name: testuser
+  password: testpassword
+dns:
+  bind_hosts:
+  - 127.0.0.1
+  port: 53
+  parental_sensitivity: 0
+  upstream_dns:
+  - tls://1.1.1.1
+  - tls://1.0.0.1
+  - quic://8.8.8.8:784
+  bootstrap_dns:
+  - 8.8.8.8:53
+  edns_client_subnet:
+    enabled:    true
+    use_custom: false
+    custom_ip:  ""
+filtering:
+  filtering_enabled: true
+  parental_enabled: false
+  safebrowsing_enabled: false
+  safe_search:
+    enabled:    false
+    bing:       true
+    duckduckgo: true
+    google:     true
+    pixabay:    true
+    yandex:     true
+    youtube:    true
+  protection_enabled: true
+  blocked_services:
+    schedule:
+      time_zone: Local
+    ids:
+    - 500px
+  blocked_response_ttl: 10
+filters:
+- url: https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
+  name: ""
+  enabled: true
+- url: https://adaway.org/hosts.txt
+  name: AdAway
+  enabled: false
+- url: https://hosts-file.net/ad_servers.txt
+  name: hpHosts - Ad and Tracking servers only
+  enabled: false
+- url: http://www.malwaredomainlist.com/hostslist/hosts.txt
+  name: MalwareDomainList.com Hosts List
+  enabled: false
+clients:
+  persistent:
+  - name: localhost
+    ids:
+    - 127.0.0.1
+    - aa:aa:aa:aa:aa:aa
+    use_global_settings: true
+    use_global_blocked_services: true
+    filtering_enabled: false
+    parental_enabled: false
+    safebrowsing_enabled: false
+    safe_search:
+      enabled:    true
+      bing:       true
+      duckduckgo: true
+      google:     true
+      pixabay:    true
+      yandex:     true
+      youtube:    true
+    blocked_services:
+      schedule:
+        time_zone: Local
+      ids:
+      - 500px
+  runtime_sources:
+    whois: true
+    arp:   true
+    rdns:  true
+    dhcp:  true
+    hosts: true
+dhcp:
+  enabled: false
+  interface_name: vboxnet0
+  local_domain_name: local
+  dhcpv4:
+    gateway_ip: 192.168.0.1
+    subnet_mask: 255.255.255.0
+    range_start: 192.168.0.10
+    range_end: 192.168.0.250
+    lease_duration: 1234
+    icmp_timeout_msec: 10
+schema_version: 26
+user_rules: []
+querylog:
+  enabled: true
+  file_enabled: true
+  interval: 720h
+  size_memory: 1000
+  ignored:
+  - '.'
+statistics:
+  enabled: true
+  interval: 240h
+  ignored:
+  - '.'
+os:
+  group: ''
+  rlimit_nofile: 123
+  user: ''
+log:
+  file: ""
+  max_backups: 0
+  max_size: 100
+  max_age: 3
+  compress: true
+  local_time: false
+  verbose: true

internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/output.yml

@@ -0,0 +1,123 @@
+http:
+  address: 127.0.0.1:3000
+  session_ttl: 3h
+  pprof:
+    enabled: true
+    port: 6060
+users:
+- name: testuser
+  password: testpassword
+dns:
+  bind_hosts:
+  - 127.0.0.1
+  port: 53
+  parental_sensitivity: 0
+  upstream_dns:
+  - tls://1.1.1.1
+  - tls://1.0.0.1
+  - quic://8.8.8.8:784
+  bootstrap_dns:
+  - 8.8.8.8:53
+  edns_client_subnet:
+    enabled:    true
+    use_custom: false
+    custom_ip:  ""
+filtering:
+  filtering_enabled: true
+  parental_enabled: false
+  safebrowsing_enabled: false
+  safe_search:
+    enabled:    false
+    bing:       true
+    duckduckgo: true
+    google:     true
+    pixabay:    true
+    yandex:     true
+    youtube:    true
+  protection_enabled: true
+  blocked_services:
+    schedule:
+      time_zone: Local
+    ids:
+    - 500px
+  blocked_response_ttl: 10
+filters:
+- url: https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
+  name: ""
+  enabled: true
+- url: https://adaway.org/hosts.txt
+  name: AdAway
+  enabled: false
+- url: https://hosts-file.net/ad_servers.txt
+  name: hpHosts - Ad and Tracking servers only
+  enabled: false
+- url: http://www.malwaredomainlist.com/hostslist/hosts.txt
+  name: MalwareDomainList.com Hosts List
+  enabled: false
+clients:
+  persistent:
+  - name: localhost
+    ids:
+    - 127.0.0.1
+    - aa:aa:aa:aa:aa:aa
+    use_global_settings: true
+    use_global_blocked_services: true
+    filtering_enabled: false
+    parental_enabled: false
+    safebrowsing_enabled: false
+    safe_search:
+      enabled:    true
+      bing:       true
+      duckduckgo: true
+      google:     true
+      pixabay:    true
+      yandex:     true
+      youtube:    true
+    blocked_services:
+      schedule:
+        time_zone: Local
+      ids:
+      - 500px
+  runtime_sources:
+    whois: true
+    arp:   true
+    rdns:  true
+    dhcp:  true
+    hosts: true
+dhcp:
+  enabled: false
+  interface_name: vboxnet0
+  local_domain_name: local
+  dhcpv4:
+    gateway_ip: 192.168.0.1
+    subnet_mask: 255.255.255.0
+    range_start: 192.168.0.10
+    range_end: 192.168.0.250
+    lease_duration: 1234
+    icmp_timeout_msec: 10
+schema_version: 27
+user_rules: []
+querylog:
+  enabled: true
+  file_enabled: true
+  interval: 720h
+  size_memory: 1000
+  ignored:
+  - '|.^'
+statistics:
+  enabled: true
+  interval: 240h
+  ignored:
+  - '|.^'
+os:
+  group: ''
+  rlimit_nofile: 123
+  user: ''
+log:
+  file: ""
+  max_backups: 0
+  max_size: 100
+  max_age: 3
+  compress: true
+  local_time: false
+  verbose: true

internal/confmigrate/v27.go

@@ -0,0 +1,77 @@
+package confmigrate
+
+// migrateTo27 performs the following changes:
+//
+//	# BEFORE:
+//	'querylog':
+//	  'ignored':
+//	  - '.'
+//	  - # …
+//	  # …
+//	'statistics':
+//	  'ignored':
+//	  - '.'
+//	  - # …
+//	  # …
+//	# …
+//
+//	# AFTER:
+//	'querylog':
+//	  'ignored':
+//	  - '|.^'
+//	  - # …
+//	  # …
+//	'statistics':
+//	  'ignored':
+//	  - '|.^'
+//	  - # …
+//	  # …
+//	# …
+func migrateTo27(diskConf yobj) (err error) {
+	diskConf["schema_version"] = 27
+
+	keys := []string{"querylog", "statistics"}
+	for _, k := range keys {
+		err = replaceDot(diskConf, k)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// replaceDot replaces rules blocking root domain "." with AdBlock style syntax
+// "|.^".
+func replaceDot(diskConf yobj, key string) (err error) {
+	var obj yobj
+	var ok bool
+	obj, ok, err = fieldVal[yobj](diskConf, key)
+	if err != nil {
+		return err
+	} else if !ok {
+		return nil
+	}
+
+	var ignored yarr
+	ignored, ok, err = fieldVal[yarr](obj, "ignored")
+	if err != nil {
+		return err
+	} else if !ok {
+		return nil
+	}
+
+	for i, hostVal := range ignored {
+		var host string
+		host, ok = hostVal.(string)
+		if !ok {
+			continue
+		}
+
+		if host == "." {
+			ignored[i] = "|.^"
+		}
+	}
+
+	return nil
+}

internal/dnsforward/access.go

@@ -28,6 +28,7 @@ type accessManager struct {
 	allowedClientIDs *stringutil.Set
 	blockedClientIDs *stringutil.Set
 
+	// TODO(s.chzhen):  Use [aghnet.IgnoreEngine].
 	blockedHostsEng *urlfilter.DNSEngine
 
 	// TODO(a.garipov): Create a type for a set of IP networks.

internal/home/config.go

@@ -22,7 +22,6 @@ import (
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/timeutil"
 	"github.com/google/renameio/v2/maybe"
-	"golang.org/x/exp/slices"
 	yaml "gopkg.in/yaml.v3"
 )
 
@@ -594,7 +593,6 @@ func (c *configuration) write() (err error) {
 		config.Stats.Interval = timeutil.Duration{Duration: statsConf.Limit}
 		config.Stats.Enabled = statsConf.Enabled
 		config.Stats.Ignored = statsConf.Ignored.Values()
-		slices.Sort(config.Stats.Ignored)
 	}
 
 	if Context.queryLog != nil {
@@ -606,7 +604,6 @@ func (c *configuration) write() (err error) {
 		config.QueryLog.Interval = timeutil.Duration{Duration: dc.RotationIvl}
 		config.QueryLog.MemSize = dc.MemSize
 		config.QueryLog.Ignored = dc.Ignored.Values()
-		slices.Sort(config.Stats.Ignored)
 	}
 
 	if Context.filters != nil {

internal/home/dns.go

@@ -59,12 +59,12 @@ func initDNS() (err error) {
 		ShouldCountClient: Context.clients.shouldCountClient,
 	}
 
-	set, err := aghnet.NewDomainNameSet(config.Stats.Ignored)
+	engine, err := aghnet.NewIgnoreEngine(config.Stats.Ignored)
 	if err != nil {
 		return fmt.Errorf("statistics: ignored list: %w", err)
 	}
 
-	statsConf.Ignored = set
+	statsConf.Ignored = engine
 	Context.stats, err = stats.New(statsConf)
 	if err != nil {
 		return fmt.Errorf("init stats: %w", err)
@@ -83,12 +83,12 @@ func initDNS() (err error) {
 		FileEnabled:       config.QueryLog.FileEnabled,
 	}
 
-	set, err = aghnet.NewDomainNameSet(config.QueryLog.Ignored)
+	engine, err = aghnet.NewIgnoreEngine(config.QueryLog.Ignored)
 	if err != nil {
 		return fmt.Errorf("querylog: ignored list: %w", err)
 	}
 
-	conf.Ignored = set
+	conf.Ignored = engine
 	Context.queryLog, err = querylog.New(conf)
 	if err != nil {
 		return fmt.Errorf("init querylog: %w", err)

internal/querylog/http.go

@@ -17,7 +17,6 @@ import (
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/stringutil"
 	"github.com/AdguardTeam/golibs/timeutil"
-	"golang.org/x/exp/slices"
 	"golang.org/x/net/idna"
 )
 
@@ -141,8 +140,6 @@ func (l *queryLog) handleGetQueryLogConfig(w http.ResponseWriter, r *http.Reques
 		}
 	}()
 
-	slices.Sort(resp.Ignored)
-
 	aghhttp.WriteJSONResponseOK(w, r, resp)
 }
 
@@ -224,7 +221,7 @@ func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	set, err := aghnet.NewDomainNameSet(newConf.Ignored)
+	engine, err := aghnet.NewIgnoreEngine(newConf.Ignored)
 	if err != nil {
 		aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err)
 
@@ -258,7 +255,7 @@ func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Reques
 
 	conf := *l.conf
 
-	conf.Ignored = set
+	conf.Ignored = engine
 	conf.RotationIvl = ivl
 	conf.Enabled = newConf.Enabled == aghalg.NBTrue
 

internal/querylog/qlog.go

@@ -127,7 +127,6 @@ func (l *queryLog) WriteDiskConfig(c *Config) {
 	defer l.confMu.RUnlock()
 
 	*c = *l.conf
-	c.Ignored = l.conf.Ignored.Clone()
 }
 
 // Clear memory buffer and remove log files

internal/querylog/qlog_test.go

@@ -5,8 +5,8 @@ import (
 	"net"
 	"testing"
 
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering"
-	"github.com/AdguardTeam/golibs/stringutil"
 	"github.com/AdguardTeam/golibs/testutil"
 	"github.com/AdguardTeam/golibs/timeutil"
 	"github.com/miekg/dns"
@@ -256,10 +256,21 @@ func TestQueryLogFileDisabled(t *testing.T) {
 
 func TestQueryLogShouldLog(t *testing.T) {
 	const (
-		ignored1 = "ignor.ed"
+		ignored1        = "ignor.ed"
-		ignored2 = "ignored.to"
+		ignored2        = "ignored.to"
+		ignoredWildcard = "*.ignored.com"
+		ignoredRoot     = "|.^"
 	)
-	set := stringutil.NewSet(ignored1, ignored2)
+
+	ignored := []string{
+		ignored1,
+		ignored2,
+		ignoredWildcard,
+		ignoredRoot,
+	}
+
+	engine, err := aghnet.NewIgnoreEngine(ignored)
+	require.NoError(t, err)
 
 	findClient := func(ids []string) (c *Client, err error) {
 		log := ids[0] == "no_log"
@@ -268,7 +279,7 @@ func TestQueryLogShouldLog(t *testing.T) {
 	}
 
 	l, err := newQueryLog(Config{
-		Ignored:     set,
+		Ignored:     engine,
 		Enabled:     true,
 		RotationIvl: timeutil.Day,
 		MemSize:     100,
@@ -297,6 +308,16 @@ func TestQueryLogShouldLog(t *testing.T) {
 		host:    ignored2,
 		ids:     []string{"whatever"},
 		wantLog: false,
+	}, {
+		name:    "no_log_ignored_wildcard",
+		host:    "www.ignored.com",
+		ids:     []string{"whatever"},
+		wantLog: false,
+	}, {
+		name:    "no_log_ignored_root",
+		host:    ".",
+		ids:     []string{"whatever"},
+		wantLog: false,
 	}, {
 		name:    "no_log_client_ignore",
 		host:    "example.com",

internal/querylog/querylog.go

@@ -11,7 +11,6 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering"
 	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/stringutil"
 	"github.com/miekg/dns"
 )
 
@@ -36,8 +35,9 @@ type QueryLog interface {
 //
 // Do not alter any fields of this structure after using it.
 type Config struct {
-	// Ignored is the list of host names, which should not be written to log.
+	// Ignored contains the list of host names, which should not be written to
-	Ignored *stringutil.Set
+	// log, and matches them.
+	Ignored *aghnet.IgnoreEngine
 
 	// Anonymizer processes the IP addresses to anonymize those if needed.
 	Anonymizer *aghnet.IPMut

internal/stats/http.go

@@ -12,7 +12,6 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/timeutil"
-	"golang.org/x/exp/slices"
 )
 
 // topAddrs is an alias for the types of the TopFoo fields of statsResponse.
@@ -140,8 +139,6 @@ func (s *StatsCtx) handleGetStatsConfig(w http.ResponseWriter, r *http.Request)
 		}
 	}()
 
-	slices.Sort(resp.Ignored)
-
 	aghhttp.WriteJSONResponseOK(w, r, resp)
 }
 
@@ -184,7 +181,7 @@ func (s *StatsCtx) handlePutStatsConfig(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	set, err := aghnet.NewDomainNameSet(reqData.Ignored)
+	engine, err := aghnet.NewIgnoreEngine(reqData.Ignored)
 	if err != nil {
 		aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err)
 
@@ -210,7 +207,7 @@ func (s *StatsCtx) handlePutStatsConfig(w http.ResponseWriter, r *http.Request)
 	s.confMu.Lock()
 	defer s.confMu.Unlock()
 
-	s.ignored = set
+	s.ignored = engine
 	s.limit = ivl
 	s.enabled = reqData.Enabled == aghalg.NBTrue
 }

internal/stats/http_test.go

@@ -75,29 +75,6 @@ func TestHandleStatsConfig(t *testing.T) {
 		},
 		wantCode: http.StatusOK,
 		wantErr:  "",
-	}, {
-		name: "ignored_duplicate",
-		body: getConfigResp{
-			Enabled:  aghalg.NBTrue,
-			Interval: float64(minIvl.Milliseconds()),
-			Ignored: []string{
-				"ignor.ed",
-				"ignor.ed",
-			},
-		},
-		wantCode: http.StatusUnprocessableEntity,
-		wantErr:  "ignored: duplicate hostname \"ignor.ed\" at index 1\n",
-	}, {
-		name: "ignored_empty",
-		body: getConfigResp{
-			Enabled:  aghalg.NBTrue,
-			Interval: float64(minIvl.Milliseconds()),
-			Ignored: []string{
-				"",
-			},
-		},
-		wantCode: http.StatusUnprocessableEntity,
-		wantErr:  "ignored: at index 0: hostname is empty\n",
 	}, {
 		name: "enabled_is_null",
 		body: getConfigResp{

internal/stats/stats.go

@@ -12,9 +12,9 @@ import (
 	"time"
 
 	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
-	"github.com/AdguardTeam/golibs/stringutil"
 	"github.com/AdguardTeam/golibs/timeutil"
 	"go.etcd.io/bbolt"
 )
@@ -58,8 +58,9 @@ type Config struct {
 	// endpoints.
 	HTTPRegister aghhttp.RegisterFunc
 
-	// Ignored is the list of host names, which should not be counted.
+	// Ignored contains the list of host names, which should not be counted,
-	Ignored *stringutil.Set
+	// and matches them.
+	Ignored *aghnet.IgnoreEngine
 
 	// Filename is the name of the database file.
 	Filename string
@@ -117,8 +118,9 @@ type StatsCtx struct {
 	// confMu protects ignored, limit, and enabled.
 	confMu *sync.RWMutex
 
-	// ignored is the list of host names, which should not be counted.
+	// ignored contains the list of host names, which should not be counted,
-	ignored *stringutil.Set
+	// and matches them.
+	ignored *aghnet.IgnoreEngine
 
 	// shouldCountClient returns client's ignore setting.
 	shouldCountClient func([]string) bool
@@ -289,7 +291,7 @@ func (s *StatsCtx) WriteDiskConfig(dc *Config) {
 	s.confMu.RLock()
 	defer s.confMu.RUnlock()
 
-	dc.Ignored = s.ignored.Clone()
+	dc.Ignored = s.ignored
 	dc.Limit = s.limit
 	dc.Enabled = s.enabled
 }

internal/stats/stats_test.go

@@ -11,9 +11,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/AdGuardHome/internal/stats"
 	"github.com/AdguardTeam/golibs/netutil"
-	"github.com/AdguardTeam/golibs/stringutil"
 	"github.com/AdguardTeam/golibs/testutil"
 	"github.com/AdguardTeam/golibs/timeutil"
 	"github.com/miekg/dns"
@@ -215,13 +215,15 @@ func TestShouldCount(t *testing.T) {
 		ignored1 = "ignor.ed"
 		ignored2 = "ignored.to"
 	)
-	set := stringutil.NewSet(ignored1, ignored2)
+	ignored := []string{ignored1, ignored2}
+	engine, err := aghnet.NewIgnoreEngine(ignored)
+	require.NoError(t, err)
 
 	s, err := stats.New(stats.Config{
 		Enabled:  true,
 		Filename: filepath.Join(t.TempDir(), "stats.db"),
 		Limit:    timeutil.Day,
-		Ignored:  set,
+		Ignored:  engine,
 		ShouldCountClient: func(ids []string) (a bool) {
 			return ids[0] != "no_count"
 		},

internal/stats/unit.go

@@ -7,9 +7,9 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
-	"github.com/AdguardTeam/golibs/stringutil"
 	"go.etcd.io/bbolt"
 	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
@@ -370,7 +370,7 @@ type pairsGetter func(u *unitDB) (pairs []countPair)
 
 // topsCollector collects statistics about highest values from the given *unitDB
 // slice using pg to retrieve data.
-func topsCollector(units []*unitDB, max int, ignored *stringutil.Set, pg pairsGetter) []map[string]uint64 {
+func topsCollector(units []*unitDB, max int, ignored *aghnet.IgnoreEngine, pg pairsGetter) []map[string]uint64 {
 	m := map[string]uint64{}
 	for _, u := range units {
 		for _, cp := range pg(u) {