Pull request: custom upstreams cache

Merge in DNS/adguard-home from custom-ups-cache to master

Squashed commit of the following:

commit 98428a87520f70cb522701d8eccfe4c529be1e40
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 17 10:53:32 2023 +0200

    all: upd dep

commit 775a639af4a2a45220b17e8b0037edc126ff62e4
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 17 09:52:31 2023 +0200

    dnsforward: imp test

commit e9e2a58b48e8588dfcb28df319d4651e1fe77af5
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 17 09:44:46 2023 +0200

    docs: changelog

commit a6d67218f037c8fec29e5fa2967476d63c3cfc32
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 17 09:37:17 2023 +0200

    all: upd dep

commit b101ff6e0cf393dacdee6fb68d33ba8f11c36280
Merge: d61f4eb88 8bb1aad73
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Nov 16 15:54:05 2023 +0200

    Merge remote-tracking branch 'origin/master' into custom-ups-cache

commit d61f4eb8871f8ae8504259998bf9015b29001cfb
Merge: 567a8a4af fdf60eeed
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon Nov 13 10:32:22 2023 +0200

    Merge remote-tracking branch 'origin/master' into custom-ups-cache

commit 567a8a4af34ad001d0e6d7d2efdc123205569e8c
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon Nov 13 10:30:24 2023 +0200

    home: imp code

commit a3c16facbebc166e5c0c731c1e892b61c0950d9e
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 10 14:34:04 2023 +0200

    all: imp code

commit 84160eafee1d0f2d0cd3f025f2d5070e4f597ad6
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 10 14:31:26 2023 +0200

    all: conf custom ups cache

commit b7f6581901ebad96c87e765a305a1fa5b336efbb
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 10 14:29:47 2023 +0200

    all: conf custom ups cache

commit d07df945d4e7614a679ef5dc77756096abf1e66c
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 10 09:26:29 2023 +0200

    all: docs

commit 998124bac08889c7d354dd1a099929726725bccc
Merge: f665e2f85 53170d871
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 10 09:24:28 2023 +0200

    Merge remote-tracking branch 'origin/master' into custom-ups-cache

commit f665e2f85bce12d95f80aba6614b6bfd4874b122
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 10 09:22:46 2023 +0200

    all: conf custom ups cache

commit a4b26973bef4f3b339198ffbe52a50baca303daf
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Nov 9 12:46:39 2023 +0200

    all: conf custom ups cache

CHANGELOG.md

@@ -27,6 +27,12 @@ NOTE: Add new changes BELOW THIS COMMENT.
 
 - Ability to specify rate limiting settings in the Web UI ([#6369]).
 
+#### Configuration changes
+
+- Added new properties `clients.persistent.upstreams_cache_enabled` and
+  `clients.persistent.upstreams_cache_size` which describe cache configuration
+  for each client custom upstream configuration.
+
 ### Fixed
 
 - Pre-filling the New static lease window with data ([#6402]).

go.mod

@@ -3,7 +3,7 @@ module github.com/AdguardTeam/AdGuardHome
 go 1.20
 
 require (
-	github.com/AdguardTeam/dnsproxy v0.58.0
+	github.com/AdguardTeam/dnsproxy v0.59.0
 	github.com/AdguardTeam/golibs v0.17.2
 	github.com/AdguardTeam/urlfilter v0.17.3
 	github.com/NYTimes/gziphandler v1.1.1
@@ -48,6 +48,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect
+	github.com/jessevdk/go-flags v1.5.0 // indirect
 	github.com/mdlayher/socket v0.5.0 // indirect
 	github.com/onsi/ginkgo/v2 v2.13.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect

go.sum

@@ -1,5 +1,9 @@
 github.com/AdguardTeam/dnsproxy v0.58.0 h1:1zPmDYWIc60D5Mn2idt3TcH+CQzKBvkWzJ5/u49wraw=
 github.com/AdguardTeam/dnsproxy v0.58.0/go.mod h1:ZvkbM71HwpilgkCnTubDiR4Ba6x5Qvnhy2iasMWaTDM=
+github.com/AdguardTeam/dnsproxy v0.58.1-0.20231114082217-26ee425d7cb9 h1:8JLgm92qlmd9LHh9maxo1Ao+vGLvLRSOClYH2/IvJ2g=
+github.com/AdguardTeam/dnsproxy v0.58.1-0.20231114082217-26ee425d7cb9/go.mod h1:ZvkbM71HwpilgkCnTubDiR4Ba6x5Qvnhy2iasMWaTDM=
+github.com/AdguardTeam/dnsproxy v0.59.0 h1:J0Iw4+GZ7U44scgkrNCM5W2G+tNS0r9nfQOuSLBqapE=
+github.com/AdguardTeam/dnsproxy v0.59.0/go.mod h1:ZvkbM71HwpilgkCnTubDiR4Ba6x5Qvnhy2iasMWaTDM=
 github.com/AdguardTeam/golibs v0.17.2 h1:vg6wHMjUKscnyPGRvxS5kAt7Uw4YxcJiITZliZ476W8=
 github.com/AdguardTeam/golibs v0.17.2/go.mod h1:DKhCIXHcUYtBhU8ibTLKh1paUL96n5zhQBlx763sj+U=
 github.com/AdguardTeam/urlfilter v0.17.3 h1:fg/ObbnO0Cv6aw0tW6N/ETDMhhNvmcUUOZ7HlmKC3rw=
@@ -52,6 +56,8 @@ github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8
 github.com/insomniacslk/dhcp v0.0.0-20231016090811-6a2c8fbdcc1c h1:PgxFEySCI41sH0mB7/2XswdXbUykQsRUGod8Rn+NubM=
 github.com/insomniacslk/dhcp v0.0.0-20231016090811-6a2c8fbdcc1c/go.mod h1:3A9PQ1cunSDF/1rbTq99Ts4pVnycWg+vlPkfeD2NLFI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 h1:elKwZS1OcdQ0WwEDBeqxKwb7WB62QX8bvZ/FJnVXIfk=
@@ -100,6 +106,7 @@ github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0
 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
@@ -145,6 +152,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

internal/aghtest/interface.go

@@ -125,7 +125,7 @@ type ClientsContainer struct {
 	OnUpstreamConfigByID func(
 		id string,
 		boot upstream.Resolver,
-	) (conf *proxy.UpstreamConfig, err error)
+	) (conf *proxy.CustomUpstreamConfig, err error)
 }
 
 // UpstreamConfigByID implements the [dnsforward.ClientsContainer] interface
@@ -133,7 +133,7 @@ type ClientsContainer struct {
 func (c *ClientsContainer) UpstreamConfigByID(
 	id string,
 	boot upstream.Resolver,
-) (conf *proxy.UpstreamConfig, err error) {
+) (conf *proxy.CustomUpstreamConfig, err error) {
 	return c.OnUpstreamConfigByID(id, boot)
 }
 

internal/dnsforward/config.go

@@ -34,7 +34,10 @@ type ClientsContainer interface {
 	// returns nil if there is no custom upstream configuration for the client.
 	// The id is expected to be either a string representation of an IP address
 	// or the ClientID.
-	UpstreamConfigByID(id string, boot upstream.Resolver) (conf *proxy.UpstreamConfig, err error)
+	UpstreamConfigByID(
+		id string,
+		boot upstream.Resolver,
+	) (conf *proxy.CustomUpstreamConfig, err error)
 }
 
 // Config represents the DNS filtering configuration of AdGuard Home. The zero

internal/dnsforward/dnsforward_test.go

@@ -644,10 +644,15 @@ func TestBlockedRequest(t *testing.T) {
 }
 
 func TestServerCustomClientUpstream(t *testing.T) {
+	const defaultCacheSize = 1024 * 1024
+
+	var upsCalledCounter uint32
+
 	forwardConf := ServerConfig{
 		UDPListenAddrs: []*net.UDPAddr{{}},
 		TCPListenAddrs: []*net.TCPAddr{{}},
 		Config: Config{
+			CacheSize: defaultCacheSize,
 			EDNSClientSubnet: &EDNSClientSubnet{
 				Enabled: false,
 			},
@@ -658,34 +663,51 @@ func TestServerCustomClientUpstream(t *testing.T) {
 	}, forwardConf, nil)
 
 	ups := aghtest.NewUpstreamMock(func(req *dns.Msg) (resp *dns.Msg, err error) {
+		atomic.AddUint32(&upsCalledCounter, 1)
+
 		return aghalg.Coalesce(
 			aghtest.MatchedResponse(req, dns.TypeA, "host", "192.168.0.1"),
 			new(dns.Msg).SetRcode(req, dns.RcodeNameError),
 		), nil
 	})
+
+	customUpsConf := proxy.NewCustomUpstreamConfig(
+		&proxy.UpstreamConfig{
+			Upstreams: []upstream.Upstream{ups},
+		},
+		true,
+		defaultCacheSize,
+		forwardConf.EDNSClientSubnet.Enabled,
+	)
+
 	s.conf.ClientsContainer = &aghtest.ClientsContainer{
 		OnUpstreamConfigByID: func(
 			_ string,
 			_ upstream.Resolver,
-		) (conf *proxy.UpstreamConfig, err error) {
+		) (conf *proxy.CustomUpstreamConfig, err error) {
-			return &proxy.UpstreamConfig{Upstreams: []upstream.Upstream{ups}}, nil
+			return customUpsConf, nil
 		},
 	}
+
 	startDeferStop(t, s)
 
-	addr := s.dnsProxy.Addr(proxy.ProtoUDP)
+	addr := s.dnsProxy.Addr(proxy.ProtoUDP).String()
 
 	// Send test request.
 	req := createTestMessage("host.")
 
-	reply, err := dns.Exchange(req, addr.String())
+	reply, err := dns.Exchange(req, addr)
 	require.NoError(t, err)
+	require.NotEmpty(t, reply.Answer)
+	require.Len(t, reply.Answer, 1)
 
 	assert.Equal(t, dns.RcodeSuccess, reply.Rcode)
-	require.NotEmpty(t, reply.Answer)
-
-	require.Len(t, reply.Answer, 1)
 	assert.Equal(t, net.IP{192, 168, 0, 1}, reply.Answer[0].(*dns.A).A)
+	assert.Equal(t, uint32(1), atomic.LoadUint32(&upsCalledCounter))
+
+	_, err = dns.Exchange(req, addr)
+	require.NoError(t, err)
+	assert.Equal(t, uint32(1), atomic.LoadUint32(&upsCalledCounter))
 }
 
 // testCNAMEs is a map of names and CNAMEs necessary for the TestUpstream work.

internal/home/client.go

@@ -14,11 +14,11 @@ import (
 
 // Client contains information about persistent clients.
 type Client struct {
-	// upstreamConfig is the custom upstream config for this client.  If
+	// upstreamConfig is the custom upstream configuration for this client.  If
-	// it's nil, it has not been initialized yet.  If it's non-nil and
+	// it's nil, it has not been initialized yet.  If it's non-nil and empty,
-	// empty, there are no valid upstreams.  If it's non-nil and non-empty,
+	// there are no valid upstreams.  If it's non-nil and non-empty, these
-	// these upstream must be used.
+	// upstream must be used.
-	upstreamConfig *proxy.UpstreamConfig
+	upstreamConfig *proxy.CustomUpstreamConfig
 
 	safeSearchConf filtering.SafeSearchConfig
 	SafeSearch     filtering.SafeSearch
@@ -32,6 +32,9 @@ type Client struct {
 	Tags      []string
 	Upstreams []string
 
+	UpstreamsCacheSize    uint32
+	UpstreamsCacheEnabled bool
+
 	UseOwnSettings        bool
 	FilteringEnabled      bool
 	SafeBrowsingEnabled   bool
@@ -57,8 +60,7 @@ func (c *Client) ShallowClone() (sh *Client) {
 // closeUpstreams closes the client-specific upstream config of c if any.
 func (c *Client) closeUpstreams() (err error) {
 	if c.upstreamConfig != nil {
-		err = c.upstreamConfig.Close()
+		if err = c.upstreamConfig.Close(); err != nil {
-		if err != nil {
 			return fmt.Errorf("closing upstreams of client %q: %w", c.Name, err)
 		}
 	}

internal/home/clients.go

@@ -185,6 +185,14 @@ type clientObject struct {
 	Tags      []string `yaml:"tags"`
 	Upstreams []string `yaml:"upstreams"`
 
+	// UpstreamsCacheSize is the DNS cache size (in bytes).
+	//
+	// TODO(d.kolyshev): Use [datasize.Bytesize].
+	UpstreamsCacheSize uint32 `yaml:"upstreams_cache_size"`
+
+	// UpstreamsCacheEnabled indicates if the DNS cache is enabled.
+	UpstreamsCacheEnabled bool `yaml:"upstreams_cache_enabled"`
+
 	UseGlobalSettings        bool `yaml:"use_global_settings"`
 	FilteringEnabled         bool `yaml:"filtering_enabled"`
 	ParentalEnabled          bool `yaml:"parental_enabled"`
@@ -216,6 +224,8 @@ func (clients *clientsContainer) addFromConfig(
 			UseOwnBlockedServices: !o.UseGlobalBlockedServices,
 			IgnoreQueryLog:        o.IgnoreQueryLog,
 			IgnoreStatistics:      o.IgnoreStatistics,
+			UpstreamsCacheEnabled: o.UpstreamsCacheEnabled,
+			UpstreamsCacheSize:    o.UpstreamsCacheSize,
 		}
 
 		if o.SafeSearchConf.Enabled {
@@ -429,11 +439,12 @@ func (clients *clientsContainer) shouldCountClient(ids []string) (y bool) {
 var _ dnsforward.ClientsContainer = (*clientsContainer)(nil)
 
 // UpstreamConfigByID implements the [dnsforward.ClientsContainer] interface for
-// *clientsContainer.
+// *clientsContainer.  upsConf is nil if the client isn't found or if the client
+// has no custom upstreams.
 func (clients *clientsContainer) UpstreamConfigByID(
 	id string,
 	bootstrap upstream.Resolver,
-) (upsConf *proxy.UpstreamConfig, err error) {
+) (conf *proxy.CustomUpstreamConfig, err error) {
 	clients.lock.Lock()
 	defer clients.lock.Unlock()
 
@@ -449,8 +460,8 @@ func (clients *clientsContainer) UpstreamConfigByID(
 		return nil, nil
 	}
 
-	var conf *proxy.UpstreamConfig
+	var upsConf *proxy.UpstreamConfig
-	conf, err = proxy.ParseUpstreamsConfig(
+	upsConf, err = proxy.ParseUpstreamsConfig(
 		upstreams,
 		&upstream.Options{
 			Bootstrap:    bootstrap,
@@ -464,6 +475,12 @@ func (clients *clientsContainer) UpstreamConfigByID(
 		return nil, err
 	}
 
+	conf = proxy.NewCustomUpstreamConfig(
+		upsConf,
+		c.UpstreamsCacheEnabled,
+		int(c.UpstreamsCacheSize),
+		config.DNS.EDNSClientSubnet.Enabled,
+	)
 	c.upstreamConfig = conf
 
 	return conf, nil

internal/home/clients_internal_test.go

@@ -355,13 +355,11 @@ func TestClientsCustomUpstream(t *testing.T) {
 	require.NoError(t, err)
 	assert.True(t, ok)
 
-	config, err := clients.UpstreamConfigByID("1.2.3.4", net.DefaultResolver)
+	upsConf, err := clients.UpstreamConfigByID("1.2.3.4", net.DefaultResolver)
-	assert.Nil(t, config)
+	assert.Nil(t, upsConf)
 	assert.NoError(t, err)
 
-	config, err = clients.UpstreamConfigByID("1.1.1.1", net.DefaultResolver)
+	upsConf, err = clients.UpstreamConfigByID("1.1.1.1", net.DefaultResolver)
-	require.NotNil(t, config)
+	require.NotNil(t, upsConf)
 	assert.NoError(t, err)
-	assert.Len(t, config.Upstreams, 1)
-	assert.Len(t, config.DomainReservedUpstreams, 1)
 }