Pull request 2297: AG-20945-filter-storage

Squashed commit of the following:

commit 2611fd5781
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Nov 1 16:29:06 2024 +0300

    dnsforward: imp test

commit 5efcfda937
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Nov 1 15:54:18 2024 +0300

    rulelist: imp docs, tests

commit 7a759c4699
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Nov 1 14:36:08 2024 +0300

    all: add filtering storage; upd golibs

go.mod

@@ -5,7 +5,7 @@ go 1.23.2
 require (
 	// TODO(a.garipov):  Update when v0.73.3 is released.
 	github.com/AdguardTeam/dnsproxy v0.73.3-0.20241004151328-c7c7b977a2a3
-	github.com/AdguardTeam/golibs v0.30.0
+	github.com/AdguardTeam/golibs v0.30.1
 	github.com/AdguardTeam/urlfilter v0.20.0
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/ameshkov/dnscrypt/v2 v2.3.0

go.sum

@@ -1,7 +1,7 @@
 github.com/AdguardTeam/dnsproxy v0.73.3-0.20241004151328-c7c7b977a2a3 h1:IGXwBjdKDzUm007QzZyxSllMnkbdXe7K79x7JWcBW/E=
 github.com/AdguardTeam/dnsproxy v0.73.3-0.20241004151328-c7c7b977a2a3/go.mod h1:356iHROxo+SOdBVifp1MXEh6qHyydtzGCcsQMfx+ZVs=
-github.com/AdguardTeam/golibs v0.30.0 h1:3pTdW1B9GZgqARrA5BvmYlAaEG1zAHI/ReikCDxrhiE=
+github.com/AdguardTeam/golibs v0.30.1 h1:/yv7dq2h7WXw/jTDxkE3FP9zHerRT+i03PZRHJX4fPU=
-github.com/AdguardTeam/golibs v0.30.0/go.mod h1:vjw1OVZG6BYyoqGRY88U4LCJLOMfhBFhU0UJBdaSAuQ=
+github.com/AdguardTeam/golibs v0.30.1/go.mod h1:FkwcNQEJoGsgDGXcalrVa/4gWbE68KsmE2guXWtBQUE=
 github.com/AdguardTeam/urlfilter v0.20.0 h1:X32qiuVCVd8WDYCEsbdZKfXMzwdVqrdulamtUi4rmzs=
 github.com/AdguardTeam/urlfilter v0.20.0/go.mod h1:gjrywLTxfJh6JOkwi9SU+frhP7kVVEZ5exFGkR99qpk=
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=

internal/aghhttp/aghhttp.go

@@ -14,12 +14,6 @@ import (
 	"github.com/AdguardTeam/golibs/logutil/slogutil"
 )
 
-// HTTP scheme constants.
-const (
-	SchemeHTTP  = "http"
-	SchemeHTTPS = "https"
-)
-
 // RegisterFunc is the function that sets the handler to handle the URL for the
 // method.
 //

internal/dnsforward/dnsforward_test.go

@@ -592,6 +592,8 @@ func TestSafeSearch(t *testing.T) {
 				r, _, errExch := client.Exchange(req, addr)
 				if assert.NoError(c, errExch) {
 					once.Do(func() { reply = r })
+				} else {
+					t.Logf("got error: %v", errExch)
 				}
 			}, testTimeout*10, testTimeout)
 

internal/filtering/filter_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/AdguardTeam/golibs/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -33,7 +33,7 @@ func serveHTTPLocally(t *testing.T, h http.Handler) (urlStr string) {
 	require.IsType(t, (*net.TCPAddr)(nil), addr)
 
 	return (&url.URL{
-		Scheme: aghhttp.SchemeHTTP,
+		Scheme: urlutil.SchemeHTTP,
 		Host:   addr.String(),
 	}).String()
 }

internal/filtering/http.go

@@ -17,6 +17,7 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/miekg/dns"
 )
 
@@ -41,19 +42,14 @@ func (d *DNSFilter) validateFilterURL(urlStr string) (err error) {
 
 	u, err := url.ParseRequestURI(urlStr)
 	if err != nil {
-		// Don't wrap the error since it's informative enough as is.
+		// Don't wrap the error, because it's informative enough as is.
 		return err
 	}
 
-	if s := u.Scheme; s != aghhttp.SchemeHTTP && s != aghhttp.SchemeHTTPS {
+	err = urlutil.ValidateHTTPURL(u)
-		return &url.Error{
+	if err != nil {
-			Op:  "Check scheme",
+		// Don't wrap the error, because it's informative enough as is.
-			URL: urlStr,
+		return err
-			Err: fmt.Errorf("only %v allowed", []string{
-				aghhttp.SchemeHTTP,
-				aghhttp.SchemeHTTPS,
-			}),
-		}
 	}
 
 	return nil

internal/filtering/rulelist/engine.go

@@ -3,11 +3,12 @@ package rulelist
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"net/http"
 	"sync"
 
 	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/log"
+	"github.com/AdguardTeam/golibs/logutil/slogutil"
 	"github.com/AdguardTeam/urlfilter"
 	"github.com/AdguardTeam/urlfilter/filterlist"
 	"github.com/c2h5oh/datasize"
@@ -18,6 +19,9 @@ import (
 //
 // TODO(a.garipov): Merge with [TextEngine] in some way?
 type Engine struct {
+	// logger is used to log the operation of the engine and its refreshes.
+	logger *slog.Logger
+
 	// mu protects engine and storage.
 	//
 	// TODO(a.garipov): See if anything else should be protected.
@@ -29,8 +33,7 @@ type Engine struct {
 	// storage is the filtering-rule storage.  It is saved here to close it.
 	storage *filterlist.RuleStorage
 
-	// name is the human-readable name of the engine, like "allowed", "blocked",
+	// name is the human-readable name of the engine.
-	// or "custom".
 	name string
 
 	// filters is the data about rule filters in this engine.
@@ -40,12 +43,15 @@ type Engine struct {
 // EngineConfig is the configuration for rule-list filtering engines created by
 // combining refreshable filters.
 type EngineConfig struct {
-	// Name is the human-readable name of this engine, like "allowed",
+	// Logger is used to log the operation of the engine.  It must not be nil.
-	// "blocked", or "custom".
+	Logger *slog.Logger
+
+	// name is the human-readable name of the engine; see [EngineNameAllow] and
+	// similar constants.
 	Name string
 
 	// Filters is the data about rule lists in this engine.  There must be no
-	// other references to the elements of this slice.
+	// other references to the items of this slice.  Each item must not be nil.
 	Filters []*Filter
 }
 
@@ -53,6 +59,7 @@ type EngineConfig struct {
 // refreshed, so a refresh should be performed before use.
 func NewEngine(c *EngineConfig) (e *Engine) {
 	return &Engine{
+		logger:  c.Logger,
 		mu:      &sync.RWMutex{},
 		name:    c.Name,
 		filters: c.Filters,
@@ -85,7 +92,7 @@ func (e *Engine) FilterRequest(
 }
 
 // currentEngine returns the current filtering engine.
-func (e *Engine) currentEngine() (enging *urlfilter.DNSEngine) {
+func (e *Engine) currentEngine() (engine *urlfilter.DNSEngine) {
 	e.mu.RLock()
 	defer e.mu.RUnlock()
 
@@ -96,7 +103,7 @@ func (e *Engine) currentEngine() (enging *urlfilter.DNSEngine) {
 // parseBuf, cli, cacheDir, and maxSize are used for updates of rule-list
 // filters; see [Filter.Refresh].
 //
-// TODO(a.garipov): Unexport and test in an internal test or through enigne
+// TODO(a.garipov): Unexport and test in an internal test or through engine
 // tests.
 func (e *Engine) Refresh(
 	ctx context.Context,
@@ -115,20 +122,20 @@ func (e *Engine) Refresh(
 	}
 
 	if len(filtersToRefresh) == 0 {
-		log.Info("filtering: updating engine %q: no rule-list filters", e.name)
+		e.logger.InfoContext(ctx, "updating: no rule-list filters")
 
 		return nil
 	}
 
 	engRefr := &engineRefresh{
-		httpCli:    cli,
+		logger:   e.logger,
-		cacheDir:   cacheDir,
+		httpCli:  cli,
-		engineName: e.name,
+		cacheDir: cacheDir,
-		parseBuf:   parseBuf,
+		parseBuf: parseBuf,
-		maxSize:    maxSize,
+		maxSize:  maxSize,
 	}
 
-	ruleLists, errs := engRefr.process(ctx, e.filters)
+	ruleLists, errs := engRefr.process(ctx, filtersToRefresh)
 	if isOneTimeoutError(errs) {
 		// Don't wrap the error since it's informative enough as is.
 		return err
@@ -141,14 +148,14 @@ func (e *Engine) Refresh(
 		return errors.Join(errs...)
 	}
 
-	e.resetStorage(storage)
+	e.resetStorage(ctx, storage)
 
 	return errors.Join(errs...)
 }
 
 // resetStorage sets e.storage and e.engine and closes the previous storage.
 // Errors from closing the previous storage are logged.
-func (e *Engine) resetStorage(storage *filterlist.RuleStorage) {
+func (e *Engine) resetStorage(ctx context.Context, storage *filterlist.RuleStorage) {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
@@ -161,7 +168,7 @@ func (e *Engine) resetStorage(storage *filterlist.RuleStorage) {
 
 	err := prevStorage.Close()
 	if err != nil {
-		log.Error("filtering: engine %q: closing old storage: %s", e.name, err)
+		e.logger.WarnContext(ctx, "closing old storage", slogutil.KeyError, err)
 	}
 }
 
@@ -179,11 +186,11 @@ func isOneTimeoutError(errs []error) (ok bool) {
 
 // engineRefresh represents a single ongoing engine refresh.
 type engineRefresh struct {
-	httpCli    *http.Client
+	logger   *slog.Logger
-	cacheDir   string
+	httpCli  *http.Client
-	engineName string
+	cacheDir string
-	parseBuf   []byte
+	parseBuf []byte
-	maxSize    datasize.ByteSize
+	maxSize  datasize.ByteSize
 }
 
 // process runs updates of all given rule-list filters.  All errors are logged
@@ -216,12 +223,12 @@ func (r *engineRefresh) process(
 		errs = append(errs, err)
 
 		// Also log immediately, since the update can take a lot of time.
-		log.Error(
+		r.logger.ErrorContext(
-			"filtering: updating engine %q: rule list %s from url %q: %s\n",
+			ctx,
-			r.engineName,
+			"updating rule list",
-			f.uid,
+			"uid", f.uid,
-			f.url,
+			"url", f.url,
-			err,
+			slogutil.KeyError, err,
 		)
 	}
 
@@ -237,17 +244,17 @@ func (r *engineRefresh) processFilter(ctx context.Context, f *Filter) (err error
 	}
 
 	if prevChecksum == parseRes.Checksum {
-		log.Info("filtering: engine %q: filter %q: no change", r.engineName, f.uid)
+		r.logger.InfoContext(ctx, "no change in filter", "uid", f.uid)
 
 		return nil
 	}
 
-	log.Info(
+	r.logger.InfoContext(
-		"filtering: updated engine %q: filter %q: %d bytes, %d rules",
+		ctx,
-		r.engineName,
+		"filter updated",
-		f.uid,
+		"uid", f.uid,
-		parseRes.BytesWritten,
+		"bytes", parseRes.BytesWritten,
-		parseRes.RulesCount,
+		"rules", parseRes.RulesCount,
 	)
 
 	return nil

internal/filtering/rulelist/engine_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
+	"github.com/AdguardTeam/golibs/logutil/slogutil"
 	"github.com/AdguardTeam/golibs/testutil"
 	"github.com/AdguardTeam/urlfilter"
 	"github.com/miekg/dns"
@@ -13,6 +14,8 @@ import (
 )
 
 func TestEngine_Refresh(t *testing.T) {
+	t.Parallel()
+
 	cacheDir := t.TempDir()
 
 	fileURL, srvURL := newFilterLocations(t, cacheDir, testRuleTextBlocked, testRuleTextBlocked2)
@@ -21,6 +24,7 @@ func TestEngine_Refresh(t *testing.T) {
 	httpFlt := newFilter(t, srvURL, "HTTP Filter")
 
 	eng := rulelist.NewEngine(&rulelist.EngineConfig{
+		Logger:  slogutil.NewDiscardLogger(),
 		Name:    "Engine",
 		Filters: []*rulelist.Filter{fileFlt, httpFlt},
 	})

internal/filtering/rulelist/filter.go

@@ -105,7 +105,7 @@ func NewFilter(c *FilterConfig) (f *Filter, err error) {
 // buffer used to parse information from the data.  cli and maxSize are only
 // used when f is a URL-based list.
 //
-// TODO(a.garipov): Unexport and test in an internal test or through enigne
+// TODO(a.garipov): Unexport and test in an internal test or through engine
 // tests.
 //
 // TODO(a.garipov): Consider not returning parseRes.

internal/filtering/rulelist/filter_test.go

@@ -8,12 +8,15 @@ import (
 	"testing"
 
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/AdguardTeam/golibs/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestFilter_Refresh(t *testing.T) {
+	t.Parallel()
+
 	cacheDir := t.TempDir()
 	uid := rulelist.MustNewUID()
 
@@ -37,7 +40,7 @@ func TestFilter_Refresh(t *testing.T) {
 	}, {
 		name: "file",
 		url: &url.URL{
-			Scheme: "file",
+			Scheme: urlutil.SchemeFile,
 			Path:   fileURL.Path,
 		},
 		wantNewErrMsg: "",
@@ -49,6 +52,8 @@ func TestFilter_Refresh(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
 			f, err := rulelist.NewFilter(&rulelist.FilterConfig{
 				URL:         tc.url,
 				Name:        tc.name,

internal/filtering/rulelist/rulelist.go

@@ -71,3 +71,10 @@ var _ fmt.Stringer = UID{}
 func (id UID) String() (s string) {
 	return uuid.UUID(id).String()
 }
+
+// Common engine names.
+const (
+	EngineNameAllow  = "allow"
+	EngineNameBlock  = "block"
+	EngineNameCustom = "custom"
+)

internal/filtering/rulelist/rulelist_test.go

@@ -6,20 +6,16 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
-	"path/filepath"
 	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/AdguardTeam/golibs/testutil"
 	"github.com/stretchr/testify/require"
 )
 
-func TestMain(m *testing.M) {
-	testutil.DiscardLogOutput(m)
-}
-
 // testTimeout is the common timeout for tests.
 const testTimeout = 1 * time.Second
 
@@ -31,6 +27,7 @@ const testTitle = "Test Title"
 
 // Common rule texts for tests.
 const (
+	testRuleTextAllowed     = "||allowed.example^\n"
 	testRuleTextBadTab      = "||bad-tab-and-comment.example^\t# A comment.\n"
 	testRuleTextBlocked     = "||blocked.example^\n"
 	testRuleTextBlocked2    = "||blocked-2.example^\n"
@@ -79,8 +76,16 @@ func newFilterLocations(
 	fileData string,
 	httpData string,
 ) (fileURL, srvURL *url.URL) {
-	filePath := filepath.Join(cacheDir, "initial.txt")
+	t.Helper()
-	err := os.WriteFile(filePath, []byte(fileData), 0o644)
+
+	f, err := os.CreateTemp(cacheDir, "")
+	require.NoError(t, err)
+
+	err = f.Close()
+	require.NoError(t, err)
+
+	filePath := f.Name()
+	err = os.WriteFile(filePath, []byte(fileData), 0o644)
 	require.NoError(t, err)
 
 	testutil.CleanupAndRequireSuccess(t, func() (err error) {
@@ -88,7 +93,7 @@ func newFilterLocations(
 	})
 
 	fileURL = &url.URL{
-		Scheme: "file",
+		Scheme: urlutil.SchemeFile,
 		Path:   filePath,
 	}
 

internal/filtering/rulelist/storage.go

@@ -0,0 +1,112 @@
+package rulelist
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"sync"
+
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/c2h5oh/datasize"
+)
+
+// Storage contains the main filtering engines, including the allowlist, the
+// blocklist, and the user's custom filtering rules.
+type Storage struct {
+	// refreshMu makes sure that only one update takes place at a time.
+	refreshMu *sync.Mutex
+
+	allow    *Engine
+	block    *Engine
+	custom   *TextEngine
+	httpCli  *http.Client
+	cacheDir string
+	parseBuf []byte
+	maxSize  datasize.ByteSize
+}
+
+// StorageConfig is the configuration for the filtering-engine storage.
+type StorageConfig struct {
+	// Logger is used to log the operation of the storage.  It must not be nil.
+	Logger *slog.Logger
+
+	// HTTPClient is the HTTP client used to perform updates of rule lists.
+	// It must not be nil.
+	HTTPClient *http.Client
+
+	// CacheDir is the path to the directory used to cache rule-list files.
+	// It must be set.
+	CacheDir string
+
+	// AllowFilters are the filtering-rule lists used to exclude domain names
+	// from the filtering.  Each item must not be nil.
+	AllowFilters []*Filter
+
+	// BlockFilters are the filtering-rule lists used to block domain names.
+	// Each item must not be nil.
+	BlockFilters []*Filter
+
+	// CustomRules contains custom rules of the user.  They have priority over
+	// both allow- and blacklist rules.
+	CustomRules []string
+
+	// MaxRuleListTextSize is the maximum size of a rule-list file.  It must be
+	// greater than zero.
+	MaxRuleListTextSize datasize.ByteSize
+}
+
+// NewStorage creates a new filtering-engine storage.  The engines are not
+// refreshed, so a refresh should be performed before use.
+func NewStorage(c *StorageConfig) (s *Storage, err error) {
+	custom, err := NewTextEngine(&TextEngineConfig{
+		Name:  EngineNameCustom,
+		Rules: c.CustomRules,
+		ID:    URLFilterIDCustom,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("creating custom engine: %w", err)
+	}
+
+	return &Storage{
+		refreshMu: &sync.Mutex{},
+		allow: NewEngine(&EngineConfig{
+			Logger:  c.Logger.With("engine", EngineNameAllow),
+			Name:    EngineNameAllow,
+			Filters: c.AllowFilters,
+		}),
+		block: NewEngine(&EngineConfig{
+			Logger:  c.Logger.With("engine", EngineNameBlock),
+			Name:    EngineNameBlock,
+			Filters: c.BlockFilters,
+		}),
+		custom:   custom,
+		httpCli:  c.HTTPClient,
+		cacheDir: c.CacheDir,
+		parseBuf: make([]byte, DefaultRuleBufSize),
+		maxSize:  c.MaxRuleListTextSize,
+	}, nil
+}
+
+// Close closes the underlying rule-list engines.
+func (s *Storage) Close() (err error) {
+	// Don't wrap the errors since they are informative enough as is.
+	return errors.Join(
+		s.allow.Close(),
+		s.block.Close(),
+	)
+}
+
+// Refresh updates all engines in s.
+//
+// TODO(a.garipov): Refresh allow and block separately?
+func (s *Storage) Refresh(ctx context.Context) (err error) {
+	s.refreshMu.Lock()
+	defer s.refreshMu.Unlock()
+
+	// Don't wrap the errors since they are informative enough as is.
+	return errors.Join(
+		s.allow.Refresh(ctx, s.parseBuf, s.httpCli, s.cacheDir, s.maxSize),
+		s.block.Refresh(ctx, s.parseBuf, s.httpCli, s.cacheDir, s.maxSize),
+	)
+}

internal/filtering/rulelist/storage_test.go

@@ -0,0 +1,49 @@
+package rulelist_test
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
+	"github.com/AdguardTeam/golibs/logutil/slogutil"
+	"github.com/AdguardTeam/golibs/testutil"
+	"github.com/c2h5oh/datasize"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStorage_Refresh(t *testing.T) {
+	t.Parallel()
+
+	cacheDir := t.TempDir()
+
+	allowedFileURL, _ := newFilterLocations(t, cacheDir, testRuleTextAllowed, "")
+	allowedFlt := newFilter(t, allowedFileURL, "Allowed 1")
+
+	blockedFileURL, _ := newFilterLocations(t, cacheDir, testRuleTextBlocked, "")
+	blockedFlt := newFilter(t, blockedFileURL, "Blocked 1")
+
+	strg, err := rulelist.NewStorage(&rulelist.StorageConfig{
+		Logger: slogutil.NewDiscardLogger(),
+		HTTPClient: &http.Client{
+			Timeout: testTimeout,
+		},
+		CacheDir: cacheDir,
+		AllowFilters: []*rulelist.Filter{
+			allowedFlt,
+		},
+		BlockFilters: []*rulelist.Filter{
+			blockedFlt,
+		},
+		CustomRules: []string{
+			testRuleTextBlocked2,
+		},
+		MaxRuleListTextSize: 1 * datasize.KB,
+	})
+	require.NoError(t, err)
+	testutil.CleanupAndRequireSuccess(t, strg.Close)
+
+	ctx := testutil.ContextWithTimeout(t, testTimeout)
+	err = strg.Refresh(ctx)
+	assert.NoError(t, err)
+}

internal/filtering/rulelist/textengine.go

@@ -20,15 +20,15 @@ type TextEngine struct {
 	// storage is the filtering-rule storage.  It is saved here to close it.
 	storage *filterlist.RuleStorage
 
-	// name is the human-readable name of the engine, like "custom".
+	// name is the human-readable name of the engine.
 	name string
 }
 
 // TextEngineConfig is the configuration for a rule-list filtering engine
 // created from a filtering rule text.
 type TextEngineConfig struct {
-	// Name is the human-readable name of this engine, like "allowed",
+	// name is the human-readable name of the engine; see [EngineNameAllow] and
-	// "blocked", or "custom".
+	// similar constants.
 	Name string
 
 	// Rules is the text of the filtering rules for this engine.

internal/filtering/rulelist/textengine_test.go

@@ -12,6 +12,8 @@ import (
 )
 
 func TestNewTextEngine(t *testing.T) {
+	t.Parallel()
+
 	eng, err := rulelist.NewTextEngine(&rulelist.TextEngineConfig{
 		Name: "RulesEngine",
 		Rules: []string{

internal/home/control.go

@@ -16,6 +16,7 @@ import (
 	"github.com/AdguardTeam/golibs/httphdr"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/netutil"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/NYTimes/gziphandler"
 )
 
@@ -376,7 +377,7 @@ func handleHTTPSRedirect(w http.ResponseWriter, r *http.Request) (proceed bool)
 	//
 	// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin.
 	originURL := &url.URL{
-		Scheme: aghhttp.SchemeHTTP,
+		Scheme: urlutil.SchemeHTTP,
 		Host:   r.Host,
 	}
 
@@ -395,7 +396,7 @@ func httpsURL(u *url.URL, host string, portHTTPS uint16) (redirectURL *url.URL)
 	}
 
 	return &url.URL{
-		Scheme:   aghhttp.SchemeHTTPS,
+		Scheme:   urlutil.SchemeHTTPS,
 		Host:     hostPort,
 		Path:     u.Path,
 		RawQuery: u.RawQuery,

internal/home/dns.go

@@ -23,6 +23,7 @@ import (
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/logutil/slogutil"
 	"github.com/AdguardTeam/golibs/netutil"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/ameshkov/dnscrypt/v2"
 	yaml "gopkg.in/yaml.v3"
 )
@@ -371,7 +372,7 @@ func getDNSEncryption() (de dnsEncryption) {
 		}
 
 		de.https = (&url.URL{
-			Scheme: "https",
+			Scheme: urlutil.SchemeHTTPS,
 			Host:   addr,
 			Path:   "/dns-query",
 		}).String()

internal/home/home.go

@@ -21,7 +21,6 @@ import (
 	"time"
 
 	"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
-	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
 	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/AdGuardHome/internal/aghos"
 	"github.com/AdguardTeam/AdGuardHome/internal/aghtls"
@@ -42,6 +41,7 @@ import (
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/logutil/slogutil"
 	"github.com/AdguardTeam/golibs/netutil"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/AdguardTeam/golibs/osutil"
 )
 
@@ -605,7 +605,7 @@ func run(opts options, clientBuildFS fs.FS, done chan struct{}) {
 	fatalOnError(errors.Annotate(err, "getting executable path: %w"))
 
 	u := &url.URL{
-		Scheme: "https",
+		Scheme: urlutil.SchemeHTTPS,
 		// TODO(a.garipov): Make configurable.
 		Host: "static.adtidy.org",
 		Path: path.Join("adguardhome", version.Channel(), "version.json"),
@@ -936,12 +936,12 @@ func printHTTPAddresses(proto string) {
 	}
 
 	port := config.HTTPConfig.Address.Port()
-	if proto == aghhttp.SchemeHTTPS {
+	if proto == urlutil.SchemeHTTPS {
 		port = tlsConf.PortHTTPS
 	}
 
 	// TODO(e.burkov): Inspect and perhaps merge with the previous condition.
-	if proto == aghhttp.SchemeHTTPS && tlsConf.ServerName != "" {
+	if proto == urlutil.SchemeHTTPS && tlsConf.ServerName != "" {
 		printWebAddrs(proto, tlsConf.ServerName, tlsConf.PortHTTPS)
 
 		return

internal/home/mobileconfig.go

@@ -8,11 +8,11 @@ import (
 	"net/url"
 	"path"
 
-	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
 	"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/httphdr"
 	"github.com/AdguardTeam/golibs/log"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/google/uuid"
 	"howett.net/plist"
 )
@@ -84,7 +84,7 @@ func encodeMobileConfig(d *dnsSettings, clientID string) ([]byte, error) {
 	case dnsProtoHTTPS:
 		dspName = fmt.Sprintf("%s DoH", d.ServerName)
 		u := &url.URL{
-			Scheme: aghhttp.SchemeHTTPS,
+			Scheme: urlutil.SchemeHTTPS,
 			Host:   d.ServerName,
 			Path:   path.Join("/dns-query", clientID),
 		}

internal/home/service.go

@@ -10,11 +10,11 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
 	"github.com/AdguardTeam/AdGuardHome/internal/aghos"
 	"github.com/AdguardTeam/AdGuardHome/internal/version"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/kardianos/service"
 )
 
@@ -336,7 +336,7 @@ AdGuard Home is successfully installed and will automatically start on boot.
 There are a few more things that must be configured before you can use it.
 Click on the link below and follow the Installation Wizard steps to finish setup.
 AdGuard Home is now available at the following addresses:`)
-		printHTTPAddresses(aghhttp.SchemeHTTP)
+		printHTTPAddresses(urlutil.SchemeHTTP)
 	}
 }
 

internal/home/web.go

@@ -11,13 +11,13 @@ import (
 	"sync"
 	"time"
 
-	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
 	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/AdGuardHome/internal/updater"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/netutil"
 	"github.com/AdguardTeam/golibs/netutil/httputil"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/NYTimes/gziphandler"
 	"github.com/quic-go/quic-go/http3"
 	"golang.org/x/net/http2"
@@ -192,7 +192,7 @@ func (web *webAPI) start() {
 
 	// this loop is used as an ability to change listening host and/or port
 	for !web.httpsServer.inShutdown {
-		printHTTPAddresses(aghhttp.SchemeHTTP)
+		printHTTPAddresses(urlutil.SchemeHTTP)
 		errs := make(chan error, 2)
 
 		// Use an h2c handler to support unencrypted HTTP/2, e.g. for proxies.
@@ -286,7 +286,7 @@ func (web *webAPI) tlsServerLoop() {
 			WriteTimeout:      web.conf.WriteTimeout,
 		}
 
-		printHTTPAddresses(aghhttp.SchemeHTTPS)
+		printHTTPAddresses(urlutil.SchemeHTTPS)
 
 		if web.conf.serveHTTP3 {
 			go web.mustStartHTTP3(addr)

internal/next/websvc/dns_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/internal/next/agh"
 	"github.com/AdguardTeam/AdGuardHome/internal/next/dnssvc"
 	"github.com/AdguardTeam/AdGuardHome/internal/next/websvc"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -49,7 +50,7 @@ func TestService_HandlePatchSettingsDNS(t *testing.T) {
 
 	_, addr := newTestServer(t, confMgr)
 	u := &url.URL{
-		Scheme: "http",
+		Scheme: urlutil.SchemeHTTP,
 		Host:   addr.String(),
 		Path:   websvc.PathV1SettingsDNS,
 	}

internal/next/websvc/http_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
 	"github.com/AdguardTeam/AdGuardHome/internal/next/agh"
 	"github.com/AdguardTeam/AdGuardHome/internal/next/websvc"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -45,7 +46,7 @@ func TestService_HandlePatchSettingsHTTP(t *testing.T) {
 
 	_, addr := newTestServer(t, confMgr)
 	u := &url.URL{
-		Scheme: "http",
+		Scheme: urlutil.SchemeHTTP,
 		Host:   addr.String(),
 		Path:   websvc.PathV1SettingsHTTP,
 	}

internal/next/websvc/settings_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/internal/next/agh"
 	"github.com/AdguardTeam/AdGuardHome/internal/next/dnssvc"
 	"github.com/AdguardTeam/AdGuardHome/internal/next/websvc"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -69,7 +70,7 @@ func TestService_HandleGetSettingsAll(t *testing.T) {
 
 	_, addr := newTestServer(t, confMgr)
 	u := &url.URL{
-		Scheme: "http",
+		Scheme: urlutil.SchemeHTTP,
 		Host:   addr.String(),
 		Path:   websvc.PathV1SettingsAll,
 	}

internal/next/websvc/system_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/AdguardTeam/AdGuardHome/internal/next/websvc"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -17,7 +18,7 @@ func TestService_handleGetV1SystemInfo(t *testing.T) {
 	confMgr := newConfigManager()
 	_, addr := newTestServer(t, confMgr)
 	u := &url.URL{
-		Scheme: "http",
+		Scheme: urlutil.SchemeHTTP,
 		Host:   addr.String(),
 		Path:   websvc.PathV1SystemInfo,
 	}

internal/next/websvc/websvc_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/internal/next/agh"
 	"github.com/AdguardTeam/AdGuardHome/internal/next/dnssvc"
 	"github.com/AdguardTeam/AdGuardHome/internal/next/websvc"
+	"github.com/AdguardTeam/golibs/netutil/urlutil"
 	"github.com/AdguardTeam/golibs/testutil"
 	"github.com/AdguardTeam/golibs/testutil/fakefs"
 	"github.com/stretchr/testify/assert"
@@ -181,7 +182,7 @@ func TestService_Start_getHealthCheck(t *testing.T) {
 	confMgr := newConfigManager()
 	_, addr := newTestServer(t, confMgr)
 	u := &url.URL{
-		Scheme: "http",
+		Scheme: urlutil.SchemeHTTP,
 		Host:   addr.String(),
 		Path:   websvc.PathHealthCheck,
 	}