Pull request: 2499 rewrite: storage vol.1

Merge in DNS/adguard-home from 2499-rewrites to master

Squashed commit of the following:

commit 3f5f8e1354cbfa2de2bea69b1caa5dfbcb84ddb1
Merge: c84a86fb fafd7a1e
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon Nov 28 13:13:31 2022 +0200

    Merge remote-tracking branch 'origin/master' into 2499-rewrites

commit c84a86fba1c9cd77c5893e056cd85f8aa6597afc
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon Nov 28 12:50:26 2022 +0200

    rewrite: todos

commit 3b33a79bea65650ee7dc920554773d1d1d2c67f5
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon Nov 28 11:45:05 2022 +0200

    rewrite: todos

commit 15022994e7af8e5ee5929edff8ce98356a1cb27b
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 25 12:15:27 2022 +0200

    rewrite: imp code

commit b3c1949a585a8ff83c046921288ce2fdb5b36cee
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 25 11:33:25 2022 +0200

    rewrite: imp code

commit 80fe50a86fc6e5ed5cf5b0d2e0be667b1b9221a8
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 25 11:13:39 2022 +0200

    rewrite: imp code

commit 5288ede0e8df5ca1d12c9e1e8341db703729c71a
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 25 11:04:11 2022 +0200

    Revert "all: rewrite"

    This reverts commit 32ad8d76861bd9919da61635e6f871adcc36c999.

commit cff6494fde44646f5ac54173380719557e0c0e02
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 25 11:04:11 2022 +0200

    Revert "all: rewrite"

    This reverts commit 65e44e92b009c561ddb2c68f56e04d57b83ba247.

commit e0fe877da4ee820838acc1ac6111809e7d0bc72e
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 25 11:04:11 2022 +0200

    Revert "filtering: imp code"

    This reverts commit c882da3309297f44ccaf38274ec4ef0ef2fec7b3.

commit 8e3f9d4a7ac2e5b114064e77ec009a2457b28a73
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 25 11:04:11 2022 +0200

    Revert "rewrite: imp code"

    This reverts commit ce2332932bef46186c0addf5e7a1ca648cdd9f22.

commit ce2332932bef46186c0addf5e7a1ca648cdd9f22
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Nov 25 10:56:48 2022 +0200

    rewrite: imp code

commit c882da3309297f44ccaf38274ec4ef0ef2fec7b3
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Nov 24 13:39:26 2022 +0200

    filtering: imp code

commit 65e44e92b009c561ddb2c68f56e04d57b83ba247
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Nov 24 13:25:10 2022 +0200

    all: rewrite

commit 32ad8d76861bd9919da61635e6f871adcc36c999
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Nov 24 13:19:55 2022 +0200

    all: rewrite

commit 941538abfe
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Tue Nov 22 12:54:55 2022 +0200

    rewrite: storage tests

commit 0a1ad86ea7
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Tue Nov 22 12:46:56 2022 +0200

    rewrite: imp code

commit f10a45361c
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon Nov 21 14:29:44 2022 +0200

    rewrite: storage

commit ff91bb81a6
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Sun Nov 20 13:25:05 2022 +0200

    rewrite: storage

internal/filtering/filtering.go

@@ -33,6 +33,7 @@ import (
 // The IDs of built-in filter lists.
 //
 // Keep in sync with client/src/helpers/constants.js.
+// TODO(d.kolyshev): Add RewritesListID and don't forget to keep in sync.
 const (
 	CustomListID = -iota
 	SysHostsListID

internal/filtering/rewrite/item.go

@@ -0,0 +1,69 @@
+package rewrite
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/miekg/dns"
+)
+
+// Item is a single DNS rewrite record.
+type Item struct {
+	// Domain is the domain pattern for which this rewrite should work.
+	Domain string `yaml:"domain"`
+
+	// Answer is the IP address, canonical name, or one of the special
+	// values: "A" or "AAAA".
+	Answer string `yaml:"answer"`
+}
+
+// equal returns true if rw is equal to other.
+func (rw *Item) equal(other *Item) (ok bool) {
+	if rw == nil {
+		return other == nil
+	} else if other == nil {
+		return false
+	}
+
+	return rw.Domain == other.Domain && rw.Answer == other.Answer
+}
+
+// toRule converts rw to a filter rule.
+func (rw *Item) toRule() (res string) {
+	domain := strings.ToLower(rw.Domain)
+
+	dType, exception := rw.rewriteParams()
+	dTypeKey := dns.TypeToString[dType]
+	if exception {
+		return fmt.Sprintf("@@||%s^$dnstype=%s,dnsrewrite", domain, dTypeKey)
+	}
+
+	return fmt.Sprintf("|%s^$dnsrewrite=NOERROR;%s;%s", domain, dTypeKey, rw.Answer)
+}
+
+// rewriteParams returns dns request type and exception flag for rw.
+func (rw *Item) rewriteParams() (dType uint16, exception bool) {
+	switch rw.Answer {
+	case "AAAA":
+		return dns.TypeAAAA, true
+	case "A":
+		return dns.TypeA, true
+	default:
+		// Go on.
+	}
+
+	ip := net.ParseIP(rw.Answer)
+	if ip == nil {
+		return dns.TypeCNAME, false
+	}
+
+	ip4 := ip.To4()
+	if ip4 != nil {
+		dType = dns.TypeA
+	} else {
+		dType = dns.TypeAAAA
+	}
+
+	return dType, false
+}

internal/filtering/rewrite/storage.go

@@ -0,0 +1,147 @@
+// Package rewrite implements DNS Rewrites storage and request matching.
+package rewrite
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/AdguardTeam/golibs/log"
+	"github.com/AdguardTeam/urlfilter"
+	"github.com/AdguardTeam/urlfilter/filterlist"
+	"golang.org/x/exp/slices"
+)
+
+// Storage is a storage for rewrite rules.
+type Storage interface {
+	// MatchRequest finds a matching rule for the specified request.
+	MatchRequest(dReq *urlfilter.DNSRequest) (res *urlfilter.DNSResult, matched bool)
+
+	// Add adds item to the storage.
+	Add(item *Item) (err error)
+
+	// Remove deletes item from the storage.
+	Remove(item *Item) (err error)
+
+	// List returns all items from the storage.
+	List() (items []*Item)
+}
+
+// DefaultStorage is the default storage for rewrite rules.
+type DefaultStorage struct {
+	// mu protects items.
+	mu *sync.RWMutex
+
+	// engine is the DNS filtering engine.
+	engine *urlfilter.DNSEngine
+
+	// ruleList is the filtering rule ruleList used by the engine.
+	ruleList filterlist.RuleList
+
+	// urlFilterID is the synthetic integer identifier for the urlfilter engine.
+	//
+	// TODO(a.garipov): Change the type to a string in module urlfilter and
+	// remove this crutch.
+	urlFilterID int
+
+	// rewrites stores the rewrite entries from configuration.
+	rewrites []*Item
+}
+
+// NewDefaultStorage returns new rewrites storage.  listID is used as an
+// identifier of the underlying rules list.  rewrites must not be nil.
+func NewDefaultStorage(listID int, rewrites []*Item) (s *DefaultStorage, err error) {
+	s = &DefaultStorage{
+		mu:          &sync.RWMutex{},
+		urlFilterID: listID,
+		rewrites:    rewrites,
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	err = s.resetRules()
+	if err != nil {
+		return nil, err
+	}
+
+	return s, nil
+}
+
+// type check
+var _ Storage = (*DefaultStorage)(nil)
+
+// MatchRequest implements the [Storage] interface for *DefaultStorage.
+func (s *DefaultStorage) MatchRequest(dReq *urlfilter.DNSRequest) (res *urlfilter.DNSResult, matched bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	return s.engine.MatchRequest(dReq)
+}
+
+// Add implements the [Storage] interface for *DefaultStorage.
+func (s *DefaultStorage) Add(item *Item) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// TODO(d.kolyshev): Handle duplicate items.
+	s.rewrites = append(s.rewrites, item)
+
+	return s.resetRules()
+}
+
+// Remove implements the [Storage] interface for *DefaultStorage.
+func (s *DefaultStorage) Remove(item *Item) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	arr := []*Item{}
+
+	// TODO(d.kolyshev): Use slices.IndexFunc + slices.Delete?
+	for _, ent := range s.rewrites {
+		if ent.equal(item) {
+			log.Debug("rewrite: removed element: %s -> %s", ent.Domain, ent.Answer)
+
+			continue
+		}
+
+		arr = append(arr, ent)
+	}
+	s.rewrites = arr
+
+	return s.resetRules()
+}
+
+// List implements the [Storage] interface for *DefaultStorage.
+func (s *DefaultStorage) List() (items []*Item) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	return slices.Clone(s.rewrites)
+}
+
+// resetRules resets the filtering rules.
+func (s *DefaultStorage) resetRules() (err error) {
+	var rulesText []string
+	for _, rewrite := range s.rewrites {
+		rulesText = append(rulesText, rewrite.toRule())
+	}
+
+	strList := &filterlist.StringRuleList{
+		ID:             s.urlFilterID,
+		RulesText:      strings.Join(rulesText, "\n"),
+		IgnoreCosmetic: true,
+	}
+
+	rs, err := filterlist.NewRuleStorage([]filterlist.RuleList{strList})
+	if err != nil {
+		return fmt.Errorf("creating list storage: %w", err)
+	}
+
+	s.ruleList = strList
+	s.engine = urlfilter.NewDNSEngine(rs)
+
+	log.Info("filter %d: reset %d rules", s.urlFilterID, s.engine.RulesCount)
+
+	return nil
+}

internal/filtering/rewrite/storage_test.go

@@ -0,0 +1,40 @@
+package rewrite
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewDefaultStorage(t *testing.T) {
+	items := []*Item{{
+		Domain: "example.com",
+		Answer: "answer.com",
+	}}
+
+	s, err := NewDefaultStorage(-1, items)
+	require.NoError(t, err)
+
+	require.Len(t, s.List(), 1)
+}
+
+func TestDefaultStorage_CRUD(t *testing.T) {
+	var items []*Item
+
+	s, err := NewDefaultStorage(-1, items)
+	require.NoError(t, err)
+	require.Len(t, s.List(), 0)
+
+	item := &Item{Domain: "example.com", Answer: "answer.com"}
+
+	err = s.Add(item)
+	require.NoError(t, err)
+
+	list := s.List()
+	require.Len(t, list, 1)
+	require.True(t, item.equal(list[0]))
+
+	err = s.Remove(item)
+	require.NoError(t, err)
+	require.Len(t, s.List(), 0)
+}

internal/filtering/rewrites.go

@@ -17,6 +17,8 @@ import (
 	"golang.org/x/exp/slices"
 )
 
+// TODO(d.kolyshev): Rename this file to rewritehttp.go.
+
 // LegacyRewrite is a single legacy DNS rewrite record.
 //
 // Instances of *LegacyRewrite must never be nil.

internal/filtering/rewrites_test.go

@@ -10,6 +10,7 @@ import (
 )
 
 // TODO(e.burkov): All the tests in this file may and should me merged together.
+// TODO(d.kolyshev): Move these tests to rewrite package.
 
 func TestRewrites(t *testing.T) {
 	d, _ := newForTest(t, nil, nil)