Pull request 1937: imp-filter-upd
Squashed commit of the following:
commit 6ce649c06398cf8a6f8e1a90f560fa8205f6500e
Merge: 1c6327e5d 996c6b3ee
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date: Tue Jul 25 17:42:01 2023 +0300
Merge branch 'master' into imp-filter-upd
commit 1c6327e5d4c04393abc5d4d3e4b8568d4c6eca23
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date: Fri Jul 21 17:32:47 2023 +0300
all: imp code; use renameio/v2 consistently
commit 1669288c9b662d1310f83a4e0d3f1f60731188cd
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date: Fri Jul 21 16:26:17 2023 +0300
all: add renameioutil; imp flt upd
This commit is contained in:
parent
996c6b3ee3
commit
698b963e11
|
@ -23,6 +23,10 @@ See also the [v0.107.35 GitHub milestone][ms-v0.107.35].
|
||||||
NOTE: Add new changes BELOW THIS COMMENT.
|
NOTE: Add new changes BELOW THIS COMMENT.
|
||||||
-->
|
-->
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- Improved reliability filtering-rule list updates on Unix systems.
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
- Occasional client information lookup failures that could lead to the DNS
|
- Occasional client information lookup failures that could lead to the DNS
|
||||||
|
|
2
go.mod
2
go.mod
|
@ -15,7 +15,7 @@ require (
|
||||||
github.com/go-ping/ping v1.1.0
|
github.com/go-ping/ping v1.1.0
|
||||||
github.com/google/go-cmp v0.5.9
|
github.com/google/go-cmp v0.5.9
|
||||||
github.com/google/gopacket v1.1.19
|
github.com/google/gopacket v1.1.19
|
||||||
github.com/google/renameio v1.0.1
|
github.com/google/renameio/v2 v2.0.0
|
||||||
github.com/google/uuid v1.3.0
|
github.com/google/uuid v1.3.0
|
||||||
github.com/insomniacslk/dhcp v0.0.0-20230612134759-b20c9ba983df
|
github.com/insomniacslk/dhcp v0.0.0-20230612134759-b20c9ba983df
|
||||||
github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86
|
github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86
|
||||||
|
|
4
go.sum
4
go.sum
|
@ -52,8 +52,8 @@ github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF
|
||||||
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
|
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
|
||||||
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs=
|
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs=
|
||||||
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
|
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
|
||||||
github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU=
|
github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
|
||||||
github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk=
|
github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
|
||||||
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
|
|
@ -14,7 +14,7 @@ import (
|
||||||
"github.com/AdguardTeam/golibs/errors"
|
"github.com/AdguardTeam/golibs/errors"
|
||||||
"github.com/AdguardTeam/golibs/log"
|
"github.com/AdguardTeam/golibs/log"
|
||||||
"github.com/AdguardTeam/golibs/stringutil"
|
"github.com/AdguardTeam/golibs/stringutil"
|
||||||
"github.com/google/renameio/maybe"
|
"github.com/google/renameio/v2/maybe"
|
||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,52 @@
|
||||||
|
// Package aghrenameio is a wrapper around package github.com/google/renameio/v2
|
||||||
|
// that provides a similar stream-based API for both Unix and Windows systems.
|
||||||
|
// While the Windows API is not technically atomic, it still provides a
|
||||||
|
// consistent stream-based interface, and atomic renames of files do not seem to
|
||||||
|
// be possible in all cases anyway.
|
||||||
|
//
|
||||||
|
// See https://github.com/google/renameio/issues/1.
|
||||||
|
//
|
||||||
|
// TODO(a.garipov): Consider moving to golibs/renameioutil once tried and
|
||||||
|
// tested.
|
||||||
|
package aghrenameio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
|
||||||
|
"github.com/AdguardTeam/golibs/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PendingFile is the interface for pending temporary files.
|
||||||
|
type PendingFile interface {
|
||||||
|
// Cleanup closes the file, and removes it without performing the renaming.
|
||||||
|
// To close and rename the file, use CloseReplace.
|
||||||
|
Cleanup() (err error)
|
||||||
|
|
||||||
|
// CloseReplace closes the temporary file and replaces the destination file
|
||||||
|
// with it, possibly atomically.
|
||||||
|
//
|
||||||
|
// This method is not safe for concurrent use by multiple goroutines.
|
||||||
|
CloseReplace() (err error)
|
||||||
|
|
||||||
|
// Write writes len(b) bytes from b to the File. It returns the number of
|
||||||
|
// bytes written and an error, if any. Write returns a non-nil error when n
|
||||||
|
// != len(b).
|
||||||
|
Write(b []byte) (n int, err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPendingFile is a wrapper around [renameio.NewPendingFile] on Unix systems
|
||||||
|
// and [os.CreateTemp] on Windows.
|
||||||
|
func NewPendingFile(filePath string, mode fs.FileMode) (f PendingFile, err error) {
|
||||||
|
return newPendingFile(filePath, mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithDeferredCleanup is a helper that performs the necessary cleanups and
|
||||||
|
// finalizations of the temporary files based on the returned error.
|
||||||
|
func WithDeferredCleanup(returned error, file PendingFile) (err error) {
|
||||||
|
// Make sure that any error returned from here is marked as a deferred one.
|
||||||
|
if returned != nil {
|
||||||
|
return errors.WithDeferred(returned, file.Cleanup())
|
||||||
|
}
|
||||||
|
|
||||||
|
return errors.WithDeferred(nil, file.CloseReplace())
|
||||||
|
}
|
|
@ -0,0 +1,101 @@
|
||||||
|
package aghrenameio_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/AdguardTeam/AdGuardHome/internal/aghrenameio"
|
||||||
|
"github.com/AdguardTeam/golibs/errors"
|
||||||
|
"github.com/AdguardTeam/golibs/testutil"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// testPerm is the common permission mode for tests.
|
||||||
|
const testPerm fs.FileMode = 0o644
|
||||||
|
|
||||||
|
// Common file data for tests.
|
||||||
|
var (
|
||||||
|
initialData = []byte("initial data\n")
|
||||||
|
newData = []byte("new data\n")
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPendingFile(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
targetPath := newInitialFile(t)
|
||||||
|
f, err := aghrenameio.NewPendingFile(targetPath, testPerm)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = f.Write(newData)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = f.CloseReplace()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
gotData, err := os.ReadFile(targetPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
assert.Equal(t, newData, gotData)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newInitialFile is a test helper that returns the path to the file containing
|
||||||
|
// [initialData].
|
||||||
|
func newInitialFile(t *testing.T) (targetPath string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
targetPath = filepath.Join(dir, "target")
|
||||||
|
|
||||||
|
err := os.WriteFile(targetPath, initialData, 0o644)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return targetPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWithDeferredCleanup(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const testError errors.Error = "test error"
|
||||||
|
|
||||||
|
testCases := []struct {
|
||||||
|
error error
|
||||||
|
name string
|
||||||
|
wantErrMsg string
|
||||||
|
wantData []byte
|
||||||
|
}{{
|
||||||
|
name: "success",
|
||||||
|
error: nil,
|
||||||
|
wantErrMsg: "",
|
||||||
|
wantData: newData,
|
||||||
|
}, {
|
||||||
|
name: "error",
|
||||||
|
error: testError,
|
||||||
|
wantErrMsg: testError.Error(),
|
||||||
|
wantData: initialData,
|
||||||
|
}}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
tc := tc
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
targetPath := newInitialFile(t)
|
||||||
|
f, err := aghrenameio.NewPendingFile(targetPath, testPerm)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = f.Write(newData)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = aghrenameio.WithDeferredCleanup(tc.error, f)
|
||||||
|
testutil.AssertErrorMsg(t, tc.wantErrMsg, err)
|
||||||
|
|
||||||
|
gotData, err := os.ReadFile(targetPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
assert.Equal(t, tc.wantData, gotData)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,48 @@
|
||||||
|
//go:build unix
|
||||||
|
|
||||||
|
package aghrenameio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
|
||||||
|
"github.com/google/renameio/v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
// pendingFile is a wrapper around [*renameio.PendingFile] making it an
|
||||||
|
// [io.WriteCloser].
|
||||||
|
type pendingFile struct {
|
||||||
|
file *renameio.PendingFile
|
||||||
|
}
|
||||||
|
|
||||||
|
// type check
|
||||||
|
var _ PendingFile = pendingFile{}
|
||||||
|
|
||||||
|
// Cleanup implements the [PendingFile] interface for pendingFile.
|
||||||
|
func (f pendingFile) Cleanup() (err error) {
|
||||||
|
return f.file.Cleanup()
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloseReplace implements the [PendingFile] interface for pendingFile.
|
||||||
|
func (f pendingFile) CloseReplace() (err error) {
|
||||||
|
return f.file.CloseAtomicallyReplace()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write implements the [PendingFile] interface for pendingFile.
|
||||||
|
func (f pendingFile) Write(b []byte) (n int, err error) {
|
||||||
|
return f.file.Write(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPendingFile is a wrapper around [renameio.NewPendingFile].
|
||||||
|
//
|
||||||
|
// f.Close must be called to finish the renaming.
|
||||||
|
func newPendingFile(filePath string, mode fs.FileMode) (f PendingFile, err error) {
|
||||||
|
file, err := renameio.NewPendingFile(filePath, renameio.WithPermissions(mode))
|
||||||
|
if err != nil {
|
||||||
|
// Don't wrap the error since it's informative enough as is.
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return pendingFile{
|
||||||
|
file: file,
|
||||||
|
}, nil
|
||||||
|
}
|
|
@ -0,0 +1,74 @@
|
||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package aghrenameio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/AdguardTeam/golibs/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
// pendingFile is a wrapper around [*os.File] calling [os.Rename] in its Close
|
||||||
|
// method.
|
||||||
|
type pendingFile struct {
|
||||||
|
file *os.File
|
||||||
|
targetPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// type check
|
||||||
|
var _ PendingFile = (*pendingFile)(nil)
|
||||||
|
|
||||||
|
// Cleanup implements the [PendingFile] interface for *pendingFile.
|
||||||
|
func (f *pendingFile) Cleanup() (err error) {
|
||||||
|
closeErr := f.file.Close()
|
||||||
|
err = os.Remove(f.file.Name())
|
||||||
|
|
||||||
|
// Put closeErr into the deferred error because that's where it is usually
|
||||||
|
// expected.
|
||||||
|
return errors.WithDeferred(err, closeErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloseReplace implements the [PendingFile] interface for *pendingFile.
|
||||||
|
func (f *pendingFile) CloseReplace() (err error) {
|
||||||
|
err = f.file.Close()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("closing: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.Rename(f.file.Name(), f.targetPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("renaming: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write implements the [PendingFile] interface for *pendingFile.
|
||||||
|
func (f *pendingFile) Write(b []byte) (n int, err error) {
|
||||||
|
return f.file.Write(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPendingFile is a wrapper around [os.CreateTemp].
|
||||||
|
//
|
||||||
|
// f.Close must be called to finish the renaming.
|
||||||
|
func newPendingFile(filePath string, mode fs.FileMode) (f PendingFile, err error) {
|
||||||
|
// Use the same directory as the file itself, because moves across
|
||||||
|
// filesystems can be especially problematic.
|
||||||
|
file, err := os.CreateTemp(filepath.Dir(filePath), "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("opening pending file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = file.Chmod(mode)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("preparing pending file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &pendingFile{
|
||||||
|
file: file,
|
||||||
|
targetPath: filePath,
|
||||||
|
}, nil
|
||||||
|
}
|
|
@ -9,7 +9,7 @@ import (
|
||||||
|
|
||||||
"github.com/AdguardTeam/golibs/errors"
|
"github.com/AdguardTeam/golibs/errors"
|
||||||
"github.com/AdguardTeam/golibs/log"
|
"github.com/AdguardTeam/golibs/log"
|
||||||
"github.com/google/renameio/maybe"
|
"github.com/google/renameio/v2/maybe"
|
||||||
"golang.org/x/exp/slices"
|
"golang.org/x/exp/slices"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -11,6 +11,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
|
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
|
||||||
|
"github.com/AdguardTeam/AdGuardHome/internal/aghrenameio"
|
||||||
"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
|
"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
|
||||||
"github.com/AdguardTeam/golibs/errors"
|
"github.com/AdguardTeam/golibs/errors"
|
||||||
"github.com/AdguardTeam/golibs/log"
|
"github.com/AdguardTeam/golibs/log"
|
||||||
|
@ -83,53 +84,53 @@ func (d *DNSFilter) filterSetProperties(
|
||||||
filters = d.WhitelistFilters
|
filters = d.WhitelistFilters
|
||||||
}
|
}
|
||||||
|
|
||||||
i := slices.IndexFunc(filters, func(filt FilterYAML) bool { return filt.URL == listURL })
|
i := slices.IndexFunc(filters, func(flt FilterYAML) bool { return flt.URL == listURL })
|
||||||
if i == -1 {
|
if i == -1 {
|
||||||
return false, errFilterNotExist
|
return false, errFilterNotExist
|
||||||
}
|
}
|
||||||
|
|
||||||
filt := &filters[i]
|
flt := &filters[i]
|
||||||
log.Debug(
|
log.Debug(
|
||||||
"filtering: set name to %q, url to %s, enabled to %t for filter %s",
|
"filtering: set name to %q, url to %s, enabled to %t for filter %s",
|
||||||
newList.Name,
|
newList.Name,
|
||||||
newList.URL,
|
newList.URL,
|
||||||
newList.Enabled,
|
newList.Enabled,
|
||||||
filt.URL,
|
flt.URL,
|
||||||
)
|
)
|
||||||
|
|
||||||
defer func(oldURL, oldName string, oldEnabled bool, oldUpdated time.Time, oldRulesCount int) {
|
defer func(oldURL, oldName string, oldEnabled bool, oldUpdated time.Time, oldRulesCount int) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
filt.URL = oldURL
|
flt.URL = oldURL
|
||||||
filt.Name = oldName
|
flt.Name = oldName
|
||||||
filt.Enabled = oldEnabled
|
flt.Enabled = oldEnabled
|
||||||
filt.LastUpdated = oldUpdated
|
flt.LastUpdated = oldUpdated
|
||||||
filt.RulesCount = oldRulesCount
|
flt.RulesCount = oldRulesCount
|
||||||
}
|
}
|
||||||
}(filt.URL, filt.Name, filt.Enabled, filt.LastUpdated, filt.RulesCount)
|
}(flt.URL, flt.Name, flt.Enabled, flt.LastUpdated, flt.RulesCount)
|
||||||
|
|
||||||
filt.Name = newList.Name
|
flt.Name = newList.Name
|
||||||
|
|
||||||
if filt.URL != newList.URL {
|
if flt.URL != newList.URL {
|
||||||
if d.filterExistsLocked(newList.URL) {
|
if d.filterExistsLocked(newList.URL) {
|
||||||
return false, errFilterExists
|
return false, errFilterExists
|
||||||
}
|
}
|
||||||
|
|
||||||
shouldRestart = true
|
shouldRestart = true
|
||||||
|
|
||||||
filt.URL = newList.URL
|
flt.URL = newList.URL
|
||||||
filt.LastUpdated = time.Time{}
|
flt.LastUpdated = time.Time{}
|
||||||
filt.unload()
|
flt.unload()
|
||||||
}
|
}
|
||||||
|
|
||||||
if filt.Enabled != newList.Enabled {
|
if flt.Enabled != newList.Enabled {
|
||||||
filt.Enabled = newList.Enabled
|
flt.Enabled = newList.Enabled
|
||||||
shouldRestart = true
|
shouldRestart = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if filt.Enabled {
|
if flt.Enabled {
|
||||||
if shouldRestart {
|
if shouldRestart {
|
||||||
// Download the filter contents.
|
// Download the filter contents.
|
||||||
shouldRestart, err = d.update(filt)
|
shouldRestart, err = d.update(flt)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// TODO(e.burkov): The validation of the contents of the new URL is
|
// TODO(e.burkov): The validation of the contents of the new URL is
|
||||||
|
@ -137,7 +138,7 @@ func (d *DNSFilter) filterSetProperties(
|
||||||
// possible to set a bad rules source, but the validation should still
|
// possible to set a bad rules source, but the validation should still
|
||||||
// kick in when the filter is enabled. Consider changing this behavior
|
// kick in when the filter is enabled. Consider changing this behavior
|
||||||
// to be stricter.
|
// to be stricter.
|
||||||
filt.unload()
|
flt.unload()
|
||||||
}
|
}
|
||||||
|
|
||||||
return shouldRestart, err
|
return shouldRestart, err
|
||||||
|
@ -250,24 +251,24 @@ func assignUniqueFilterID() int64 {
|
||||||
// Sets up a timer that will be checking for filters updates periodically
|
// Sets up a timer that will be checking for filters updates periodically
|
||||||
func (d *DNSFilter) periodicallyRefreshFilters() {
|
func (d *DNSFilter) periodicallyRefreshFilters() {
|
||||||
const maxInterval = 1 * 60 * 60
|
const maxInterval = 1 * 60 * 60
|
||||||
intval := 5 // use a dynamically increasing time interval
|
ivl := 5 // use a dynamically increasing time interval
|
||||||
for {
|
for {
|
||||||
isNetErr, ok := false, false
|
isNetErr, ok := false, false
|
||||||
if d.FiltersUpdateIntervalHours != 0 {
|
if d.FiltersUpdateIntervalHours != 0 {
|
||||||
_, isNetErr, ok = d.tryRefreshFilters(true, true, false)
|
_, isNetErr, ok = d.tryRefreshFilters(true, true, false)
|
||||||
if ok && !isNetErr {
|
if ok && !isNetErr {
|
||||||
intval = maxInterval
|
ivl = maxInterval
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if isNetErr {
|
if isNetErr {
|
||||||
intval *= 2
|
ivl *= 2
|
||||||
if intval > maxInterval {
|
if ivl > maxInterval {
|
||||||
intval = maxInterval
|
ivl = maxInterval
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
time.Sleep(time.Duration(intval) * time.Second)
|
time.Sleep(time.Duration(ivl) * time.Second)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -329,20 +330,20 @@ func (d *DNSFilter) refreshFiltersArray(filters *[]FilterYAML, force bool) (int,
|
||||||
return 0, nil, nil, false
|
return 0, nil, nil, false
|
||||||
}
|
}
|
||||||
|
|
||||||
nfail := 0
|
failNum := 0
|
||||||
for i := range updateFilters {
|
for i := range updateFilters {
|
||||||
uf := &updateFilters[i]
|
uf := &updateFilters[i]
|
||||||
updated, err := d.update(uf)
|
updated, err := d.update(uf)
|
||||||
updateFlags = append(updateFlags, updated)
|
updateFlags = append(updateFlags, updated)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
nfail++
|
failNum++
|
||||||
log.Info("filtering: updating filter from url %q: %s\n", uf.URL, err)
|
log.Error("filtering: updating filter from url %q: %s\n", uf.URL, err)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if nfail == len(updateFilters) {
|
if failNum == len(updateFilters) {
|
||||||
return 0, nil, nil, true
|
return 0, nil, nil, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -464,48 +465,6 @@ func (d *DNSFilter) update(filter *FilterYAML) (b bool, err error) {
|
||||||
return b, err
|
return b, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// finalizeUpdate closes and gets rid of temporary file f with filter's content
|
|
||||||
// according to updated. It also saves new values of flt's name, rules number
|
|
||||||
// and checksum if succeeded.
|
|
||||||
func (d *DNSFilter) finalizeUpdate(
|
|
||||||
file *os.File,
|
|
||||||
flt *FilterYAML,
|
|
||||||
updated bool,
|
|
||||||
res *rulelist.ParseResult,
|
|
||||||
) (err error) {
|
|
||||||
tmpFileName := file.Name()
|
|
||||||
|
|
||||||
// Close the file before renaming it because it's required on Windows.
|
|
||||||
//
|
|
||||||
// See https://github.com/adguardTeam/adGuardHome/issues/1553.
|
|
||||||
err = file.Close()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("closing temporary file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !updated {
|
|
||||||
log.Debug("filtering: filter %d from url %q has no changes, skipping", flt.ID, flt.URL)
|
|
||||||
|
|
||||||
return os.Remove(tmpFileName)
|
|
||||||
}
|
|
||||||
|
|
||||||
fltPath := flt.Path(d.DataDir)
|
|
||||||
|
|
||||||
log.Info("filtering: saving contents of filter %d into %q", flt.ID, fltPath)
|
|
||||||
|
|
||||||
// Don't use renameio or maybe packages, since those will require loading
|
|
||||||
// the whole filter content to the memory on Windows.
|
|
||||||
err = os.Rename(tmpFileName, fltPath)
|
|
||||||
if err != nil {
|
|
||||||
return errors.WithDeferred(err, os.Remove(tmpFileName))
|
|
||||||
}
|
|
||||||
|
|
||||||
flt.Name = aghalg.Coalesce(flt.Name, res.Title)
|
|
||||||
flt.checksum, flt.RulesCount = res.Checksum, res.RulesCount
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// updateIntl updates the flt rewriting it's actual file. It returns true if
|
// updateIntl updates the flt rewriting it's actual file. It returns true if
|
||||||
// the actual update has been performed.
|
// the actual update has been performed.
|
||||||
func (d *DNSFilter) updateIntl(flt *FilterYAML) (ok bool, err error) {
|
func (d *DNSFilter) updateIntl(flt *FilterYAML) (ok bool, err error) {
|
||||||
|
@ -513,63 +472,22 @@ func (d *DNSFilter) updateIntl(flt *FilterYAML) (ok bool, err error) {
|
||||||
|
|
||||||
var res *rulelist.ParseResult
|
var res *rulelist.ParseResult
|
||||||
|
|
||||||
var tmpFile *os.File
|
|
||||||
tmpFile, err = os.CreateTemp(filepath.Join(d.DataDir, filterDir), "")
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
finErr := d.finalizeUpdate(tmpFile, flt, ok, res)
|
|
||||||
if ok && finErr == nil {
|
|
||||||
log.Info(
|
|
||||||
"filtering: updated filter %d: %d bytes, %d rules",
|
|
||||||
flt.ID,
|
|
||||||
res.BytesWritten,
|
|
||||||
res.RulesCount,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err = errors.WithDeferred(err, finErr)
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Change the default 0o600 permission to something more acceptable by end
|
// Change the default 0o600 permission to something more acceptable by end
|
||||||
// users.
|
// users.
|
||||||
//
|
//
|
||||||
// See https://github.com/AdguardTeam/AdGuardHome/issues/3198.
|
// See https://github.com/AdguardTeam/AdGuardHome/issues/3198.
|
||||||
if err = tmpFile.Chmod(0o644); err != nil {
|
tmpFile, err := aghrenameio.NewPendingFile(flt.Path(d.DataDir), 0o644)
|
||||||
return false, fmt.Errorf("changing file mode: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var r io.Reader
|
|
||||||
if !filepath.IsAbs(flt.URL) {
|
|
||||||
var resp *http.Response
|
|
||||||
resp, err = d.HTTPClient.Get(flt.URL)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Info("filtering: requesting filter from %q: %s, skipping", flt.URL, err)
|
|
||||||
|
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
defer func() { err = errors.WithDeferred(err, resp.Body.Close()) }()
|
defer func() { err = d.finalizeUpdate(tmpFile, flt, res, err, ok) }()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
r, err := d.reader(flt.URL)
|
||||||
log.Info("filtering got status code %d from %q, skipping", resp.StatusCode, flt.URL)
|
|
||||||
|
|
||||||
return false, fmt.Errorf("got status code %d, want %d", resp.StatusCode, http.StatusOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
r = resp.Body
|
|
||||||
} else {
|
|
||||||
var f *os.File
|
|
||||||
f, err = os.Open(flt.URL)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("open file: %w", err)
|
// Don't wrap the error since it's informative enough as is.
|
||||||
}
|
return false, err
|
||||||
defer func() { err = errors.WithDeferred(err, f.Close()) }()
|
|
||||||
|
|
||||||
r = f
|
|
||||||
}
|
}
|
||||||
|
defer func() { err = errors.WithDeferred(err, r.Close()) }()
|
||||||
|
|
||||||
bufPtr := d.bufPool.Get().(*[]byte)
|
bufPtr := d.bufPool.Get().(*[]byte)
|
||||||
defer d.bufPool.Put(bufPtr)
|
defer d.bufPool.Put(bufPtr)
|
||||||
|
@ -580,6 +498,78 @@ func (d *DNSFilter) updateIntl(flt *FilterYAML) (ok bool, err error) {
|
||||||
return res.Checksum != flt.checksum && err == nil, err
|
return res.Checksum != flt.checksum && err == nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// finalizeUpdate closes and gets rid of temporary file f with filter's content
|
||||||
|
// according to updated. It also saves new values of flt's name, rules number
|
||||||
|
// and checksum if succeeded.
|
||||||
|
func (d *DNSFilter) finalizeUpdate(
|
||||||
|
file aghrenameio.PendingFile,
|
||||||
|
flt *FilterYAML,
|
||||||
|
res *rulelist.ParseResult,
|
||||||
|
returned error,
|
||||||
|
updated bool,
|
||||||
|
) (err error) {
|
||||||
|
id := flt.ID
|
||||||
|
if !updated {
|
||||||
|
if returned == nil {
|
||||||
|
log.Debug("filtering: filter %d from url %q has no changes, skipping", id, flt.URL)
|
||||||
|
}
|
||||||
|
|
||||||
|
return errors.WithDeferred(returned, file.Cleanup())
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("filtering: saving contents of filter %d into %q", id, flt.Path(d.DataDir))
|
||||||
|
|
||||||
|
err = file.CloseReplace()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("finalizing update: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rulesCount := res.RulesCount
|
||||||
|
log.Info("filtering: updated filter %d: %d bytes, %d rules", id, res.BytesWritten, rulesCount)
|
||||||
|
|
||||||
|
flt.Name = aghalg.Coalesce(flt.Name, res.Title)
|
||||||
|
flt.checksum = res.Checksum
|
||||||
|
flt.RulesCount = rulesCount
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// reader returns an io.ReadCloser reading filtering-rule list data form either
|
||||||
|
// a file on the filesystem or the filter's HTTP URL.
|
||||||
|
func (d *DNSFilter) reader(fltURL string) (r io.ReadCloser, err error) {
|
||||||
|
if !filepath.IsAbs(fltURL) {
|
||||||
|
r, err = d.readerFromURL(fltURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading from url: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
r, err = os.Open(fltURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("opening file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readerFromURL returns an io.ReadCloser reading filtering-rule list data form
|
||||||
|
// the filter's URL.
|
||||||
|
func (d *DNSFilter) readerFromURL(fltURL string) (r io.ReadCloser, err error) {
|
||||||
|
resp, err := d.HTTPClient.Get(fltURL)
|
||||||
|
if err != nil {
|
||||||
|
// Don't wrap the error since it's informative enough as is.
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("got status code %d, want %d", resp.StatusCode, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp.Body, nil
|
||||||
|
}
|
||||||
|
|
||||||
// loads filter contents from the file in dataDir
|
// loads filter contents from the file in dataDir
|
||||||
func (d *DNSFilter) load(flt *FilterYAML) (err error) {
|
func (d *DNSFilter) load(flt *FilterYAML) (err error) {
|
||||||
fileName := flt.Path(d.DataDir)
|
fileName := flt.Path(d.DataDir)
|
||||||
|
|
|
@ -20,7 +20,7 @@ import (
|
||||||
"github.com/AdguardTeam/golibs/errors"
|
"github.com/AdguardTeam/golibs/errors"
|
||||||
"github.com/AdguardTeam/golibs/log"
|
"github.com/AdguardTeam/golibs/log"
|
||||||
"github.com/AdguardTeam/golibs/timeutil"
|
"github.com/AdguardTeam/golibs/timeutil"
|
||||||
"github.com/google/renameio/maybe"
|
"github.com/google/renameio/v2/maybe"
|
||||||
"golang.org/x/exp/slices"
|
"golang.org/x/exp/slices"
|
||||||
yaml "gopkg.in/yaml.v3"
|
yaml "gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
|
@ -17,7 +17,7 @@ import (
|
||||||
"github.com/AdguardTeam/golibs/log"
|
"github.com/AdguardTeam/golibs/log"
|
||||||
"github.com/AdguardTeam/golibs/netutil"
|
"github.com/AdguardTeam/golibs/netutil"
|
||||||
"github.com/AdguardTeam/golibs/timeutil"
|
"github.com/AdguardTeam/golibs/timeutil"
|
||||||
"github.com/google/renameio/maybe"
|
"github.com/google/renameio/v2/maybe"
|
||||||
"golang.org/x/crypto/bcrypt"
|
"golang.org/x/crypto/bcrypt"
|
||||||
yaml "gopkg.in/yaml.v3"
|
yaml "gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
|
@ -8,7 +8,7 @@ import (
|
||||||
"github.com/AdguardTeam/AdGuardHome/internal/next/agh"
|
"github.com/AdguardTeam/AdGuardHome/internal/next/agh"
|
||||||
"github.com/AdguardTeam/AdGuardHome/internal/next/configmgr"
|
"github.com/AdguardTeam/AdGuardHome/internal/next/configmgr"
|
||||||
"github.com/AdguardTeam/golibs/log"
|
"github.com/AdguardTeam/golibs/log"
|
||||||
"github.com/google/renameio/maybe"
|
"github.com/google/renameio/v2/maybe"
|
||||||
)
|
)
|
||||||
|
|
||||||
// signalHandler processes incoming signals and shuts services down.
|
// signalHandler processes incoming signals and shuts services down.
|
||||||
|
|
|
@ -19,7 +19,7 @@ import (
|
||||||
"github.com/AdguardTeam/golibs/errors"
|
"github.com/AdguardTeam/golibs/errors"
|
||||||
"github.com/AdguardTeam/golibs/log"
|
"github.com/AdguardTeam/golibs/log"
|
||||||
"github.com/AdguardTeam/golibs/timeutil"
|
"github.com/AdguardTeam/golibs/timeutil"
|
||||||
"github.com/google/renameio/maybe"
|
"github.com/google/renameio/v2/maybe"
|
||||||
"golang.org/x/exp/slices"
|
"golang.org/x/exp/slices"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
|
@ -176,6 +176,7 @@ run_linter gocognit --over 10\
|
||||||
./internal/aghchan/\
|
./internal/aghchan/\
|
||||||
./internal/aghhttp/\
|
./internal/aghhttp/\
|
||||||
./internal/aghio/\
|
./internal/aghio/\
|
||||||
|
./internal/aghrenameio/\
|
||||||
./internal/client/\
|
./internal/client/\
|
||||||
./internal/dhcpsvc\
|
./internal/dhcpsvc\
|
||||||
./internal/filtering/hashprefix/\
|
./internal/filtering/hashprefix/\
|
||||||
|
@ -223,6 +224,7 @@ run_linter gosec --quiet\
|
||||||
./internal/aghio\
|
./internal/aghio\
|
||||||
./internal/aghnet\
|
./internal/aghnet\
|
||||||
./internal/aghos\
|
./internal/aghos\
|
||||||
|
./internal/aghrenameio/\
|
||||||
./internal/aghtest\
|
./internal/aghtest\
|
||||||
./internal/client\
|
./internal/client\
|
||||||
./internal/dhcpd\
|
./internal/dhcpd\
|
||||||
|
|
|
@ -12,7 +12,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/AdguardTeam/golibs/log"
|
"github.com/AdguardTeam/golibs/log"
|
||||||
"github.com/google/renameio/maybe"
|
"github.com/google/renameio/v2/maybe"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
|
Loading…
Reference in New Issue