+ filters: support filters from local disk

This commit is contained in:
Simon Zolin 2020-03-05 14:37:43 +03:00
parent fcaed16638
commit 81309c707a
4 changed files with 38 additions and 24 deletions

View File

@ -1439,7 +1439,7 @@ Request:
{
"name": "..."
"url": "..."
"url": "..." // URL or an absolute file path
"whitelist": true
}

View File

@ -8,15 +8,22 @@ import (
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/AdguardTeam/AdGuardHome/util"
"github.com/AdguardTeam/golibs/log"
"github.com/miekg/dns"
)
// IsValidURL - return TRUE if URL is valid
// IsValidURL - return TRUE if URL or file path is valid
func IsValidURL(rawurl string) bool {
if filepath.IsAbs(rawurl) {
// this is a file path
return util.FileExists(rawurl)
}
url, err := url.ParseRequestURI(rawurl)
if err != nil {
return false //Couldn't even parse the rawurl
@ -42,7 +49,7 @@ func (f *Filtering) handleFilteringAddURL(w http.ResponseWriter, r *http.Request
}
if !IsValidURL(fj.URL) {
http.Error(w, "Invalid URL", http.StatusBadRequest)
http.Error(w, "Invalid URL or file path", http.StatusBadRequest)
return
}
@ -100,11 +107,6 @@ func (f *Filtering) handleFilteringRemoveURL(w http.ResponseWriter, r *http.Requ
return
}
if !IsValidURL(req.URL) {
http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest)
return
}
// go through each element and delete if url matches
config.Lock()
newFilters := []filter{}
@ -154,7 +156,7 @@ func (f *Filtering) handleFilteringSetURL(w http.ResponseWriter, r *http.Request
}
if !IsValidURL(fj.URL) {
http.Error(w, "invalid URL", http.StatusBadRequest)
http.Error(w, "invalid URL or file path", http.StatusBadRequest)
return
}

View File

@ -69,7 +69,7 @@ func defaultFilters() []filter {
// field ordering is important -- yaml fields will mirror ordering from here
type filter struct {
Enabled bool
URL string
URL string // URL or a file path
Name string `yaml:"name"`
RulesCount int `yaml:"-"`
LastUpdated time.Time `yaml:"-"`
@ -500,6 +500,7 @@ func (f *Filtering) update(filter *filter) (bool, error) {
return b, err
}
// nolint(gocyclo)
func (f *Filtering) updateIntl(filter *filter) (bool, error) {
log.Tracef("Downloading update for filter %d from %s", filter.ID, filter.URL)
@ -514,6 +515,15 @@ func (f *Filtering) updateIntl(filter *filter) (bool, error) {
}
}()
var reader io.Reader
if filepath.IsAbs(filter.URL) {
f, err := os.Open(filter.URL)
if err != nil {
return false, fmt.Errorf("open file: %s", err)
}
defer f.Close()
reader = f
} else {
resp, err := Context.client.Get(filter.URL)
if resp != nil && resp.Body != nil {
defer resp.Body.Close()
@ -527,6 +537,8 @@ func (f *Filtering) updateIntl(filter *filter) (bool, error) {
log.Printf("Got status code %d from URL %s, skipping", resp.StatusCode, filter.URL)
return false, fmt.Errorf("got status code != 200: %d", resp.StatusCode)
}
reader = resp.Body
}
htmlTest := true
firstChunk := make([]byte, 4*1024)
@ -534,7 +546,7 @@ func (f *Filtering) updateIntl(filter *filter) (bool, error) {
buf := make([]byte, 64*1024)
total := 0
for {
n, err := resp.Body.Read(buf)
n, err := reader.Read(buf)
total += n
if htmlTest {

View File

@ -507,7 +507,7 @@ paths:
tags:
- filtering
operationId: filteringAddURL
summary: 'Add filter URL'
summary: 'Add filter URL or an absolute file path'
consumes:
- application/json
parameters:
@ -1495,7 +1495,7 @@ definitions:
name:
type: "string"
url:
description: "URL containing filtering rules"
description: "URL or an absolute path to the file containing filtering rules"
type: "string"
example: "https://filters.adtidy.org/windows/filters/15.txt"
RemoveUrlRequest: