2022-11-02 13:18:02 +00:00
|
|
|
// vetted-filters fetches the most recent Hostlists Registry filtering rule list
|
|
|
|
// index and transforms the filters from it to AdGuard Home's format.
|
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
|
|
|
"net/http"
|
|
|
|
"net/url"
|
|
|
|
"os"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/AdguardTeam/golibs/log"
|
2023-07-26 11:18:44 +01:00
|
|
|
"github.com/google/renameio/v2/maybe"
|
2022-11-02 13:18:02 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func main() {
|
|
|
|
urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/filters.json"
|
|
|
|
if v, ok := os.LookupEnv("URL"); ok {
|
|
|
|
urlStr = v
|
|
|
|
}
|
|
|
|
|
|
|
|
// Validate the URL.
|
|
|
|
_, err := url.Parse(urlStr)
|
|
|
|
check(err)
|
|
|
|
|
|
|
|
c := &http.Client{
|
|
|
|
Timeout: 10 * time.Second,
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := c.Get(urlStr)
|
|
|
|
check(err)
|
|
|
|
defer log.OnCloserError(resp.Body, log.ERROR)
|
|
|
|
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
|
|
panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode))
|
|
|
|
}
|
|
|
|
|
|
|
|
hlFlt := &hlFilters{}
|
|
|
|
err = json.NewDecoder(resp.Body).Decode(hlFlt)
|
|
|
|
check(err)
|
|
|
|
|
|
|
|
aghFlt := &aghFilters{
|
|
|
|
Categories: map[string]*aghFiltersCategory{
|
|
|
|
"general": {
|
|
|
|
Name: "filter_category_general",
|
|
|
|
Description: "filter_category_general_desc",
|
|
|
|
},
|
|
|
|
"other": {
|
|
|
|
Name: "filter_category_other",
|
|
|
|
Description: "filter_category_other_desc",
|
|
|
|
},
|
|
|
|
"regional": {
|
|
|
|
Name: "filter_category_regional",
|
|
|
|
Description: "filter_category_regional_desc",
|
|
|
|
},
|
|
|
|
"security": {
|
|
|
|
Name: "filter_category_security",
|
|
|
|
Description: "filter_category_security_desc",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Filters: map[string]*aghFiltersFilter{},
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, f := range hlFlt.Filters {
|
|
|
|
id := f.FilterID
|
|
|
|
cat := f.category()
|
|
|
|
if cat == "" {
|
|
|
|
log.Info("warning: filter %s at index %d does not have a fitting category", id, i)
|
|
|
|
}
|
|
|
|
|
|
|
|
aghFlt.Filters[id] = &aghFiltersFilter{
|
|
|
|
Name: f.Name,
|
|
|
|
CategoryID: cat,
|
|
|
|
Homepage: f.Homepage,
|
2022-12-07 13:46:59 +00:00
|
|
|
// NOTE: The source URL in filters.json is not guaranteed to contain
|
|
|
|
// the URL of the filtering rule list. So, use our mirror for the
|
|
|
|
// vetted blocklists, which are mostly guaranteed to be valid and
|
|
|
|
// available lists.
|
|
|
|
Source: f.DownloadURL,
|
2022-11-02 13:18:02 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
buf := &bytes.Buffer{}
|
|
|
|
_, _ = buf.WriteString(jsHeader)
|
|
|
|
|
|
|
|
enc := json.NewEncoder(buf)
|
|
|
|
enc.SetIndent("", " ")
|
|
|
|
|
|
|
|
err = enc.Encode(aghFlt)
|
|
|
|
check(err)
|
|
|
|
|
|
|
|
err = maybe.WriteFile("client/src/helpers/filters/filters.js", buf.Bytes(), 0o644)
|
|
|
|
check(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// jsHeader is the header for the generated JavaScript file. It informs the
|
|
|
|
// reader that the file is generated and disables some style-related eslint
|
|
|
|
// checks.
|
|
|
|
const jsHeader = `// Code generated by go run ./scripts/vetted-filters/main.go; DO NOT EDIT.
|
|
|
|
|
|
|
|
/* eslint quote-props: 'off', quotes: 'off', comma-dangle: 'off', semi: 'off' */
|
|
|
|
|
|
|
|
export default `
|
|
|
|
|
|
|
|
// check is a simple error-checking helper for scripts.
|
|
|
|
func check(err error) {
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// hlFilters is the JSON structure for the Hostlists Registry rule list index.
|
|
|
|
type hlFilters struct {
|
|
|
|
Filters []*hlFiltersFilter `json:"filters"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// hlFiltersFilter is the JSON structure for a filter in the Hostlists Registry.
|
|
|
|
type hlFiltersFilter struct {
|
2022-12-07 13:46:59 +00:00
|
|
|
DownloadURL string `json:"downloadUrl"`
|
|
|
|
FilterID string `json:"filterId"`
|
|
|
|
Homepage string `json:"homepage"`
|
|
|
|
Name string `json:"name"`
|
|
|
|
Tags []string `json:"tags"`
|
2022-11-02 13:18:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// category returns the AdGuard Home category for this filter. If there is no
|
|
|
|
// fitting category, cat is empty.
|
|
|
|
func (f *hlFiltersFilter) category() (cat string) {
|
|
|
|
for _, t := range f.Tags {
|
|
|
|
switch t {
|
|
|
|
case "purpose:general":
|
|
|
|
return "general"
|
|
|
|
case "purpose:other":
|
|
|
|
return "other"
|
|
|
|
case "purpose:regional":
|
|
|
|
return "regional"
|
|
|
|
case "purpose:security":
|
|
|
|
return "security"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
|
|
|
// aghFilters is the JSON structure for AdGuard Home's list of vetted filtering
|
|
|
|
// rule list in file client/src/helpers/filters/filters.js.
|
|
|
|
type aghFilters struct {
|
|
|
|
Categories map[string]*aghFiltersCategory `json:"categories"`
|
|
|
|
Filters map[string]*aghFiltersFilter `json:"filters"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// aghFiltersCategory is the JSON structure for a category in the vetted
|
|
|
|
// filtering rule list file.
|
|
|
|
type aghFiltersCategory struct {
|
|
|
|
Name string `json:"name"`
|
|
|
|
Description string `json:"description"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// aghFiltersFilter is the JSON structure for a filter in the vetted filtering
|
|
|
|
// rule list file.
|
|
|
|
type aghFiltersFilter struct {
|
|
|
|
Name string `json:"name"`
|
|
|
|
CategoryID string `json:"categoryId"`
|
|
|
|
Homepage string `json:"homepage"`
|
|
|
|
Source string `json:"source"`
|
|
|
|
}
|