// vetted-filters fetches the most recent Hostlists Registry filtering rule list // index and transforms the filters from it to AdGuard Home's format. package main import ( "bytes" "context" "encoding/json" "fmt" "log/slog" "net/http" "net/url" "os" "time" "github.com/AdguardTeam/golibs/logutil/slogutil" "github.com/google/renameio/v2/maybe" ) func main() { ctx := context.Background() l := slogutil.New(nil) urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/filters.json" if v, ok := os.LookupEnv("URL"); ok { urlStr = v } // Validate the URL. _, err := url.Parse(urlStr) check(err) c := &http.Client{ Timeout: 10 * time.Second, } resp, err := c.Get(urlStr) check(err) defer slogutil.CloseAndLog(ctx, l, resp.Body, slog.LevelError) if resp.StatusCode != http.StatusOK { panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode)) } hlFlt := &hlFilters{} err = json.NewDecoder(resp.Body).Decode(hlFlt) check(err) aghFlt := &aghFilters{ Categories: map[string]*aghFiltersCategory{ "general": { Name: "filter_category_general", Description: "filter_category_general_desc", }, "other": { Name: "filter_category_other", Description: "filter_category_other_desc", }, "regional": { Name: "filter_category_regional", Description: "filter_category_regional_desc", }, "security": { Name: "filter_category_security", Description: "filter_category_security_desc", }, }, Filters: map[string]*aghFiltersFilter{}, } for i, f := range hlFlt.Filters { key := f.FilterKey cat := f.category() if cat == "" { l.WarnContext(ctx, "no fitting category for filter", "key", key, "idx", i) } aghFlt.Filters[key] = &aghFiltersFilter{ Name: f.Name, CategoryID: cat, Homepage: f.Homepage, // NOTE: The source URL in filters.json is not guaranteed to contain // the URL of the filtering rule list. So, use our mirror for the // vetted blocklists, which are mostly guaranteed to be valid and // available lists. Source: f.DownloadURL, } } buf := &bytes.Buffer{} _, _ = buf.WriteString(jsHeader) enc := json.NewEncoder(buf) enc.SetIndent("", " ") err = enc.Encode(aghFlt) check(err) err = maybe.WriteFile("client/src/helpers/filters/filters.ts", buf.Bytes(), 0o644) check(err) } // jsHeader is the header for the generated JavaScript file. It informs the // reader that the file is generated and disables some style-related eslint // checks. const jsHeader = `// Code generated by go run ./scripts/vetted-filters/main.go; DO NOT EDIT. /* eslint quote-props: 'off', quotes: 'off', comma-dangle: 'off', semi: 'off' */ export default ` // check is a simple error-checking helper for scripts. func check(err error) { if err != nil { panic(err) } } // hlFilters is the JSON structure for the Hostlists Registry rule list index. type hlFilters struct { Filters []*hlFiltersFilter `json:"filters"` } // hlFiltersFilter is the JSON structure for a filter in the Hostlists Registry. type hlFiltersFilter struct { DownloadURL string `json:"downloadUrl"` FilterKey string `json:"filterKey"` Homepage string `json:"homepage"` Name string `json:"name"` Tags []int `json:"tags"` } // Known tag IDs. Keep in sync with tags/metadata.json in the source repo. const ( tagIDGeneral = 1 tagIDSecurity = 2 tagIDRegional = 3 tagIDOther = 4 ) // category returns the AdGuard Home category for this filter. If there is no // fitting category, cat is empty. func (f *hlFiltersFilter) category() (cat string) { for _, t := range f.Tags { switch t { case tagIDGeneral: return "general" case tagIDSecurity: return "security" case tagIDRegional: return "regional" case tagIDOther: return "other" } } return "" } // aghFilters is the JSON structure for AdGuard Home's list of vetted filtering // rule list in file client/src/helpers/filters/filters.ts. type aghFilters struct { Categories map[string]*aghFiltersCategory `json:"categories"` Filters map[string]*aghFiltersFilter `json:"filters"` } // aghFiltersCategory is the JSON structure for a category in the vetted // filtering rule list file. type aghFiltersCategory struct { Name string `json:"name"` Description string `json:"description"` } // aghFiltersFilter is the JSON structure for a filter in the vetted filtering // rule list file. type aghFiltersFilter struct { Name string `json:"name"` CategoryID string `json:"categoryId"` Homepage string `json:"homepage"` Source string `json:"source"` }