From 3993f4c47614887faafde09cf5e61a4b08184d5a Mon Sep 17 00:00:00 2001 From: Ainar Garipov Date: Thu, 27 Jun 2024 20:18:17 +0300 Subject: [PATCH] Pull request 2243: AG-33443-upd-vetted-script Squashed commit of the following: commit 85ce53f0fc53c5422d49dc50d9017a7dd009f7ba Author: Ainar Garipov Date: Thu Jun 27 20:11:01 2024 +0300 client: upd commit 2588807dfdf4e7e0159ada57d6973194eaf3e286 Merge: c5fc7fbf4 a1a31cd91 Author: Ainar Garipov Date: Thu Jun 27 20:10:34 2024 +0300 Merge branch 'master' into AG-33443-upd-vetted-script commit c5fc7fbf4f85cb7e123a58f42c1ee83b1b369013 Author: Ainar Garipov Date: Mon Jun 24 13:39:46 2024 +0300 scripts: imp log commit af420878b4f5753187b7afa6c2c3f3db54cf7711 Author: Ainar Garipov Date: Mon Jun 24 13:12:12 2024 +0300 scripts: upd vetter-filters --- client/src/helpers/filters/filters.ts | 2 +- scripts/vetted-filters/main.go | 45 +++++++++++++++++---------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/client/src/helpers/filters/filters.ts b/client/src/helpers/filters/filters.ts index ba80449a..6582b280 100644 --- a/client/src/helpers/filters/filters.ts +++ b/client/src/helpers/filters/filters.ts @@ -295,7 +295,7 @@ export default { "phishing_army": { "name": "Phishing Army", "categoryId": "security", - "homepage": "https://gitlab.com/malware-filter/phishing-filter", + "homepage": "https://phishing.army/", "source": "https://adguardteam.github.io/HostlistsRegistry/assets/filter_18.txt" }, "scam_blocklist_by_durablenapkin": { diff --git a/scripts/vetted-filters/main.go b/scripts/vetted-filters/main.go index 663f86a2..4ccb29e5 100644 --- a/scripts/vetted-filters/main.go +++ b/scripts/vetted-filters/main.go @@ -4,18 +4,23 @@ package main import ( "bytes" + "context" "encoding/json" "fmt" + "log/slog" "net/http" "net/url" "os" "time" - "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/logutil/slogutil" "github.com/google/renameio/v2/maybe" ) func main() { + ctx := context.Background() + l := slogutil.New(nil) + urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/filters.json" if v, ok := os.LookupEnv("URL"); ok { urlStr = v @@ -31,7 +36,7 @@ func main() { resp, err := c.Get(urlStr) check(err) - defer log.OnCloserError(resp.Body, log.ERROR) + defer slogutil.CloseAndLog(ctx, l, resp.Body, slog.LevelError) if resp.StatusCode != http.StatusOK { panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode)) @@ -64,13 +69,13 @@ func main() { } for i, f := range hlFlt.Filters { - id := f.FilterID + key := f.FilterKey cat := f.category() if cat == "" { - log.Info("warning: filter %s at index %d does not have a fitting category", id, i) + l.WarnContext(ctx, "no fitting category for filter", "key", key, "idx", i) } - aghFlt.Filters[id] = &aghFiltersFilter{ + aghFlt.Filters[key] = &aghFiltersFilter{ Name: f.Name, CategoryID: cat, Homepage: f.Homepage, @@ -118,26 +123,34 @@ type hlFilters struct { // hlFiltersFilter is the JSON structure for a filter in the Hostlists Registry. type hlFiltersFilter struct { - DownloadURL string `json:"downloadUrl"` - FilterID string `json:"filterId"` - Homepage string `json:"homepage"` - Name string `json:"name"` - Tags []string `json:"tags"` + DownloadURL string `json:"downloadUrl"` + FilterKey string `json:"filterKey"` + Homepage string `json:"homepage"` + Name string `json:"name"` + Tags []int `json:"tags"` } +// Known tag IDs. Keep in sync with tags/metadata.json in the source repo. +const ( + tagIDGeneral = 1 + tagIDSecurity = 2 + tagIDRegional = 3 + tagIDOther = 4 +) + // category returns the AdGuard Home category for this filter. If there is no // fitting category, cat is empty. func (f *hlFiltersFilter) category() (cat string) { for _, t := range f.Tags { switch t { - case "purpose:general": + case tagIDGeneral: return "general" - case "purpose:other": - return "other" - case "purpose:regional": - return "regional" - case "purpose:security": + case tagIDSecurity: return "security" + case tagIDRegional: + return "regional" + case tagIDOther: + return "other" } }