mirrors
/
AdGuardHome
mirror of https://github.com/AdguardTeam/AdGuardHome.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pull request: 3972-hostlists-registry 

Updates #3972.

Squashed commit of the following:

commit 8341c13c39f38048796bc42c5f5337e1c8d3bcd4
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Oct 21 13:06:55 2022 +0300

    all: imp docs

commit b7f961c879abf288c347e5bf55182e7dfa8e84e4
Merge: 607077ca 68d13fcc
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Oct 20 18:23:57 2022 +0300

    Merge branch 'master' into 3972-hostlists-registry

commit 607077ca8141f732d4678955c62cc2345958766b
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Oct 20 18:07:09 2022 +0300

    all: use hostlists registry
This commit is contained in:
Ainar Garipov 2022-10-21 14:05:10 +03:00
parent 68d13fcc2b
commit 2a85d7dd7e
7 changed files with 382 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/ISSUE_TEMPLATE/config.yml vendored
View File

@ -6,6 +6,11 @@
website
'name': 'AdGuard filters issues'
'url': 'https://link.adtidy.org/forward.html?action=report&app=home&from=github'
- 'about': >
Please send requests for addition to the vetted filtering lists to the
Hostlists Registry repository.
'name': 'AdGuard Hostlists Registry'
'url': 'https://github.com/AdguardTeam/HostlistsRegistry'
- 'about': >
Please use GitHub Discussions for questions
'name': 'Q&A Discussions'

2
client/src/components/Filters/DnsBlocklist.js
View File

@ -15,7 +15,7 @@ import {
getObjDiff,
} from '../../helpers/helpers';
const filtersCatalog = require('../../helpers/filters/filters.json');
import filtersCatalog from '../../helpers/filters/filters';
class DnsBlocklist extends Component {
componentDidMount() {

3
client/src/components/Filters/Form.js
View File

@ -7,8 +7,7 @@ import classNames from 'classnames';
import { validatePath, validateRequiredValue } from '../../helpers/validators';
import { CheckboxField, renderInputField } from '../../helpers/form';
import { MODAL_OPEN_TIMEOUT, MODAL_TYPE, FORM_NAME } from '../../helpers/constants';
const filtersCatalog = require('../../helpers/filters/filters.json');
import filtersCatalog from '../../helpers/filters/filters';
const getIconsData = (homepage, source) => ([
{

322
client/src/helpers/filters/filters.json → client/src/helpers/filters/filters.js
View File

@ -1,162 +1,220 @@
{
// Code generated by go run ./scripts/vetted-filters/main.go; DO NOT EDIT.
/* eslint quote-props: 'off', quotes: 'off', comma-dangle: 'off', semi: 'off' */
export default {
"categories": {
"general": {
"name": "filter_category_general",
"description": "filter_category_general_desc"
},
"security": {
"name": "filter_category_security",
"description": "filter_category_security_desc"
"other": {
"name": "filter_category_other",
"description": "filter_category_other_desc"
},
"regional": {
"name": "filter_category_regional",
"description": "filter_category_regional_desc"
},
"other": {
"name": "filter_category_other",
"description": "filter_category_other_desc"
"security": {
"name": "filter_category_security",
"description": "filter_category_security_desc"
}
},
"filters": {
"adguard-dns-filter": {
"name": "AdGuard DNS filter",
"1hosts_lite": {
"name": "1Hosts (Lite)",
"categoryId": "general",
"homepage": "https://github.com/AdguardTeam/AdGuardSDNSFilter",
"source": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt"
"homepage": "https://badmojr.github.io/1Hosts/",
"source": "https://badmojr.gitlab.io/1hosts/Lite/adblock.txt"
},
"adaway-default-blocklist": {
"name": "AdAway Default Blocklist",
"categoryId": "general",
"homepage": "https://github.com/AdAway/adaway.github.io/",
"source": "https://adaway.org/hosts.txt"
},
"peter-lowe-list": {
"name": "Peter Lowe's List",
"categoryId": "general",
"homepage": "https://pgl.yoyo.org/adservers/",
"source": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=adblockplus&showintro=1&mimetype=plaintext"
},
"dan-pollock-list": {
"name": "Dan Pollock's List",
"categoryId": "general",
"homepage": "https://someonewhocares.org/",
"source": "https://someonewhocares.org/hosts/zero/hosts"
},
"oisd": {
"name": "OISD Blocklist Basic",
"categoryId": "general",
"homepage": "https://oisd.nl/",
"source": "https://abp.oisd.nl/basic/"
},
"game-console-adblock-list": {
"name": "Game Console Adblock List",
"categoryId": "general",
"homepage": "https://github.com/DandelionSprout/adfilt",
"source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt"
},
"perflyst-dandelion-sprout-smart-tv-blocklist-for-adguard-home": {
"name": "Perflyst and Dandelion Sprout's Smart-TV Blocklist",
"categoryId": "general",
"homepage": "https://github.com/Perflyst/PiHoleBlocklist",
"source": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt"
},
"nocoin-filter-list": {
"name": "NoCoin Filter List",
"categoryId": "security",
"homepage": "https://github.com/hoshsadiq/adblock-nocoin-list/",
"source": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt"
},
"the-big-list-of-hacked-malware-web-sites": {
"name": "The Big List of Hacked Malware Web Sites",
"categoryId": "security",
"homepage": "https://github.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites",
"source": "https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/hosts"
},
"scam-blocklist-by-durable-napkin": {
"name": "Scam Blocklist by DurableNapkin",
"categoryId": "security",
"homepage": "https://github.com/durablenapkin/scamblocklist",
"source": "https://raw.githubusercontent.com/durablenapkin/scamblocklist/master/adguard.txt"
},
"urlhaus-filter-online": {
"name": "Online Malicious URL Blocklist",
"categoryId": "security",
"homepage": "https://gitlab.com/malware-filter/urlhaus-filter",
"source": "https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-agh-online.txt"
},
"dandelion-sprouts-anti-malware-list": {
"name": "Dandelion Sprout's Anti-Malware List",
"categoryId": "security",
"homepage": "https://github.com/DandelionSprout/adfilt",
"source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareAdGuardHome.txt"
},
"NOR-dandelion-sprouts-nordiske-filtre": {
"name": "NOR: Dandelion Sprouts nordiske filtre",
"CHN_adrules": {
"name": "CHN: AdRules DNS List",
"categoryId": "regional",
"homepage": "https://github.com/DandelionSprout/adfilt",
"source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/NorwegianExperimentalList%20alternate%20versions/NordicFiltersAdGuardHome.txt"
"homepage": "https://github.com/Cats-Team/AdRules",
"source": "https://raw.githubusercontent.com/Cats-Team/AdRules/main/dns.txt"
},
"POL-polish-filters-for-pihole": {
"name": "POL: Polish filters for Pi hole",
"categoryId": "regional",
"homepage": "https://www.certyficate.it/",
"source": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-pihole-filters/hostfile.txt"
},
"KOR-youslist": {
"name": "KOR: YousList",
"categoryId": "regional",
"homepage": "https://github.com/yous/YousList",
"source": "https://raw.githubusercontent.com/yous/YousList/master/hosts.txt"
},
"VNM-abpvn-list": {
"name": "VNM: ABPVN List",
"categoryId": "regional",
"homepage": "http://abpvn.com/",
"source": "https://abpvn.com/android/abpvn.txt"
},
"SWE-frellwit-swedish-hosts-file": {
"name": "SWE: Frellwit's Swedish Hosts File",
"categoryId": "regional",
"homepage": "https://github.com/lassekongo83/Frellwits-filter-lists/",
"source": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt"
},
"ITA-filtri-dns": {
"name": "ITA: Filtri-DNS",
"categoryId": "regional",
"homepage": "https://filtri-dns.ga/",
"source": "https://filtri-dns.ga/filtri.txt"
},
"IRN-unwanted-iranian-domains": {
"name": "IRN: Unwanted Iranian domains",
"categoryId": "regional",
"homepage": "https://github.com/DRSDavidSoft/additional-hosts",
"source": "https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/unwanted-iranian.txt"
},
"MKD-macedonian-pi-hole-blocklist": {
"name": "MKD: Macedonian Pi-hole Blocklist",
"categoryId": "regional",
"homepage": "https://github.com/cchevy/macedonian-pi-hole-blocklist",
"source": "https://raw.githubusercontent.com/cchevy/macedonian-pi-hole-blocklist/master/hosts.txt"
},
"CHN-anti-ad" : {
"CHN_anti_ad": {
"name": "CHN: anti-AD",
"categoryId": "regional",
"homepage": "https://anti-ad.net/",
"source": "https://anti-ad.net/easylist.txt"
},
"IDN-abpindo": {
"IDN_abpindo": {
"name": "IDN: ABPindo",
"categoryId": "regional",
"homepage": "https://github.com/ABPindo/indonesianadblockrules/",
"source": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt"
"homepage": "https://github.com/ABPindo/indonesianadblockrules",
"source": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/aghome.txt"
},
"NLD-Easylist": {
"name": "NLD: Easylist",
"IRN_unwanted_iranian_domains": {
"name": "IRN: PersianBlocker list",
"categoryId": "regional",
"homepage": "https://forums.lanik.us/viewforum.php?f=100",
"source": "https://easylist-downloads.adblockplus.org/easylistdutch.txt"
"homepage": "https://github.com/MasterKia/PersianBlocker",
"source": "https://raw.githubusercontent.com/MasterKia/PersianBlocker/main/PersianBlockerHosts.txt"
},
"windows-spy-blocker" : {
"ITA_filtri_dns": {
"name": "ITA: Filtri-DNS",
"categoryId": "regional",
"homepage": "https://filtri-dns.ga/",
"source": "https://filtri-dns.ga/filtri.txt"
},
"KOR_list_kr": {
"name": "KOR: List-KR DNS",
"categoryId": "regional",
"homepage": "https://github.com/List-KR/List-KR",
"source": "https://github.com/List-KR/List-KR"
},
"KOR_youslist": {
"name": "KOR: YousList",
"categoryId": "regional",
"homepage": "https://github.com/yous/YousList",
"source": "https://raw.githubusercontent.com/yous/YousList/master/hosts.txt"
},
"MKD_macedonian_pi_hole_blocklist": {
"name": "MKD: Macedonian Pi-hole Blocklist",
"categoryId": "regional",
"homepage": "https://github.com/cchevy/macedonian-pi-hole-blocklist",
"source": "https://raw.githubusercontent.com/cchevy/macedonian-pi-hole-blocklist/master/hosts.txt"
},
"NOR_dandelion_sprouts_anti_malware_list": {
"name": "NOR: Dandelion Sprouts nordiske filtre",
"categoryId": "regional",
"homepage": "https://github.com/DandelionSprout/adfilt",
"source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/NorwegianExperimentalList%20alternate%20versions/NordicFiltersAdGuardHome.txt"
},
"POL_polish_filters_for_pi_hole": {
"name": "POL: Polish filters for Pi hole",
"categoryId": "regional",
"homepage": "https://www.certyficate.it/",
"source": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-pihole-filters/hostfile.txt"
},
"SWE_frellwit_swedish_hosts_file": {
"name": "SWE: Frellwit's Swedish Hosts File",
"categoryId": "regional",
"homepage": "https://github.com/lassekongo83/Frellwits-filter-lists/",
"source": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt"
},
"TUR_turk_adlist": {
"name": "TUR: turk-adlist",
"categoryId": "regional",
"homepage": "https://github.com/bkrucarci/turk-adlist",
"source": "https://raw.githubusercontent.com/bkrucarci/turk-adlist/master/hosts"
},
"VNM_abpvn": {
"name": "VNM: ABPVN List",
"categoryId": "regional",
"homepage": "http://abpvn.com/",
"source": "https://abpvn.com/android/abpvn.txt"
},
"adguard_dns_filter": {
"name": "AdGuard DNS filter",
"categoryId": "general",
"homepage": "https://github.com/AdguardTeam/AdGuardSDNSFilter",
"source": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt"
},
"adway_default_blocklist": {
"name": "AdAway Default Blocklist",
"categoryId": "general",
"homepage": "https://github.com/AdAway/adaway.github.io/",
"source": "https://adaway.org/hosts.txt"
},
"curben_phishing_filter": {
"name": "Phishing URL Blocklist (PhishTank and OpenPhish)",
"categoryId": "security",
"homepage": "https://gitlab.com/malware-filter/phishing-filter",
"source": "https://malware-filter.gitlab.io/malware-filter/phishing-filter-agh.txt"
},
"dan_pollocks_list": {
"name": "Dan Pollock's List",
"categoryId": "general",
"homepage": "https://someonewhocares.org/",
"source": "https://someonewhocares.org/hosts/zero/hosts"
},
"dandelion_sprouts_anti_malware_list": {
"name": "Dandelion Sprout's Anti-Malware List",
"categoryId": "security",
"homepage": "https://github.com/DandelionSprout/adfilt",
"source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareAdGuardHome.txt"
},
"dandelion_sprouts_game_console_adblock_list": {
"name": "Dandelion Sprout's Game Console Adblock List",
"categoryId": "other",
"homepage": "https://github.com/DandelionSprout/adfilt",
"source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt"
},
"energized_spark": {
"name": "Energized Spark",
"categoryId": "general",
"homepage": "https://energized.pro/",
"source": "https://block.energized.pro/spark/formats/filter"
},
"nocoin_filter_list": {
"name": "NoCoin Filter List",
"categoryId": "security",
"homepage": "https://github.com/hoshsadiq/adblock-nocoin-list/",
"source": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt"
},
"notracking_hosts_blocklists": {
"name": "The NoTracking blocklist",
"categoryId": "general",
"homepage": "https://github.com/notracking/hosts-blocklists",
"source": "https://raw.githubusercontent.com/notracking/hosts-blocklists/master/adblock/adblock.txt"
},
"oisd_basic": {
"name": "OISD Blocklist Basic",
"categoryId": "general",
"homepage": "https://oisd.nl/",
"source": "https://abp.oisd.nl/basic/"
},
"oisd_full": {
"name": "OISD Blocklist Full",
"categoryId": "general",
"homepage": "https://oisd.nl/",
"source": "https://abp.oisd.nl/"
},
"perflyst_dandelion_sprout_smart_tv_blocklist_for_adguard_home": {
"name": "Perflyst and Dandelion Sprout's Smart-TV Blocklist",
"categoryId": "other",
"homepage": "https://github.com/Perflyst/PiHoleBlocklist",
"source": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt"
},
"peter_lowe_list": {
"name": "Peter Lowe's Blocklist",
"categoryId": "general",
"homepage": "https://pgl.yoyo.org/adservers/",
"source": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=adblockplus\u0026showintro=1\u0026mimetype=plaintext"
},
"scam_blocklist_by_durablenapkin": {
"name": "Scam Blocklist by DurableNapkin",
"categoryId": "security",
"homepage": "https://github.com/durablenapkin/scamblocklist",
"source": "https://raw.githubusercontent.com/durablenapkin/scamblocklist/master/adguard.txt"
},
"staklerware_indicators_list": {
"name": "Stalkerware Indicators List",
"categoryId": "security",
"homepage": "https://github.com/AssoEchap/stalkerware-indicators",
"source": "https://raw.githubusercontent.com/AssoEchap/stalkerware-indicators/master/generated/hosts"
},
"steven_blacks_list": {
"name": "Steven Black's List",
"categoryId": "general",
"homepage": "https://github.com/StevenBlack/hosts",
"source": "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
},
"the_big_list_of_hacked_malware_web_sites": {
"name": "The Big List of Hacked Malware Web Sites",
"categoryId": "security",
"homepage": "https://github.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites",
"source": "https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/hosts"
},
"urlhaus_filter_online": {
"name": "Malicious URL Blocklist (URLHaus)",
"categoryId": "security",
"homepage": "https://gitlab.com/malware-filter/urlhaus-filter",
"source": "https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-agh.txt"
},
"windowsspyblocker_hosts_spy_rules": {
"name": "WindowsSpyBlocker - Hosts spy rules",
"categoryId": "other",
"homepage": "https://github.com/crazy-max/WindowsSpyBlocker",

20
scripts/README.md
View File

@ -205,3 +205,23 @@ code from [the repo][companiesrepo].
```
[companiesrepo]: https://github.com/AdguardTeam/companiesdb
## `vetted-filters/`: Vetted Filters Updater
A simple script that downloads and updates the vetted filtering list data from
AdGuard's [Hostlists Registry][reg].
Optional environment:
* `URL`: the URL of the index file. By default it's
`https://adguardteam.github.io/HostlistsRegistry/assets/filters.json`.
### Usage
```sh
go run ./scripts/vetted-filters/main.go
```
[reg]: https://github.com/AdguardTeam/HostlistsRegistry

2
scripts/make/go-lint.sh
View File

@ -229,7 +229,7 @@ gocyclo --over 13 ./internal/dhcpd ./internal/filtering/ ./internal/home/
gocyclo --over 10 ./internal/aghio/ ./internal/aghnet/ ./internal/aghos/\
./internal/aghtest/ ./internal/dnsforward/ ./internal/stats/\
./internal/tools/ ./internal/updater/ ./internal/next/ ./internal/version/\
./main.go
./scripts/vetted-filters/ ./main.go
ineffassign ./...

164
scripts/vetted-filters/main.go Normal file
View File

@ -0,0 +1,164 @@
// vetted-filters fetches the most recent Hostlists Registry index and
// transforms the filters from it to AdGuard Home's format.
package main
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"time"
"github.com/AdguardTeam/golibs/log"
"github.com/google/renameio/maybe"
)
func main() {
urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/filters.json"
if v, ok := os.LookupEnv("URL"); ok {
urlStr = v
}
// Validate the URL.
_, err := url.Parse(urlStr)
check(err)
c := &http.Client{
Timeout: 10 * time.Second,
}
resp, err := c.Get(urlStr)
check(err)
defer log.OnCloserError(resp.Body, log.ERROR)
if resp.StatusCode != http.StatusOK {
panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode))
}
hlFlt := &hlFilters{}
err = json.NewDecoder(resp.Body).Decode(hlFlt)
check(err)
aghFlt := &aghFilters{
Categories: map[string]*aghFiltersCategory{
"general": {
Name: "filter_category_general",
Description: "filter_category_general_desc",
},
"other": {
Name: "filter_category_other",
Description: "filter_category_other_desc",
},
"regional": {
Name: "filter_category_regional",
Description: "filter_category_regional_desc",
},
"security": {
Name: "filter_category_security",
Description: "filter_category_security_desc",
},
},
Filters: map[string]*aghFiltersFilter{},
}
for i, f := range hlFlt.Filters {
id := f.FilterID
cat := f.category()
if cat == "" {
log.Info("warning: filter %s at index %d does not have a fitting category", id, i)
}
aghFlt.Filters[id] = &aghFiltersFilter{
Name: f.Name,
CategoryID: cat,
Homepage: f.Homepage,
Source: f.SourceURL,
}
}
buf := &bytes.Buffer{}
_, _ = buf.WriteString(jsHeader)
enc := json.NewEncoder(buf)
enc.SetIndent("", " ")
err = enc.Encode(aghFlt)
check(err)
err = maybe.WriteFile("client/src/helpers/filters/filters.js", buf.Bytes(), 0o644)
check(err)
}
// jsHeader is the header for the generates JavaScript file. It informs the
// reader that the file is generated and disables some style-related eslint
// checks.
const jsHeader = `// Code generated by go run ./scripts/vetted-filters/main.go; DO NOT EDIT.
/* eslint quote-props: 'off', quotes: 'off', comma-dangle: 'off', semi: 'off' */
export default `
// check is a simple error-checking helper for scripts.
func check(err error) {
if err != nil {
panic(err)
}
}
// hlFilters is the JSON structure for the Hostlists Registry rule list index.
type hlFilters struct {
Filters []*hlFiltersFilter `json:"filters"`
}
// hlFiltersFilter is the JSON structure for a filter in the Hostlists Registry.
type hlFiltersFilter struct {
FilterID string `json:"filterId"`
Name string `json:"name"`
Homepage string `json:"homepage"`
SourceURL string `json:"sourceUrl"`
Tags []string `json:"tags"`
}
// category returns the AdGuard Home category for this filter. If there is no
// fitting category, cat is empty.
func (f *hlFiltersFilter) category() (cat string) {
for _, t := range f.Tags {
switch t {
case "purpose:general":
return "general"
case "purpose:other":
return "other"
case "purpose:regional":
return "regional"
case "purpose:security":
return "security"
}
}
return ""
}
// aghFilters is the JSON structure for AdGuard Home's list of vetted filtering
// rule list in file client/src/helpers/filters/filters.js.
type aghFilters struct {
Categories map[string]*aghFiltersCategory `json:"categories"`
Filters map[string]*aghFiltersFilter `json:"filters"`
}
// aghFiltersCategory is the JSON structure for a category in the vetted
// filtering rule list file.
type aghFiltersCategory struct {
Name string `json:"name"`
Description string `json:"description"`
}
// aghFiltersFilter is the JSON structure for a filter in the vetted filtering
// rule list file.
type aghFiltersFilter struct {
Name string `json:"name"`
CategoryID string `json:"categoryId"`
Homepage string `json:"homepage"`
Source string `json:"source"`
}