From 3378d1c10304c54f16f1466bd34715975fdbe4df Mon Sep 17 00:00:00 2001 From: Ainar Garipov Date: Tue, 14 Feb 2023 13:52:22 +0300 Subject: [PATCH] Pull request: 5471-new-trackers-db Merge in DNS/adguard-home from 5471-new-trackers-db to master Squashed commit of the following: commit d90742c2902afd3424fb8721419231493610048b Merge: 5e87a1f2 a50a8abb Author: Ainar Garipov Date: Tue Feb 14 13:38:40 2023 +0300 Merge branch 'master' into 5471-new-trackers-db commit 5e87a1f25cd1d1dc52629d36e3d50d032ada1093 Author: Ainar Garipov Date: Mon Feb 13 18:38:54 2023 +0300 all: use new tracker source --- client/src/helpers/constants.js | 2 +- client/src/helpers/trackers/adguard.json | 175 ----------- client/src/helpers/trackers/trackers.js | 43 ++- .../{whotracksme.json => trackers.json} | 272 ++++++++++++++++-- scripts/README.md | 2 +- scripts/companiesdb/download.sh | 12 +- 6 files changed, 268 insertions(+), 238 deletions(-) delete mode 100644 client/src/helpers/trackers/adguard.json rename client/src/helpers/trackers/{whotracksme.json => trackers.json} (98%) diff --git a/client/src/helpers/constants.js b/client/src/helpers/constants.js index 4f6803cb..61160d58 100644 --- a/client/src/helpers/constants.js +++ b/client/src/helpers/constants.js @@ -53,7 +53,7 @@ export const STATUS_COLORS = { export const REPOSITORY = { URL: 'https://github.com/AdguardTeam/AdGuardHome', TRACKERS_DB: - 'https://github.com/AdguardTeam/AdGuardHome/tree/master/client/src/helpers/trackers/adguard.json', + 'https://github.com/AdguardTeam/AdGuardHome/tree/master/client/src/helpers/trackers/trackers.json', ISSUES: 'https://github.com/AdguardTeam/AdGuardHome/issues/new/choose', }; diff --git a/client/src/helpers/trackers/adguard.json b/client/src/helpers/trackers/adguard.json deleted file mode 100644 index 1e3bda05..00000000 --- a/client/src/helpers/trackers/adguard.json +++ /dev/null @@ -1,175 +0,0 @@ -{ - "timeUpdated": "2021-12-15", - "categories": { - "0": "audio_video_player", - "1": "comments", - "2": "customer_interaction", - "3": "pornvertising", - "4": "advertising", - "5": "essential", - "6": "site_analytics", - "7": "social_media", - "8": "misc", - "9": "cdn", - "10": "hosting", - "11": "unknown", - "12": "extensions", - "13": "email", - "14": "consent", - "15": "telemetry", - "101": "mobile_analytics" - }, - "trackers": { - "akamai_technologies": { - "name": "Akamai Technologies", - "categoryId": 9, - "url": "https://www.akamai.com/", - "companyId": "akamai" - }, - "apple": { - "name": "Apple", - "categoryId": 8, - "url": "https://www.apple.com/", - "companyId": "apple" - }, - "apple_ads": { - "name": "Apple Search Ads", - "categoryId": 4, - "url": "https://searchads.apple.com/", - "companyId": "apple" - }, - "facebook_audience": { - "name": "Facebook Audience Network", - "categoryId": 4, - "url": "https://www.facebook.com/business/products/audience-network", - "companyId": "facebook" - }, - "crashlytics": { - "name": "Crashlytics", - "categoryId": 101, - "url": "https://crashlytics.com/", - "companyId": null - }, - "flurry": { - "name": "Flurry", - "categoryId": 101, - "url": "http://www.flurry.com/", - "companyId": "verizon" - }, - "hockeyapp": { - "name": "HockeyApp", - "categoryId": 101, - "url": "https://hockeyapp.net/", - "companyId": null - }, - "firebase": { - "name": "Firebase", - "categoryId": 101, - "url": "https://firebase.google.com/", - "companyId": "google" - }, - "appsflyer": { - "name": "AppsFlyer", - "categoryId": 101, - "url": "https://www.appsflyer.com/", - "companyId": "appsflyer" - }, - "yandex_appmetrica": { - "name": "Yandex AppMetrica", - "categoryId": 101, - "url": "https://appmetrica.yandex.com/", - "companyId": "yandex" - }, - "adjust": { - "name": "Adjust", - "categoryId": 101, - "url": "https://www.adjust.com/", - "companyId": "adjust" - }, - "branch": { - "name": "Branch.io", - "categoryId": 101, - "url": "https://branch.io/", - "companyId": "branch_metrics_inc" - }, - "markmonitor": { - "name": "MarkMonitor", - "categoryId": 4, - "url": "https://www.markmonitor.com/", - "companyId": "markmonitor" - }, - "appcenter": { - "name": "Microsoft App Center", - "categoryId": 5, - "url": "https://appcenter.ms/", - "companyId": null - }, - "unity_ads": { - "name": "Unity Ads", - "categoryId": 4, - "url": "https://unity.com/solutions/mobile-business/monetize-your-game", - "companyId": null - }, - "azure": { - "name": "Microsoft Azure", - "categoryId": 10, - "url": "https://azure.microsoft.com/", - "companyId": "microsoft" - }, - "button": { - "name": "Button", - "categoryId": 4, - "url": "https://www.usebutton.com/", - "companyId": null - }, - "netflix": { - "name": "Netflix", - "categoryId": 8, - "url": "https://www.netflix.com/", - "companyId": null - }, - "mail.ru_banner": { - "name": "Mail.Ru Banner Network", - "categoryId": 4, - "url": "http://mail.ru/", - "companyId": "vk" - }, - "mail.ru_counter": { - "name": "Mail.Ru Counter", - "categoryId": 2, - "url": "http://mail.ru/", - "companyId": "vk" - }, - "mail.ru_group": { - "name": "Mail.Ru Group", - "categoryId": 7, - "url": "http://mail.ru/", - "companyId": "vk" - } - }, - "trackerDomains": { - "akadns.net": "akamai_technologies", - "akamaiedge.net": "akamai_technologies", - "apple.com": "apple", - "apple.news": "apple", - "apple-dns.net": "apple", - "aaplimg.com": "apple", - "icloud.com": "apple", - "mzstatic.com": "apple", - "iadsdk.apple.com": "apple_ads", - "graph.facebook.com": "facebook_audience", - "crashlytics.com": "crashlytics", - "flurry.com": "flurry", - "hockeyapp.net": "hockeyapp", - "app-measurement.com": "firebase", - "appsflyer.com": "appsflyer", - "appmetrica.yandex.com": "yandex_appmetrica", - "adjust.com": "adjust", - "mobileapptracking.com": "branch", - "edgecastcdn.net": "markmonitor", - "appcenter.ms": "appcenter", - "unityads.unity3d.com": "unity_ads", - "azure.com": "azure", - "bttn.io": "button" - } -} diff --git a/client/src/helpers/trackers/trackers.js b/client/src/helpers/trackers/trackers.js index 5327d18e..1e41a4bc 100644 --- a/client/src/helpers/trackers/trackers.js +++ b/client/src/helpers/trackers/trackers.js @@ -1,6 +1,5 @@ -import whotracksmeDb from './whotracksme.json'; import whotracksmeWebsites from './whotracksme_web.json'; -import adguardDb from './adguard.json'; +import trackersDb from './trackers.json'; import { REPOSITORY } from '../constants'; /** @@ -64,14 +63,25 @@ export const getSourceData = (trackerData) => { }; /** - * Gets tracker data in the specified database + * Converts the JSON string source into numeric source for AdGuard Home + * @param {TrackerData} trackerData tracker data + * @returns {number} source number + */ +const convertSource = (sourceStr) => { + if (!sourceStr || sourceStr !== 'AdGuard') { + return sources.WHOTRACKSME; + } + + return sources.ADGUARD; +}; + +/** + * Gets tracker data from the trackers database * * @param {String} domainName domain name to check - * @param {*} trackersDb trackers database - * @param {number} source source ID * @returns {TrackerData} tracker data or null if no matching tracker found */ -const getTrackerDataFromDb = (domainName, trackersDb, source) => { +export const getTrackerData = (domainName) => { if (!domainName) { return null; } @@ -88,7 +98,7 @@ const getTrackerDataFromDb = (domainName, trackersDb, source) => { if (trackerId) { const trackerData = trackersDb.trackers[trackerId]; const categoryName = trackersDb.categories[trackerData.categoryId]; - trackerData.source = source; + const source = convertSource(trackerData.source); const sourceData = getSourceData(trackerData); return { @@ -105,22 +115,3 @@ const getTrackerDataFromDb = (domainName, trackersDb, source) => { // No tracker found for the specified domain return null; }; - -/** - * Gets tracker data from the trackers database - * - * @param {String} domainName domain name to check - * @returns {TrackerData} tracker data or null if no matching tracker found - */ -export const getTrackerData = (domainName) => { - if (!domainName) { - return null; - } - - let data = getTrackerDataFromDb(domainName, adguardDb, sources.ADGUARD); - if (!data) { - data = getTrackerDataFromDb(domainName, whotracksmeDb, sources.WHOTRACKSME); - } - - return data; -}; diff --git a/client/src/helpers/trackers/whotracksme.json b/client/src/helpers/trackers/trackers.json similarity index 98% rename from client/src/helpers/trackers/whotracksme.json rename to client/src/helpers/trackers/trackers.json index bc8e7fed..485b759e 100644 --- a/client/src/helpers/trackers/whotracksme.json +++ b/client/src/helpers/trackers/trackers.json @@ -1,5 +1,5 @@ { - "timeUpdated": "2022-10-15T00:14:03.765Z", + "timeUpdated": "2023-02-09T12:31:34.007Z", "categories": { "0": "audio_video_player", "1": "comments", @@ -16,7 +16,8 @@ "12": "extensions", "13": "email", "14": "consent", - "15": "telemetry" + "15": "telemetry", + "101": "mobile_analytics" }, "trackers": { "163": { @@ -872,10 +873,11 @@ "companyId": "adgoto" }, "adguard": { - "name": "Adguard", - "categoryId": 12, + "name": "AdGuard", + "categoryId": 8, "url": "https://adguard.com/", - "companyId": null + "companyId": "adguard", + "source": "AdGuard" }, "adhands": { "name": "AdHands", @@ -951,9 +953,10 @@ }, "adjust": { "name": "Adjust", - "categoryId": 6, + "categoryId": 101, "url": "https://www.adjust.com/", - "companyId": "adjust" + "companyId": "adjust", + "source": "AdGuard" }, "adk2": { "name": "adk2", @@ -2141,7 +2144,8 @@ "name": "Akamai Technologies", "categoryId": 9, "url": "https://www.akamai.com/", - "companyId": "akamai" + "companyId": "akamai", + "source": "AdGuard" }, "akamoihd.net": { "name": "akamoihd.net", @@ -2541,9 +2545,10 @@ }, "appsflyer": { "name": "AppsFlyer", - "categoryId": 6, + "categoryId": 101, "url": "https://www.appsflyer.com/", - "companyId": "appsflyer" + "companyId": "appsflyer", + "source": "AdGuard" }, "apptv": { "name": "appTV", @@ -7095,9 +7100,10 @@ }, "flurry": { "name": "Flurry", - "categoryId": 6, + "categoryId": 101, "url": "http://www.flurry.com/", - "companyId": "verizon" + "companyId": "verizon", + "source": "AdGuard" }, "flxone": { "name": "FLXONE", @@ -10493,19 +10499,22 @@ "name": "Mail.Ru Banner Network", "categoryId": 4, "url": "http://mail.ru/", - "companyId": "megafon" + "companyId": "vk", + "source": "AdGuard" }, "mail.ru_counter": { "name": "Mail.Ru Counter", "categoryId": 2, - "url": "https://corp.megafon.com/", - "companyId": "megafon" + "url": "http://mail.ru/", + "companyId": "vk", + "source": "AdGuard" }, "mail.ru_group": { "name": "Mail.Ru Group", "categoryId": 7, "url": "http://mail.ru/", - "companyId": "megafon" + "companyId": "vk", + "source": "AdGuard" }, "mailchimp_tracking": { "name": "MailChimp Tracking", @@ -10631,7 +10640,8 @@ "name": "MarkMonitor", "categoryId": 4, "url": "https://www.markmonitor.com/", - "companyId": "markmonitor" + "companyId": "markmonitor", + "source": "AdGuard" }, "marktest": { "name": "Marktest", @@ -11691,9 +11701,10 @@ }, "netflix": { "name": "Netflix", - "categoryId": 8, - "url": null, - "companyId": null + "categoryId": 0, + "url": "https://www.netflix.com/", + "companyId": "netflix", + "source": "AdGuard" }, "netletix": { "name": "Netletix", @@ -17969,19 +17980,22 @@ "name": "Vk.com", "categoryId": 7, "url": "https://vk.com/", - "companyId": "megafon" + "companyId": "vk", + "source": "AdGuard" }, "vkontakte": { "name": "VKontakte", "categoryId": 7, "url": "https://vk.com/", - "companyId": "megafon" + "companyId": "vk", + "source": "AdGuard" }, "vkontakte_widgets": { "name": "VKontakte Widgets", "categoryId": 7, - "url": "http://vk.com/developers.php", - "companyId": "megafon" + "url": "https://dev.vk.com/", + "companyId": "vk", + "source": "AdGuard" }, "vntsm.com": { "name": "Venatus Media", @@ -19200,6 +19214,160 @@ "categoryId": 4, "url": "http://www.zypmedia.com/", "companyId": "zypmedia" + }, + "slack": { + "name": "Slack", + "categoryId": 8, + "url": "https://www.slack.com/", + "companyId": "salesforce", + "source": "AdGuard" + }, + "apple": { + "name": "Apple", + "categoryId": 8, + "url": "https://www.apple.com/", + "companyId": "apple", + "source": "AdGuard" + }, + "apple_ads": { + "name": "Apple Search Ads", + "categoryId": 4, + "url": "https://searchads.apple.com/", + "companyId": "apple", + "source": "AdGuard" + }, + "facebook_audience": { + "name": "Facebook Audience Network", + "categoryId": 4, + "url": "https://www.facebook.com/business/products/audience-network", + "companyId": "facebook", + "source": "AdGuard" + }, + "crashlytics": { + "name": "Crashlytics", + "categoryId": 101, + "url": "https://crashlytics.com/", + "companyId": null, + "source": "AdGuard" + }, + "showrss": { + "name": "showRSS", + "categoryId": 8, + "url": "https://showrss.info/", + "companyId": "showrss", + "source": "AdGuard" + }, + "hockeyapp": { + "name": "HockeyApp", + "categoryId": 101, + "url": "https://hockeyapp.net/", + "companyId": null, + "source": "AdGuard" + }, + "google_trust_services": { + "name": "Google Trust Services", + "categoryId": 5, + "url": "https://pki.goog/", + "companyId": "google", + "source": "AdGuard" + }, + "firebase": { + "name": "Firebase", + "categoryId": 101, + "url": "https://firebase.google.com/", + "companyId": "google", + "source": "AdGuard" + }, + "yandex_appmetrica": { + "name": "Yandex AppMetrica", + "categoryId": 101, + "url": "https://appmetrica.yandex.com/", + "companyId": "yandex", + "source": "AdGuard" + }, + "branch": { + "name": "Branch.io", + "categoryId": 101, + "url": "https://branch.io/", + "companyId": "branch_metrics_inc", + "source": "AdGuard" + }, + "qualcomm": { + "name": "Qualcomm", + "categoryId": 8, + "url": "https://www.qualcomm.com/", + "companyId": "qualcomm", + "source": "AdGuard" + }, + "solaredge": { + "name": "SolarEdge Technologies, Inc.", + "categoryId": 8, + "url": "https://www.solaredge.com/", + "companyId": "solaredge", + "source": "AdGuard" + }, + "element": { + "name": "Element", + "categoryId": 7, + "url": "https://element.io/", + "companyId": "element", + "source": "AdGuard" + }, + "outlook": { + "name": "Microsoft Outlook", + "categoryId": 13, + "url": "https://outlook.live.com/", + "companyId": "microsoft", + "source": "AdGuard" + }, + "appcenter": { + "name": "Microsoft App Center", + "categoryId": 5, + "url": "https://appcenter.ms/", + "companyId": null, + "source": "AdGuard" + }, + "unity_ads": { + "name": "Unity Ads", + "categoryId": 4, + "url": "https://unity.com/solutions/mobile-business/monetize-your-game", + "companyId": null, + "source": "AdGuard" + }, + "azure": { + "name": "Microsoft Azure", + "categoryId": 10, + "url": "https://azure.microsoft.com/", + "companyId": "microsoft", + "source": "AdGuard" + }, + "button": { + "name": "Button", + "categoryId": 4, + "url": "https://www.usebutton.com/", + "companyId": null, + "source": "AdGuard" + }, + "lets_encrypt": { + "name": "Let's Encrypt", + "categoryId": 5, + "url": "https://letsencrypt.org/", + "companyId": "lets_encrypt", + "source": "AdGuard" + }, + "plex": { + "name": "Plex", + "categoryId": 0, + "url": "https://www.plex.tv/", + "companyId": "plex", + "source": "AdGuard" + }, + "matrix": { + "name": "Matrix", + "categoryId": 5, + "url": "https://matrix.org/", + "companyId": "matrix", + "source": "AdGuard" } }, "trackerDomains": { @@ -20740,7 +20908,7 @@ "facebook.net": "facebook", "fbcdn.net": "facebook_cdn", "fbsbx.com": "facebook_cdn", - "graph.facebook.com": "facebook_graph", + "graph.facebook.com": "facebook_audience", "facetz.net": "facetz.dca", "adsfac.eu": "facilitate_digital", "adsfac.net": "facilitate_digital", @@ -21273,7 +21441,7 @@ "isocket.com": "isocket", "ispot.tv": "ispot.tv", "itineraire.info": "itineraire.info", - "apple.com": "itunes_link_maker", + "apple.com": "apple", "autolinkmaker.itunes.apple.com": "itunes_link_maker", "ity.im": "ity.im", "iubenda.com": "iubenda.com", @@ -21727,7 +21895,7 @@ "azurewebsites.net": "microsoft", "cloudapp.net": "microsoft", "gfx.ms": "microsoft", - "live.com": "microsoft", + "live.com": "outlook", "microsoft.com": "microsoft", "microsoftonline-p.com": "microsoft", "microsoftonline.com": "microsoft", @@ -23529,6 +23697,54 @@ "zukxd6fkxqn.com": "zukxd6fkxqn.com", "zwaar.net": "zwaar", "zwaar.org": "zwaar", - "extend.tv": "zypmedia" + "extend.tv": "zypmedia", + "slack.com": "slack", + "slackb.com": "slack", + "slack-edge.com": "slack", + "slack-imgs.com": "slack", + "adguard.app": "adguard", + "adguard.io": "adguard", + "adguard.org": "adguard", + "adguard-dns.com": "adguard", + "adguard-dns.io": "adguard", + "adguard-vpn.com": "adguard", + "adguardvpn.com": "adguard", + "adguard-vpn.online": "adguard", + "nflximg.com": "netflix", + "element.io": "element", + "riot.im": "element", + "akadns.net": "akamai_technologies", + "akamaiedge.net": "akamai_technologies", + "apple.news": "apple", + "apple-dns.net": "apple", + "aaplimg.com": "apple", + "icloud.com": "apple", + "icloud-content.com": "apple", + "mzstatic.com": "apple", + "matrix.org": "matrix", + "l-msedge.net": "microsoft", + "iadsdk.apple.com": "apple_ads", + "showrss.info": "showrss", + "solaredge.com": "solaredge", + "crashlytics.com": "crashlytics", + "flurry.com": "flurry", + "hockeyapp.net": "hockeyapp", + "app-measurement.com": "firebase", + "appmetrica.yandex.com": "yandex_appmetrica", + "letsencrypt.org": "lets_encrypt", + "lencr.org": "lets_encrypt", + "mobileapptracking.com": "branch", + "plex.tv": "plex", + "plex.direct": "plex", + "edgecastcdn.net": "markmonitor", + "appcenter.ms": "appcenter", + "unityads.unity3d.com": "unity_ads", + "azure.com": "azure", + "trafficmanager.net": "azure", + "hotmail.com": "outlook", + "bttn.io": "button", + "pki.goog": "google_trust_services", + "xtracloud.net": "qualcomm", + "qualcomm.com": "qualcomm" } } \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md index 460f5067..5b7475bb 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -205,7 +205,7 @@ code from [the repo][companiesrepo]. ### Usage ```sh -( cd scripts/companiesdb && sh ./download.sh ) +sh ./scripts/companiesdb/download.sh ``` [companiesrepo]: https://github.com/AdguardTeam/companiesdb diff --git a/scripts/companiesdb/download.sh b/scripts/companiesdb/download.sh index e56166e0..8aa8c8cb 100755 --- a/scripts/companiesdb/download.sh +++ b/scripts/companiesdb/download.sh @@ -4,11 +4,9 @@ set -e -f -u -x # This script syncs companies DB that we bundle with AdGuard Home. The source # for this database is https://github.com/AdguardTeam/companiesdb. +# +trackers_url='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/trackers.json' +output='./client/src/helpers/trackers/trackers.json' +readonly trackers_url output -whotracksme='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/whotracksme.json' -adguard='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/adguard.json' -base_path='../../client/src/helpers/trackers' -readonly whotracksme adguard base_path - -curl -o "${base_path}/whotracksme.json" -v "$whotracksme" -curl -o "${base_path}/adguard.json" -v "$adguard" +curl -o "$output" -v "$trackers_url"