Pull request: 5471-new-trackers-db

Merge in DNS/adguard-home from 5471-new-trackers-db to master Squashed commit of the following: commit d90742c2902afd3424fb8721419231493610048b Merge: 5e87a1f2 a50a8abb Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Tue Feb 14 13:38:40 2023 +0300 Merge branch 'master' into 5471-new-trackers-db commit 5e87a1f25cd1d1dc52629d36e3d50d032ada1093 Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Mon Feb 13 18:38:54 2023 +0300 all: use new tracker source
2023-02-14 13:52:22 +03:00 · 2023-02-14 13:52:22 +03:00 · 3378d1c103
parent a50a8abb45
commit 3378d1c103
6 changed files with 268 additions and 238 deletions
--- a/client/src/helpers/constants.js
+++ b/client/src/helpers/constants.js
@ -53,7 +53,7 @@ export const STATUS_COLORS = {
 export const REPOSITORY = {
    URL: 'https://github.com/AdguardTeam/AdGuardHome',
    TRACKERS_DB:
-        'https://github.com/AdguardTeam/AdGuardHome/tree/master/client/src/helpers/trackers/adguard.json',
+        'https://github.com/AdguardTeam/AdGuardHome/tree/master/client/src/helpers/trackers/trackers.json',
    ISSUES: 'https://github.com/AdguardTeam/AdGuardHome/issues/new/choose',
 };

--- a/client/src/helpers/trackers/adguard.json
+++ b/client/src/helpers/trackers/adguard.json
@ -1,175 +0,0 @@
-{
-    "timeUpdated": "2021-12-15",
-    "categories": {
-        "0": "audio_video_player",
-        "1": "comments",
-        "2": "customer_interaction",
-        "3": "pornvertising",
-        "4": "advertising",
-        "5": "essential",
-        "6": "site_analytics",
-        "7": "social_media",
-        "8": "misc",
-        "9": "cdn",
-        "10": "hosting",
-        "11": "unknown",
-        "12": "extensions",
-        "13": "email",
-        "14": "consent",
-        "15": "telemetry",
-        "101": "mobile_analytics"
-    },
-    "trackers": {
-        "akamai_technologies": {
-            "name": "Akamai Technologies",
-            "categoryId": 9,
-            "url": "https://www.akamai.com/",
-            "companyId": "akamai"
-        },
-        "apple": {
-            "name": "Apple",
-            "categoryId": 8,
-            "url": "https://www.apple.com/",
-            "companyId": "apple"
-        },
-        "apple_ads": {
-            "name": "Apple Search Ads",
-            "categoryId": 4,
-            "url": "https://searchads.apple.com/",
-            "companyId": "apple"
-        },
-        "facebook_audience": {
-            "name": "Facebook Audience Network",
-            "categoryId": 4,
-            "url": "https://www.facebook.com/business/products/audience-network",
-            "companyId": "facebook"
-        },
-        "crashlytics": {
-            "name": "Crashlytics",
-            "categoryId": 101,
-            "url": "https://crashlytics.com/",
-            "companyId": null
-        },
-        "flurry": {
-            "name": "Flurry",
-            "categoryId": 101,
-            "url": "http://www.flurry.com/",
-            "companyId": "verizon"
-        },
-        "hockeyapp": {
-            "name": "HockeyApp",
-            "categoryId": 101,
-            "url": "https://hockeyapp.net/",
-            "companyId": null
-        },
-        "firebase": {
-            "name": "Firebase",
-            "categoryId": 101,
-            "url": "https://firebase.google.com/",
-            "companyId": "google"
-        },
-        "appsflyer": {
-            "name": "AppsFlyer",
-            "categoryId": 101,
-            "url": "https://www.appsflyer.com/",
-            "companyId": "appsflyer"
-        },
-        "yandex_appmetrica": {
-            "name": "Yandex AppMetrica",
-            "categoryId": 101,
-            "url": "https://appmetrica.yandex.com/",
-            "companyId": "yandex"
-        },
-        "adjust": {
-            "name": "Adjust",
-            "categoryId": 101,
-            "url": "https://www.adjust.com/",
-            "companyId": "adjust"
-        },
-        "branch": {
-            "name": "Branch.io",
-            "categoryId": 101,
-            "url": "https://branch.io/",
-            "companyId": "branch_metrics_inc"
-        },
-        "markmonitor": {
-            "name": "MarkMonitor",
-            "categoryId": 4,
-            "url": "https://www.markmonitor.com/",
-            "companyId": "markmonitor"
-        },
-        "appcenter": {
-            "name": "Microsoft App Center",
-            "categoryId": 5,
-            "url": "https://appcenter.ms/",
-            "companyId": null
-        },
-        "unity_ads": {
-            "name": "Unity Ads",
-            "categoryId": 4,
-            "url": "https://unity.com/solutions/mobile-business/monetize-your-game",
-            "companyId": null
-        },
-        "azure": {
-            "name": "Microsoft Azure",
-            "categoryId": 10,
-            "url": "https://azure.microsoft.com/",
-            "companyId": "microsoft"
-        },
-        "button": {
-            "name": "Button",
-            "categoryId": 4,
-            "url": "https://www.usebutton.com/",
-            "companyId": null
-        },
-        "netflix": {
-            "name": "Netflix",
-            "categoryId": 8,
-            "url": "https://www.netflix.com/",
-            "companyId": null
-        },
-        "mail.ru_banner": {
-            "name": "Mail.Ru Banner Network",
-            "categoryId": 4,
-            "url": "http://mail.ru/",
-            "companyId": "vk"
-        },
-        "mail.ru_counter": {
-            "name": "Mail.Ru Counter",
-            "categoryId": 2,
-            "url": "http://mail.ru/",
-            "companyId": "vk"
-        },
-        "mail.ru_group": {
-            "name": "Mail.Ru Group",
-            "categoryId": 7,
-            "url": "http://mail.ru/",
-            "companyId": "vk"
-        }
-    },
-    "trackerDomains": {
-        "akadns.net": "akamai_technologies",
-        "akamaiedge.net": "akamai_technologies",
-        "apple.com": "apple",
-        "apple.news": "apple",
-        "apple-dns.net": "apple",
-        "aaplimg.com": "apple",
-        "icloud.com": "apple",
-        "mzstatic.com": "apple",
-        "iadsdk.apple.com": "apple_ads",
-        "graph.facebook.com": "facebook_audience",
-        "crashlytics.com": "crashlytics",
-        "flurry.com": "flurry",
-        "hockeyapp.net": "hockeyapp",
-        "app-measurement.com": "firebase",
-        "appsflyer.com": "appsflyer",
-        "appmetrica.yandex.com": "yandex_appmetrica",
-        "adjust.com": "adjust",
-        "mobileapptracking.com": "branch",
-        "edgecastcdn.net": "markmonitor",
-        "appcenter.ms": "appcenter",
-        "unityads.unity3d.com": "unity_ads",
-        "azure.com": "azure",
-        "bttn.io": "button"
-    }
-}
--- a/client/src/helpers/trackers/trackers.js
+++ b/client/src/helpers/trackers/trackers.js
@ -1,6 +1,5 @@
-import whotracksmeDb from './whotracksme.json';
 import whotracksmeWebsites from './whotracksme_web.json';
-import adguardDb from './adguard.json';
+import trackersDb from './trackers.json';
 import { REPOSITORY } from '../constants';

 /**
@ -64,14 +63,25 @@ export const getSourceData = (trackerData) => {
 };

 /**
- * Gets tracker data in the specified database
+ * Converts the JSON string source into numeric source for AdGuard Home
+ * @param {TrackerData} trackerData tracker data
+ * @returns {number} source number
+ */
+const convertSource = (sourceStr) => {
+    if (!sourceStr || sourceStr !== 'AdGuard') {
+        return sources.WHOTRACKSME;
+    }
+
+    return sources.ADGUARD;
+};
+
+/**
+ * Gets tracker data from the trackers database
 *
 * @param {String} domainName domain name to check
- * @param {*} trackersDb trackers database
- * @param {number} source source ID
 * @returns {TrackerData} tracker data or null if no matching tracker found
 */
-const getTrackerDataFromDb = (domainName, trackersDb, source) => {
+export const getTrackerData = (domainName) => {
    if (!domainName) {
        return null;
    }
@ -88,7 +98,7 @@ const getTrackerDataFromDb = (domainName, trackersDb, source) => {
        if (trackerId) {
            const trackerData = trackersDb.trackers[trackerId];
            const categoryName = trackersDb.categories[trackerData.categoryId];
-            trackerData.source = source;
+            const source = convertSource(trackerData.source);
            const sourceData = getSourceData(trackerData);

            return {
@ -105,22 +115,3 @@ const getTrackerDataFromDb = (domainName, trackersDb, source) => {
    // No tracker found for the specified domain
    return null;
 };
-
-/**
- * Gets tracker data from the trackers database
- *
- * @param {String} domainName domain name to check
- * @returns {TrackerData} tracker data or null if no matching tracker found
- */
-export const getTrackerData = (domainName) => {
-    if (!domainName) {
-        return null;
-    }
-
-    let data = getTrackerDataFromDb(domainName, adguardDb, sources.ADGUARD);
-    if (!data) {
-        data = getTrackerDataFromDb(domainName, whotracksmeDb, sources.WHOTRACKSME);
-    }
-
-    return data;
-};
--- a/client/src/helpers/trackers/whotracksme.json
+++ b/client/src/helpers/trackers/whotracksme.json
@ -1,5 +1,5 @@
 {
-    "timeUpdated": "2022-10-15T00:14:03.765Z",
+    "timeUpdated": "2023-02-09T12:31:34.007Z",
    "categories": {
        "0": "audio_video_player",
        "1": "comments",
@ -16,7 +16,8 @@
        "12": "extensions",
        "13": "email",
        "14": "consent",
-        "15": "telemetry"
+        "15": "telemetry",
+        "101": "mobile_analytics"
    },
    "trackers": {
        "163": {
@ -872,10 +873,11 @@
            "companyId": "adgoto"
        },
        "adguard": {
-            "name": "Adguard",
-            "categoryId": 12,
+            "name": "AdGuard",
+            "categoryId": 8,
            "url": "https://adguard.com/",
-            "companyId": null
+            "companyId": "adguard",
+            "source": "AdGuard"
        },
        "adhands": {
            "name": "AdHands",
@ -951,9 +953,10 @@
        },
        "adjust": {
            "name": "Adjust",
-            "categoryId": 6,
+            "categoryId": 101,
            "url": "https://www.adjust.com/",
-            "companyId": "adjust"
+            "companyId": "adjust",
+            "source": "AdGuard"
        },
        "adk2": {
            "name": "adk2",
@ -2141,7 +2144,8 @@
            "name": "Akamai Technologies",
            "categoryId": 9,
            "url": "https://www.akamai.com/",
-            "companyId": "akamai"
+            "companyId": "akamai",
+            "source": "AdGuard"
        },
        "akamoihd.net": {
            "name": "akamoihd.net",
@ -2541,9 +2545,10 @@
        },
        "appsflyer": {
            "name": "AppsFlyer",
-            "categoryId": 6,
+            "categoryId": 101,
            "url": "https://www.appsflyer.com/",
-            "companyId": "appsflyer"
+            "companyId": "appsflyer",
+            "source": "AdGuard"
        },
        "apptv": {
            "name": "appTV",
@ -7095,9 +7100,10 @@
        },
        "flurry": {
            "name": "Flurry",
-            "categoryId": 6,
+            "categoryId": 101,
            "url": "http://www.flurry.com/",
-            "companyId": "verizon"
+            "companyId": "verizon",
+            "source": "AdGuard"
        },
        "flxone": {
            "name": "FLXONE",
@ -10493,19 +10499,22 @@
            "name": "Mail.Ru Banner Network",
            "categoryId": 4,
            "url": "http://mail.ru/",
-            "companyId": "megafon"
+            "companyId": "vk",
+            "source": "AdGuard"
        },
        "mail.ru_counter": {
            "name": "Mail.Ru Counter",
            "categoryId": 2,
-            "url": "https://corp.megafon.com/",
-            "companyId": "megafon"
+            "url": "http://mail.ru/",
+            "companyId": "vk",
+            "source": "AdGuard"
        },
        "mail.ru_group": {
            "name": "Mail.Ru Group",
            "categoryId": 7,
            "url": "http://mail.ru/",
-            "companyId": "megafon"
+            "companyId": "vk",
+            "source": "AdGuard"
        },
        "mailchimp_tracking": {
            "name": "MailChimp Tracking",
@ -10631,7 +10640,8 @@
            "name": "MarkMonitor",
            "categoryId": 4,
            "url": "https://www.markmonitor.com/",
-            "companyId": "markmonitor"
+            "companyId": "markmonitor",
+            "source": "AdGuard"
        },
        "marktest": {
            "name": "Marktest",
@ -11691,9 +11701,10 @@
        },
        "netflix": {
            "name": "Netflix",
-            "categoryId": 8,
-            "url": null,
-            "companyId": null
+            "categoryId": 0,
+            "url": "https://www.netflix.com/",
+            "companyId": "netflix",
+            "source": "AdGuard"
        },
        "netletix": {
            "name": "Netletix",
@ -17969,19 +17980,22 @@
            "name": "Vk.com",
            "categoryId": 7,
            "url": "https://vk.com/",
-            "companyId": "megafon"
+            "companyId": "vk",
+            "source": "AdGuard"
        },
        "vkontakte": {
            "name": "VKontakte",
            "categoryId": 7,
            "url": "https://vk.com/",
-            "companyId": "megafon"
+            "companyId": "vk",
+            "source": "AdGuard"
        },
        "vkontakte_widgets": {
            "name": "VKontakte Widgets",
            "categoryId": 7,
-            "url": "http://vk.com/developers.php",
-            "companyId": "megafon"
+            "url": "https://dev.vk.com/",
+            "companyId": "vk",
+            "source": "AdGuard"
        },
        "vntsm.com": {
            "name": "Venatus Media",
@ -19200,6 +19214,160 @@
            "categoryId": 4,
            "url": "http://www.zypmedia.com/",
            "companyId": "zypmedia"
+        },
+        "slack": {
+            "name": "Slack",
+            "categoryId": 8,
+            "url": "https://www.slack.com/",
+            "companyId": "salesforce",
+            "source": "AdGuard"
+        },
+        "apple": {
+            "name": "Apple",
+            "categoryId": 8,
+            "url": "https://www.apple.com/",
+            "companyId": "apple",
+            "source": "AdGuard"
+        },
+        "apple_ads": {
+            "name": "Apple Search Ads",
+            "categoryId": 4,
+            "url": "https://searchads.apple.com/",
+            "companyId": "apple",
+            "source": "AdGuard"
+        },
+        "facebook_audience": {
+            "name": "Facebook Audience Network",
+            "categoryId": 4,
+            "url": "https://www.facebook.com/business/products/audience-network",
+            "companyId": "facebook",
+            "source": "AdGuard"
+        },
+        "crashlytics": {
+            "name": "Crashlytics",
+            "categoryId": 101,
+            "url": "https://crashlytics.com/",
+            "companyId": null,
+            "source": "AdGuard"
+        },
+        "showrss": {
+            "name": "showRSS",
+            "categoryId": 8,
+            "url": "https://showrss.info/",
+            "companyId": "showrss",
+            "source": "AdGuard"
+        },
+        "hockeyapp": {
+            "name": "HockeyApp",
+            "categoryId": 101,
+            "url": "https://hockeyapp.net/",
+            "companyId": null,
+            "source": "AdGuard"
+        },
+        "google_trust_services": {
+            "name": "Google Trust Services",
+            "categoryId": 5,
+            "url": "https://pki.goog/",
+            "companyId": "google",
+            "source": "AdGuard"
+        },
+        "firebase": {
+            "name": "Firebase",
+            "categoryId": 101,
+            "url": "https://firebase.google.com/",
+            "companyId": "google",
+            "source": "AdGuard"
+        },
+        "yandex_appmetrica": {
+            "name": "Yandex AppMetrica",
+            "categoryId": 101,
+            "url": "https://appmetrica.yandex.com/",
+            "companyId": "yandex",
+            "source": "AdGuard"
+        },
+        "branch": {
+            "name": "Branch.io",
+            "categoryId": 101,
+            "url": "https://branch.io/",
+            "companyId": "branch_metrics_inc",
+            "source": "AdGuard"
+        },
+        "qualcomm": {
+            "name": "Qualcomm",
+            "categoryId": 8,
+            "url": "https://www.qualcomm.com/",
+            "companyId": "qualcomm",
+            "source": "AdGuard"
+        },
+        "solaredge": {
+            "name": "SolarEdge Technologies, Inc.",
+            "categoryId": 8,
+            "url": "https://www.solaredge.com/",
+            "companyId": "solaredge",
+            "source": "AdGuard"
+        },
+        "element": {
+            "name": "Element",
+            "categoryId": 7,
+            "url": "https://element.io/",
+            "companyId": "element",
+            "source": "AdGuard"
+        },
+        "outlook": {
+            "name": "Microsoft Outlook",
+            "categoryId": 13,
+            "url": "https://outlook.live.com/",
+            "companyId": "microsoft",
+            "source": "AdGuard"
+        },
+        "appcenter": {
+            "name": "Microsoft App Center",
+            "categoryId": 5,
+            "url": "https://appcenter.ms/",
+            "companyId": null,
+            "source": "AdGuard"
+        },
+        "unity_ads": {
+            "name": "Unity Ads",
+            "categoryId": 4,
+            "url": "https://unity.com/solutions/mobile-business/monetize-your-game",
+            "companyId": null,
+            "source": "AdGuard"
+        },
+        "azure": {
+            "name": "Microsoft Azure",
+            "categoryId": 10,
+            "url": "https://azure.microsoft.com/",
+            "companyId": "microsoft",
+            "source": "AdGuard"
+        },
+        "button": {
+            "name": "Button",
+            "categoryId": 4,
+            "url": "https://www.usebutton.com/",
+            "companyId": null,
+            "source": "AdGuard"
+        },
+        "lets_encrypt": {
+            "name": "Let's Encrypt",
+            "categoryId": 5,
+            "url": "https://letsencrypt.org/",
+            "companyId": "lets_encrypt",
+            "source": "AdGuard"
+        },
+        "plex": {
+            "name": "Plex",
+            "categoryId": 0,
+            "url": "https://www.plex.tv/",
+            "companyId": "plex",
+            "source": "AdGuard"
+        },
+        "matrix": {
+            "name": "Matrix",
+            "categoryId": 5,
+            "url": "https://matrix.org/",
+            "companyId": "matrix",
+            "source": "AdGuard"
        }
    },
    "trackerDomains": {
@ -20740,7 +20908,7 @@
        "facebook.net": "facebook",
        "fbcdn.net": "facebook_cdn",
        "fbsbx.com": "facebook_cdn",
-        "graph.facebook.com": "facebook_graph",
+        "graph.facebook.com": "facebook_audience",
        "facetz.net": "facetz.dca",
        "adsfac.eu": "facilitate_digital",
        "adsfac.net": "facilitate_digital",
@ -21273,7 +21441,7 @@
        "isocket.com": "isocket",
        "ispot.tv": "ispot.tv",
        "itineraire.info": "itineraire.info",
-        "apple.com": "itunes_link_maker",
+        "apple.com": "apple",
        "autolinkmaker.itunes.apple.com": "itunes_link_maker",
        "ity.im": "ity.im",
        "iubenda.com": "iubenda.com",
@ -21727,7 +21895,7 @@
        "azurewebsites.net": "microsoft",
        "cloudapp.net": "microsoft",
        "gfx.ms": "microsoft",
-        "live.com": "microsoft",
+        "live.com": "outlook",
        "microsoft.com": "microsoft",
        "microsoftonline-p.com": "microsoft",
        "microsoftonline.com": "microsoft",
@ -23529,6 +23697,54 @@
        "zukxd6fkxqn.com": "zukxd6fkxqn.com",
        "zwaar.net": "zwaar",
        "zwaar.org": "zwaar",
-        "extend.tv": "zypmedia"
+        "extend.tv": "zypmedia",
+        "slack.com": "slack",
+        "slackb.com": "slack",
+        "slack-edge.com": "slack",
+        "slack-imgs.com": "slack",
+        "adguard.app": "adguard",
+        "adguard.io": "adguard",
+        "adguard.org": "adguard",
+        "adguard-dns.com": "adguard",
+        "adguard-dns.io": "adguard",
+        "adguard-vpn.com": "adguard",
+        "adguardvpn.com": "adguard",
+        "adguard-vpn.online": "adguard",
+        "nflximg.com": "netflix",
+        "element.io": "element",
+        "riot.im": "element",
+        "akadns.net": "akamai_technologies",
+        "akamaiedge.net": "akamai_technologies",
+        "apple.news": "apple",
+        "apple-dns.net": "apple",
+        "aaplimg.com": "apple",
+        "icloud.com": "apple",
+        "icloud-content.com": "apple",
+        "mzstatic.com": "apple",
+        "matrix.org": "matrix",
+        "l-msedge.net": "microsoft",
+        "iadsdk.apple.com": "apple_ads",
+        "showrss.info": "showrss",
+        "solaredge.com": "solaredge",
+        "crashlytics.com": "crashlytics",
+        "flurry.com": "flurry",
+        "hockeyapp.net": "hockeyapp",
+        "app-measurement.com": "firebase",
+        "appmetrica.yandex.com": "yandex_appmetrica",
+        "letsencrypt.org": "lets_encrypt",
+        "lencr.org": "lets_encrypt",
+        "mobileapptracking.com": "branch",
+        "plex.tv": "plex",
+        "plex.direct": "plex",
+        "edgecastcdn.net": "markmonitor",
+        "appcenter.ms": "appcenter",
+        "unityads.unity3d.com": "unity_ads",
+        "azure.com": "azure",
+        "trafficmanager.net": "azure",
+        "hotmail.com": "outlook",
+        "bttn.io": "button",
+        "pki.goog": "google_trust_services",
+        "xtracloud.net": "qualcomm",
+        "qualcomm.com": "qualcomm"
    }
 }
--- a/scripts/README.md
+++ b/scripts/README.md
@ -205,7 +205,7 @@ code from [the repo][companiesrepo].
 ###  Usage

 ```sh
-( cd scripts/companiesdb && sh ./download.sh )
+sh ./scripts/companiesdb/download.sh
 ```

 [companiesrepo]: https://github.com/AdguardTeam/companiesdb
--- a/scripts/companiesdb/download.sh
+++ b/scripts/companiesdb/download.sh
@ -4,11 +4,9 @@ set -e -f -u -x

 # This script syncs companies DB that we bundle with AdGuard Home.  The source
 # for this database is https://github.com/AdguardTeam/companiesdb.
+#
+trackers_url='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/trackers.json'
+output='./client/src/helpers/trackers/trackers.json'
+readonly trackers_url output

-whotracksme='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/whotracksme.json'
-adguard='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/adguard.json'
-base_path='../../client/src/helpers/trackers'
-readonly whotracksme adguard base_path
-
-curl -o "${base_path}/whotracksme.json" -v "$whotracksme"
-curl -o "${base_path}/adguard.json"     -v "$adguard"
+curl -o "$output" -v "$trackers_url"