Pull request: 5471-new-trackers-db

Merge in DNS/adguard-home from 5471-new-trackers-db to master

Squashed commit of the following:

commit d90742c2902afd3424fb8721419231493610048b
Merge: 5e87a1f2 a50a8abb
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Feb 14 13:38:40 2023 +0300

    Merge branch 'master' into 5471-new-trackers-db

commit 5e87a1f25cd1d1dc52629d36e3d50d032ada1093
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Feb 13 18:38:54 2023 +0300

    all: use new tracker source

client/src/helpers/constants.js

@@ -53,7 +53,7 @@ export const STATUS_COLORS = {
 export const REPOSITORY = {
     URL: 'https://github.com/AdguardTeam/AdGuardHome',
     TRACKERS_DB:
-        'https://github.com/AdguardTeam/AdGuardHome/tree/master/client/src/helpers/trackers/adguard.json',
+        'https://github.com/AdguardTeam/AdGuardHome/tree/master/client/src/helpers/trackers/trackers.json',
     ISSUES: 'https://github.com/AdguardTeam/AdGuardHome/issues/new/choose',
 };
 

client/src/helpers/trackers/adguard.json

@@ -1,175 +0,0 @@
-{
-    "timeUpdated": "2021-12-15",
-    "categories": {
-        "0": "audio_video_player",
-        "1": "comments",
-        "2": "customer_interaction",
-        "3": "pornvertising",
-        "4": "advertising",
-        "5": "essential",
-        "6": "site_analytics",
-        "7": "social_media",
-        "8": "misc",
-        "9": "cdn",
-        "10": "hosting",
-        "11": "unknown",
-        "12": "extensions",
-        "13": "email",
-        "14": "consent",
-        "15": "telemetry",
-        "101": "mobile_analytics"
-    },
-    "trackers": {
-        "akamai_technologies": {
-            "name": "Akamai Technologies",
-            "categoryId": 9,
-            "url": "https://www.akamai.com/",
-            "companyId": "akamai"
-        },
-        "apple": {
-            "name": "Apple",
-            "categoryId": 8,
-            "url": "https://www.apple.com/",
-            "companyId": "apple"
-        },
-        "apple_ads": {
-            "name": "Apple Search Ads",
-            "categoryId": 4,
-            "url": "https://searchads.apple.com/",
-            "companyId": "apple"
-        },
-        "facebook_audience": {
-            "name": "Facebook Audience Network",
-            "categoryId": 4,
-            "url": "https://www.facebook.com/business/products/audience-network",
-            "companyId": "facebook"
-        },
-        "crashlytics": {
-            "name": "Crashlytics",
-            "categoryId": 101,
-            "url": "https://crashlytics.com/",
-            "companyId": null
-        },
-        "flurry": {
-            "name": "Flurry",
-            "categoryId": 101,
-            "url": "http://www.flurry.com/",
-            "companyId": "verizon"
-        },
-        "hockeyapp": {
-            "name": "HockeyApp",
-            "categoryId": 101,
-            "url": "https://hockeyapp.net/",
-            "companyId": null
-        },
-        "firebase": {
-            "name": "Firebase",
-            "categoryId": 101,
-            "url": "https://firebase.google.com/",
-            "companyId": "google"
-        },
-        "appsflyer": {
-            "name": "AppsFlyer",
-            "categoryId": 101,
-            "url": "https://www.appsflyer.com/",
-            "companyId": "appsflyer"
-        },
-        "yandex_appmetrica": {
-            "name": "Yandex AppMetrica",
-            "categoryId": 101,
-            "url": "https://appmetrica.yandex.com/",
-            "companyId": "yandex"
-        },
-        "adjust": {
-            "name": "Adjust",
-            "categoryId": 101,
-            "url": "https://www.adjust.com/",
-            "companyId": "adjust"
-        },
-        "branch": {
-            "name": "Branch.io",
-            "categoryId": 101,
-            "url": "https://branch.io/",
-            "companyId": "branch_metrics_inc"
-        },
-        "markmonitor": {
-            "name": "MarkMonitor",
-            "categoryId": 4,
-            "url": "https://www.markmonitor.com/",
-            "companyId": "markmonitor"
-        },
-        "appcenter": {
-            "name": "Microsoft App Center",
-            "categoryId": 5,
-            "url": "https://appcenter.ms/",
-            "companyId": null
-        },
-        "unity_ads": {
-            "name": "Unity Ads",
-            "categoryId": 4,
-            "url": "https://unity.com/solutions/mobile-business/monetize-your-game",
-            "companyId": null
-        },
-        "azure": {
-            "name": "Microsoft Azure",
-            "categoryId": 10,
-            "url": "https://azure.microsoft.com/",
-            "companyId": "microsoft"
-        },
-        "button": {
-            "name": "Button",
-            "categoryId": 4,
-            "url": "https://www.usebutton.com/",
-            "companyId": null
-        },
-        "netflix": {
-            "name": "Netflix",
-            "categoryId": 8,
-            "url": "https://www.netflix.com/",
-            "companyId": null
-        },
-        "mail.ru_banner": {
-            "name": "Mail.Ru Banner Network",
-            "categoryId": 4,
-            "url": "http://mail.ru/",
-            "companyId": "vk"
-        },
-        "mail.ru_counter": {
-            "name": "Mail.Ru Counter",
-            "categoryId": 2,
-            "url": "http://mail.ru/",
-            "companyId": "vk"
-        },
-        "mail.ru_group": {
-            "name": "Mail.Ru Group",
-            "categoryId": 7,
-            "url": "http://mail.ru/",
-            "companyId": "vk"
-        }
-    },
-    "trackerDomains": {
-        "akadns.net": "akamai_technologies",
-        "akamaiedge.net": "akamai_technologies",
-        "apple.com": "apple",
-        "apple.news": "apple",
-        "apple-dns.net": "apple",
-        "aaplimg.com": "apple",
-        "icloud.com": "apple",
-        "mzstatic.com": "apple",
-        "iadsdk.apple.com": "apple_ads",
-        "graph.facebook.com": "facebook_audience",
-        "crashlytics.com": "crashlytics",
-        "flurry.com": "flurry",
-        "hockeyapp.net": "hockeyapp",
-        "app-measurement.com": "firebase",
-        "appsflyer.com": "appsflyer",
-        "appmetrica.yandex.com": "yandex_appmetrica",
-        "adjust.com": "adjust",
-        "mobileapptracking.com": "branch",
-        "edgecastcdn.net": "markmonitor",
-        "appcenter.ms": "appcenter",
-        "unityads.unity3d.com": "unity_ads",
-        "azure.com": "azure",
-        "bttn.io": "button"
-    }
-}

client/src/helpers/trackers/trackers.js

@@ -1,6 +1,5 @@
-import whotracksmeDb from './whotracksme.json';
 import whotracksmeWebsites from './whotracksme_web.json';
-import adguardDb from './adguard.json';
+import trackersDb from './trackers.json';
 import { REPOSITORY } from '../constants';
 
 /**
@@ -64,14 +63,25 @@ export const getSourceData = (trackerData) => {
 };
 
 /**
- * Gets tracker data in the specified database
+ * Converts the JSON string source into numeric source for AdGuard Home
+ * @param {TrackerData} trackerData tracker data
+ * @returns {number} source number
+ */
+const convertSource = (sourceStr) => {
+    if (!sourceStr || sourceStr !== 'AdGuard') {
+        return sources.WHOTRACKSME;
+    }
+
+    return sources.ADGUARD;
+};
+
+/**
+ * Gets tracker data from the trackers database
  *
  * @param {String} domainName domain name to check
- * @param {*} trackersDb trackers database
- * @param {number} source source ID
  * @returns {TrackerData} tracker data or null if no matching tracker found
  */
-const getTrackerDataFromDb = (domainName, trackersDb, source) => {
+export const getTrackerData = (domainName) => {
     if (!domainName) {
         return null;
     }
@@ -88,7 +98,7 @@ const getTrackerDataFromDb = (domainName, trackersDb, source) => {
         if (trackerId) {
             const trackerData = trackersDb.trackers[trackerId];
             const categoryName = trackersDb.categories[trackerData.categoryId];
-            trackerData.source = source;
+            const source = convertSource(trackerData.source);
             const sourceData = getSourceData(trackerData);
 
             return {
@@ -105,22 +115,3 @@ const getTrackerDataFromDb = (domainName, trackersDb, source) => {
     // No tracker found for the specified domain
     return null;
 };
-
-/**
- * Gets tracker data from the trackers database
- *
- * @param {String} domainName domain name to check
- * @returns {TrackerData} tracker data or null if no matching tracker found
- */
-export const getTrackerData = (domainName) => {
-    if (!domainName) {
-        return null;
-    }
-
-    let data = getTrackerDataFromDb(domainName, adguardDb, sources.ADGUARD);
-    if (!data) {
-        data = getTrackerDataFromDb(domainName, whotracksmeDb, sources.WHOTRACKSME);
-    }
-
-    return data;
-};

client/src/helpers/trackers/trackers.json

@@ -1,5 +1,5 @@
 {
-    "timeUpdated": "2022-10-15T00:14:03.765Z",
+    "timeUpdated": "2023-02-09T12:31:34.007Z",
     "categories": {
         "0": "audio_video_player",
         "1": "comments",
@@ -16,7 +16,8 @@
         "12": "extensions",
         "13": "email",
         "14": "consent",
-        "15": "telemetry"
+        "15": "telemetry",
+        "101": "mobile_analytics"
     },
     "trackers": {
         "163": {
@@ -872,10 +873,11 @@
             "companyId": "adgoto"
         },
         "adguard": {
-            "name": "Adguard",
+            "name": "AdGuard",
-            "categoryId": 12,
+            "categoryId": 8,
             "url": "https://adguard.com/",
-            "companyId": null
+            "companyId": "adguard",
+            "source": "AdGuard"
         },
         "adhands": {
             "name": "AdHands",
@@ -951,9 +953,10 @@
         },
         "adjust": {
             "name": "Adjust",
-            "categoryId": 6,
+            "categoryId": 101,
             "url": "https://www.adjust.com/",
-            "companyId": "adjust"
+            "companyId": "adjust",
+            "source": "AdGuard"
         },
         "adk2": {
             "name": "adk2",
@@ -2141,7 +2144,8 @@
             "name": "Akamai Technologies",
             "categoryId": 9,
             "url": "https://www.akamai.com/",
-            "companyId": "akamai"
+            "companyId": "akamai",
+            "source": "AdGuard"
         },
         "akamoihd.net": {
             "name": "akamoihd.net",
@@ -2541,9 +2545,10 @@
         },
         "appsflyer": {
             "name": "AppsFlyer",
-            "categoryId": 6,
+            "categoryId": 101,
             "url": "https://www.appsflyer.com/",
-            "companyId": "appsflyer"
+            "companyId": "appsflyer",
+            "source": "AdGuard"
         },
         "apptv": {
             "name": "appTV",
@@ -7095,9 +7100,10 @@
         },
         "flurry": {
             "name": "Flurry",
-            "categoryId": 6,
+            "categoryId": 101,
             "url": "http://www.flurry.com/",
-            "companyId": "verizon"
+            "companyId": "verizon",
+            "source": "AdGuard"
         },
         "flxone": {
             "name": "FLXONE",
@@ -10493,19 +10499,22 @@
             "name": "Mail.Ru Banner Network",
             "categoryId": 4,
             "url": "http://mail.ru/",
-            "companyId": "megafon"
+            "companyId": "vk",
+            "source": "AdGuard"
         },
         "mail.ru_counter": {
             "name": "Mail.Ru Counter",
             "categoryId": 2,
-            "url": "https://corp.megafon.com/",
+            "url": "http://mail.ru/",
-            "companyId": "megafon"
+            "companyId": "vk",
+            "source": "AdGuard"
         },
         "mail.ru_group": {
             "name": "Mail.Ru Group",
             "categoryId": 7,
             "url": "http://mail.ru/",
-            "companyId": "megafon"
+            "companyId": "vk",
+            "source": "AdGuard"
         },
         "mailchimp_tracking": {
             "name": "MailChimp Tracking",
@@ -10631,7 +10640,8 @@
             "name": "MarkMonitor",
             "categoryId": 4,
             "url": "https://www.markmonitor.com/",
-            "companyId": "markmonitor"
+            "companyId": "markmonitor",
+            "source": "AdGuard"
         },
         "marktest": {
             "name": "Marktest",
@@ -11691,9 +11701,10 @@
         },
         "netflix": {
             "name": "Netflix",
-            "categoryId": 8,
+            "categoryId": 0,
-            "url": null,
+            "url": "https://www.netflix.com/",
-            "companyId": null
+            "companyId": "netflix",
+            "source": "AdGuard"
         },
         "netletix": {
             "name": "Netletix",
@@ -17969,19 +17980,22 @@
             "name": "Vk.com",
             "categoryId": 7,
             "url": "https://vk.com/",
-            "companyId": "megafon"
+            "companyId": "vk",
+            "source": "AdGuard"
         },
         "vkontakte": {
             "name": "VKontakte",
             "categoryId": 7,
             "url": "https://vk.com/",
-            "companyId": "megafon"
+            "companyId": "vk",
+            "source": "AdGuard"
         },
         "vkontakte_widgets": {
             "name": "VKontakte Widgets",
             "categoryId": 7,
-            "url": "http://vk.com/developers.php",
+            "url": "https://dev.vk.com/",
-            "companyId": "megafon"
+            "companyId": "vk",
+            "source": "AdGuard"
         },
         "vntsm.com": {
             "name": "Venatus Media",
@@ -19200,6 +19214,160 @@
             "categoryId": 4,
             "url": "http://www.zypmedia.com/",
             "companyId": "zypmedia"
+        },
+        "slack": {
+            "name": "Slack",
+            "categoryId": 8,
+            "url": "https://www.slack.com/",
+            "companyId": "salesforce",
+            "source": "AdGuard"
+        },
+        "apple": {
+            "name": "Apple",
+            "categoryId": 8,
+            "url": "https://www.apple.com/",
+            "companyId": "apple",
+            "source": "AdGuard"
+        },
+        "apple_ads": {
+            "name": "Apple Search Ads",
+            "categoryId": 4,
+            "url": "https://searchads.apple.com/",
+            "companyId": "apple",
+            "source": "AdGuard"
+        },
+        "facebook_audience": {
+            "name": "Facebook Audience Network",
+            "categoryId": 4,
+            "url": "https://www.facebook.com/business/products/audience-network",
+            "companyId": "facebook",
+            "source": "AdGuard"
+        },
+        "crashlytics": {
+            "name": "Crashlytics",
+            "categoryId": 101,
+            "url": "https://crashlytics.com/",
+            "companyId": null,
+            "source": "AdGuard"
+        },
+        "showrss": {
+            "name": "showRSS",
+            "categoryId": 8,
+            "url": "https://showrss.info/",
+            "companyId": "showrss",
+            "source": "AdGuard"
+        },
+        "hockeyapp": {
+            "name": "HockeyApp",
+            "categoryId": 101,
+            "url": "https://hockeyapp.net/",
+            "companyId": null,
+            "source": "AdGuard"
+        },
+        "google_trust_services": {
+            "name": "Google Trust Services",
+            "categoryId": 5,
+            "url": "https://pki.goog/",
+            "companyId": "google",
+            "source": "AdGuard"
+        },
+        "firebase": {
+            "name": "Firebase",
+            "categoryId": 101,
+            "url": "https://firebase.google.com/",
+            "companyId": "google",
+            "source": "AdGuard"
+        },
+        "yandex_appmetrica": {
+            "name": "Yandex AppMetrica",
+            "categoryId": 101,
+            "url": "https://appmetrica.yandex.com/",
+            "companyId": "yandex",
+            "source": "AdGuard"
+        },
+        "branch": {
+            "name": "Branch.io",
+            "categoryId": 101,
+            "url": "https://branch.io/",
+            "companyId": "branch_metrics_inc",
+            "source": "AdGuard"
+        },
+        "qualcomm": {
+            "name": "Qualcomm",
+            "categoryId": 8,
+            "url": "https://www.qualcomm.com/",
+            "companyId": "qualcomm",
+            "source": "AdGuard"
+        },
+        "solaredge": {
+            "name": "SolarEdge Technologies, Inc.",
+            "categoryId": 8,
+            "url": "https://www.solaredge.com/",
+            "companyId": "solaredge",
+            "source": "AdGuard"
+        },
+        "element": {
+            "name": "Element",
+            "categoryId": 7,
+            "url": "https://element.io/",
+            "companyId": "element",
+            "source": "AdGuard"
+        },
+        "outlook": {
+            "name": "Microsoft Outlook",
+            "categoryId": 13,
+            "url": "https://outlook.live.com/",
+            "companyId": "microsoft",
+            "source": "AdGuard"
+        },
+        "appcenter": {
+            "name": "Microsoft App Center",
+            "categoryId": 5,
+            "url": "https://appcenter.ms/",
+            "companyId": null,
+            "source": "AdGuard"
+        },
+        "unity_ads": {
+            "name": "Unity Ads",
+            "categoryId": 4,
+            "url": "https://unity.com/solutions/mobile-business/monetize-your-game",
+            "companyId": null,
+            "source": "AdGuard"
+        },
+        "azure": {
+            "name": "Microsoft Azure",
+            "categoryId": 10,
+            "url": "https://azure.microsoft.com/",
+            "companyId": "microsoft",
+            "source": "AdGuard"
+        },
+        "button": {
+            "name": "Button",
+            "categoryId": 4,
+            "url": "https://www.usebutton.com/",
+            "companyId": null,
+            "source": "AdGuard"
+        },
+        "lets_encrypt": {
+            "name": "Let's Encrypt",
+            "categoryId": 5,
+            "url": "https://letsencrypt.org/",
+            "companyId": "lets_encrypt",
+            "source": "AdGuard"
+        },
+        "plex": {
+            "name": "Plex",
+            "categoryId": 0,
+            "url": "https://www.plex.tv/",
+            "companyId": "plex",
+            "source": "AdGuard"
+        },
+        "matrix": {
+            "name": "Matrix",
+            "categoryId": 5,
+            "url": "https://matrix.org/",
+            "companyId": "matrix",
+            "source": "AdGuard"
         }
     },
     "trackerDomains": {
@@ -20740,7 +20908,7 @@
         "facebook.net": "facebook",
         "fbcdn.net": "facebook_cdn",
         "fbsbx.com": "facebook_cdn",
-        "graph.facebook.com": "facebook_graph",
+        "graph.facebook.com": "facebook_audience",
         "facetz.net": "facetz.dca",
         "adsfac.eu": "facilitate_digital",
         "adsfac.net": "facilitate_digital",
@@ -21273,7 +21441,7 @@
         "isocket.com": "isocket",
         "ispot.tv": "ispot.tv",
         "itineraire.info": "itineraire.info",
-        "apple.com": "itunes_link_maker",
+        "apple.com": "apple",
         "autolinkmaker.itunes.apple.com": "itunes_link_maker",
         "ity.im": "ity.im",
         "iubenda.com": "iubenda.com",
@@ -21727,7 +21895,7 @@
         "azurewebsites.net": "microsoft",
         "cloudapp.net": "microsoft",
         "gfx.ms": "microsoft",
-        "live.com": "microsoft",
+        "live.com": "outlook",
         "microsoft.com": "microsoft",
         "microsoftonline-p.com": "microsoft",
         "microsoftonline.com": "microsoft",
@@ -23529,6 +23697,54 @@
         "zukxd6fkxqn.com": "zukxd6fkxqn.com",
         "zwaar.net": "zwaar",
         "zwaar.org": "zwaar",
-        "extend.tv": "zypmedia"
+        "extend.tv": "zypmedia",
+        "slack.com": "slack",
+        "slackb.com": "slack",
+        "slack-edge.com": "slack",
+        "slack-imgs.com": "slack",
+        "adguard.app": "adguard",
+        "adguard.io": "adguard",
+        "adguard.org": "adguard",
+        "adguard-dns.com": "adguard",
+        "adguard-dns.io": "adguard",
+        "adguard-vpn.com": "adguard",
+        "adguardvpn.com": "adguard",
+        "adguard-vpn.online": "adguard",
+        "nflximg.com": "netflix",
+        "element.io": "element",
+        "riot.im": "element",
+        "akadns.net": "akamai_technologies",
+        "akamaiedge.net": "akamai_technologies",
+        "apple.news": "apple",
+        "apple-dns.net": "apple",
+        "aaplimg.com": "apple",
+        "icloud.com": "apple",
+        "icloud-content.com": "apple",
+        "mzstatic.com": "apple",
+        "matrix.org": "matrix",
+        "l-msedge.net": "microsoft",
+        "iadsdk.apple.com": "apple_ads",
+        "showrss.info": "showrss",
+        "solaredge.com": "solaredge",
+        "crashlytics.com": "crashlytics",
+        "flurry.com": "flurry",
+        "hockeyapp.net": "hockeyapp",
+        "app-measurement.com": "firebase",
+        "appmetrica.yandex.com": "yandex_appmetrica",
+        "letsencrypt.org": "lets_encrypt",
+        "lencr.org": "lets_encrypt",
+        "mobileapptracking.com": "branch",
+        "plex.tv": "plex",
+        "plex.direct": "plex",
+        "edgecastcdn.net": "markmonitor",
+        "appcenter.ms": "appcenter",
+        "unityads.unity3d.com": "unity_ads",
+        "azure.com": "azure",
+        "trafficmanager.net": "azure",
+        "hotmail.com": "outlook",
+        "bttn.io": "button",
+        "pki.goog": "google_trust_services",
+        "xtracloud.net": "qualcomm",
+        "qualcomm.com": "qualcomm"
     }
 }

scripts/README.md

@@ -205,7 +205,7 @@ code from [the repo][companiesrepo].
  ###  Usage
 
 ```sh
-( cd scripts/companiesdb && sh ./download.sh )
+sh ./scripts/companiesdb/download.sh
 ```
 
 [companiesrepo]: https://github.com/AdguardTeam/companiesdb

scripts/companiesdb/download.sh

@@ -4,11 +4,9 @@ set -e -f -u -x
 
 # This script syncs companies DB that we bundle with AdGuard Home.  The source
 # for this database is https://github.com/AdguardTeam/companiesdb.
+#
+trackers_url='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/trackers.json'
+output='./client/src/helpers/trackers/trackers.json'
+readonly trackers_url output
 
-whotracksme='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/whotracksme.json'
+curl -o "$output" -v "$trackers_url"
-adguard='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/adguard.json'
-base_path='../../client/src/helpers/trackers'
-readonly whotracksme adguard base_path
-
-curl -o "${base_path}/whotracksme.json" -v "$whotracksme"
-curl -o "${base_path}/adguard.json"     -v "$adguard"