Pull request: 5471-new-trackers-db

Merge in DNS/adguard-home from 5471-new-trackers-db to master

Squashed commit of the following:

commit d90742c2902afd3424fb8721419231493610048b
Merge: 5e87a1f2 a50a8abb
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Feb 14 13:38:40 2023 +0300

    Merge branch 'master' into 5471-new-trackers-db

commit 5e87a1f25cd1d1dc52629d36e3d50d032ada1093
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Feb 13 18:38:54 2023 +0300

    all: use new tracker source
This commit is contained in:
Ainar Garipov 2023-02-14 13:52:22 +03:00
parent a50a8abb45
commit 3378d1c103
6 changed files with 268 additions and 238 deletions

View File

@ -53,7 +53,7 @@ export const STATUS_COLORS = {
export const REPOSITORY = { export const REPOSITORY = {
URL: 'https://github.com/AdguardTeam/AdGuardHome', URL: 'https://github.com/AdguardTeam/AdGuardHome',
TRACKERS_DB: TRACKERS_DB:
'https://github.com/AdguardTeam/AdGuardHome/tree/master/client/src/helpers/trackers/adguard.json', 'https://github.com/AdguardTeam/AdGuardHome/tree/master/client/src/helpers/trackers/trackers.json',
ISSUES: 'https://github.com/AdguardTeam/AdGuardHome/issues/new/choose', ISSUES: 'https://github.com/AdguardTeam/AdGuardHome/issues/new/choose',
}; };

View File

@ -1,175 +0,0 @@
{
"timeUpdated": "2021-12-15",
"categories": {
"0": "audio_video_player",
"1": "comments",
"2": "customer_interaction",
"3": "pornvertising",
"4": "advertising",
"5": "essential",
"6": "site_analytics",
"7": "social_media",
"8": "misc",
"9": "cdn",
"10": "hosting",
"11": "unknown",
"12": "extensions",
"13": "email",
"14": "consent",
"15": "telemetry",
"101": "mobile_analytics"
},
"trackers": {
"akamai_technologies": {
"name": "Akamai Technologies",
"categoryId": 9,
"url": "https://www.akamai.com/",
"companyId": "akamai"
},
"apple": {
"name": "Apple",
"categoryId": 8,
"url": "https://www.apple.com/",
"companyId": "apple"
},
"apple_ads": {
"name": "Apple Search Ads",
"categoryId": 4,
"url": "https://searchads.apple.com/",
"companyId": "apple"
},
"facebook_audience": {
"name": "Facebook Audience Network",
"categoryId": 4,
"url": "https://www.facebook.com/business/products/audience-network",
"companyId": "facebook"
},
"crashlytics": {
"name": "Crashlytics",
"categoryId": 101,
"url": "https://crashlytics.com/",
"companyId": null
},
"flurry": {
"name": "Flurry",
"categoryId": 101,
"url": "http://www.flurry.com/",
"companyId": "verizon"
},
"hockeyapp": {
"name": "HockeyApp",
"categoryId": 101,
"url": "https://hockeyapp.net/",
"companyId": null
},
"firebase": {
"name": "Firebase",
"categoryId": 101,
"url": "https://firebase.google.com/",
"companyId": "google"
},
"appsflyer": {
"name": "AppsFlyer",
"categoryId": 101,
"url": "https://www.appsflyer.com/",
"companyId": "appsflyer"
},
"yandex_appmetrica": {
"name": "Yandex AppMetrica",
"categoryId": 101,
"url": "https://appmetrica.yandex.com/",
"companyId": "yandex"
},
"adjust": {
"name": "Adjust",
"categoryId": 101,
"url": "https://www.adjust.com/",
"companyId": "adjust"
},
"branch": {
"name": "Branch.io",
"categoryId": 101,
"url": "https://branch.io/",
"companyId": "branch_metrics_inc"
},
"markmonitor": {
"name": "MarkMonitor",
"categoryId": 4,
"url": "https://www.markmonitor.com/",
"companyId": "markmonitor"
},
"appcenter": {
"name": "Microsoft App Center",
"categoryId": 5,
"url": "https://appcenter.ms/",
"companyId": null
},
"unity_ads": {
"name": "Unity Ads",
"categoryId": 4,
"url": "https://unity.com/solutions/mobile-business/monetize-your-game",
"companyId": null
},
"azure": {
"name": "Microsoft Azure",
"categoryId": 10,
"url": "https://azure.microsoft.com/",
"companyId": "microsoft"
},
"button": {
"name": "Button",
"categoryId": 4,
"url": "https://www.usebutton.com/",
"companyId": null
},
"netflix": {
"name": "Netflix",
"categoryId": 8,
"url": "https://www.netflix.com/",
"companyId": null
},
"mail.ru_banner": {
"name": "Mail.Ru Banner Network",
"categoryId": 4,
"url": "http://mail.ru/",
"companyId": "vk"
},
"mail.ru_counter": {
"name": "Mail.Ru Counter",
"categoryId": 2,
"url": "http://mail.ru/",
"companyId": "vk"
},
"mail.ru_group": {
"name": "Mail.Ru Group",
"categoryId": 7,
"url": "http://mail.ru/",
"companyId": "vk"
}
},
"trackerDomains": {
"akadns.net": "akamai_technologies",
"akamaiedge.net": "akamai_technologies",
"apple.com": "apple",
"apple.news": "apple",
"apple-dns.net": "apple",
"aaplimg.com": "apple",
"icloud.com": "apple",
"mzstatic.com": "apple",
"iadsdk.apple.com": "apple_ads",
"graph.facebook.com": "facebook_audience",
"crashlytics.com": "crashlytics",
"flurry.com": "flurry",
"hockeyapp.net": "hockeyapp",
"app-measurement.com": "firebase",
"appsflyer.com": "appsflyer",
"appmetrica.yandex.com": "yandex_appmetrica",
"adjust.com": "adjust",
"mobileapptracking.com": "branch",
"edgecastcdn.net": "markmonitor",
"appcenter.ms": "appcenter",
"unityads.unity3d.com": "unity_ads",
"azure.com": "azure",
"bttn.io": "button"
}
}

View File

@ -1,6 +1,5 @@
import whotracksmeDb from './whotracksme.json';
import whotracksmeWebsites from './whotracksme_web.json'; import whotracksmeWebsites from './whotracksme_web.json';
import adguardDb from './adguard.json'; import trackersDb from './trackers.json';
import { REPOSITORY } from '../constants'; import { REPOSITORY } from '../constants';
/** /**
@ -64,14 +63,25 @@ export const getSourceData = (trackerData) => {
}; };
/** /**
* Gets tracker data in the specified database * Converts the JSON string source into numeric source for AdGuard Home
* @param {TrackerData} trackerData tracker data
* @returns {number} source number
*/
const convertSource = (sourceStr) => {
if (!sourceStr || sourceStr !== 'AdGuard') {
return sources.WHOTRACKSME;
}
return sources.ADGUARD;
};
/**
* Gets tracker data from the trackers database
* *
* @param {String} domainName domain name to check * @param {String} domainName domain name to check
* @param {*} trackersDb trackers database
* @param {number} source source ID
* @returns {TrackerData} tracker data or null if no matching tracker found * @returns {TrackerData} tracker data or null if no matching tracker found
*/ */
const getTrackerDataFromDb = (domainName, trackersDb, source) => { export const getTrackerData = (domainName) => {
if (!domainName) { if (!domainName) {
return null; return null;
} }
@ -88,7 +98,7 @@ const getTrackerDataFromDb = (domainName, trackersDb, source) => {
if (trackerId) { if (trackerId) {
const trackerData = trackersDb.trackers[trackerId]; const trackerData = trackersDb.trackers[trackerId];
const categoryName = trackersDb.categories[trackerData.categoryId]; const categoryName = trackersDb.categories[trackerData.categoryId];
trackerData.source = source; const source = convertSource(trackerData.source);
const sourceData = getSourceData(trackerData); const sourceData = getSourceData(trackerData);
return { return {
@ -105,22 +115,3 @@ const getTrackerDataFromDb = (domainName, trackersDb, source) => {
// No tracker found for the specified domain // No tracker found for the specified domain
return null; return null;
}; };
/**
* Gets tracker data from the trackers database
*
* @param {String} domainName domain name to check
* @returns {TrackerData} tracker data or null if no matching tracker found
*/
export const getTrackerData = (domainName) => {
if (!domainName) {
return null;
}
let data = getTrackerDataFromDb(domainName, adguardDb, sources.ADGUARD);
if (!data) {
data = getTrackerDataFromDb(domainName, whotracksmeDb, sources.WHOTRACKSME);
}
return data;
};

View File

@ -1,5 +1,5 @@
{ {
"timeUpdated": "2022-10-15T00:14:03.765Z", "timeUpdated": "2023-02-09T12:31:34.007Z",
"categories": { "categories": {
"0": "audio_video_player", "0": "audio_video_player",
"1": "comments", "1": "comments",
@ -16,7 +16,8 @@
"12": "extensions", "12": "extensions",
"13": "email", "13": "email",
"14": "consent", "14": "consent",
"15": "telemetry" "15": "telemetry",
"101": "mobile_analytics"
}, },
"trackers": { "trackers": {
"163": { "163": {
@ -872,10 +873,11 @@
"companyId": "adgoto" "companyId": "adgoto"
}, },
"adguard": { "adguard": {
"name": "Adguard", "name": "AdGuard",
"categoryId": 12, "categoryId": 8,
"url": "https://adguard.com/", "url": "https://adguard.com/",
"companyId": null "companyId": "adguard",
"source": "AdGuard"
}, },
"adhands": { "adhands": {
"name": "AdHands", "name": "AdHands",
@ -951,9 +953,10 @@
}, },
"adjust": { "adjust": {
"name": "Adjust", "name": "Adjust",
"categoryId": 6, "categoryId": 101,
"url": "https://www.adjust.com/", "url": "https://www.adjust.com/",
"companyId": "adjust" "companyId": "adjust",
"source": "AdGuard"
}, },
"adk2": { "adk2": {
"name": "adk2", "name": "adk2",
@ -2141,7 +2144,8 @@
"name": "Akamai Technologies", "name": "Akamai Technologies",
"categoryId": 9, "categoryId": 9,
"url": "https://www.akamai.com/", "url": "https://www.akamai.com/",
"companyId": "akamai" "companyId": "akamai",
"source": "AdGuard"
}, },
"akamoihd.net": { "akamoihd.net": {
"name": "akamoihd.net", "name": "akamoihd.net",
@ -2541,9 +2545,10 @@
}, },
"appsflyer": { "appsflyer": {
"name": "AppsFlyer", "name": "AppsFlyer",
"categoryId": 6, "categoryId": 101,
"url": "https://www.appsflyer.com/", "url": "https://www.appsflyer.com/",
"companyId": "appsflyer" "companyId": "appsflyer",
"source": "AdGuard"
}, },
"apptv": { "apptv": {
"name": "appTV", "name": "appTV",
@ -7095,9 +7100,10 @@
}, },
"flurry": { "flurry": {
"name": "Flurry", "name": "Flurry",
"categoryId": 6, "categoryId": 101,
"url": "http://www.flurry.com/", "url": "http://www.flurry.com/",
"companyId": "verizon" "companyId": "verizon",
"source": "AdGuard"
}, },
"flxone": { "flxone": {
"name": "FLXONE", "name": "FLXONE",
@ -10493,19 +10499,22 @@
"name": "Mail.Ru Banner Network", "name": "Mail.Ru Banner Network",
"categoryId": 4, "categoryId": 4,
"url": "http://mail.ru/", "url": "http://mail.ru/",
"companyId": "megafon" "companyId": "vk",
"source": "AdGuard"
}, },
"mail.ru_counter": { "mail.ru_counter": {
"name": "Mail.Ru Counter", "name": "Mail.Ru Counter",
"categoryId": 2, "categoryId": 2,
"url": "https://corp.megafon.com/", "url": "http://mail.ru/",
"companyId": "megafon" "companyId": "vk",
"source": "AdGuard"
}, },
"mail.ru_group": { "mail.ru_group": {
"name": "Mail.Ru Group", "name": "Mail.Ru Group",
"categoryId": 7, "categoryId": 7,
"url": "http://mail.ru/", "url": "http://mail.ru/",
"companyId": "megafon" "companyId": "vk",
"source": "AdGuard"
}, },
"mailchimp_tracking": { "mailchimp_tracking": {
"name": "MailChimp Tracking", "name": "MailChimp Tracking",
@ -10631,7 +10640,8 @@
"name": "MarkMonitor", "name": "MarkMonitor",
"categoryId": 4, "categoryId": 4,
"url": "https://www.markmonitor.com/", "url": "https://www.markmonitor.com/",
"companyId": "markmonitor" "companyId": "markmonitor",
"source": "AdGuard"
}, },
"marktest": { "marktest": {
"name": "Marktest", "name": "Marktest",
@ -11691,9 +11701,10 @@
}, },
"netflix": { "netflix": {
"name": "Netflix", "name": "Netflix",
"categoryId": 8, "categoryId": 0,
"url": null, "url": "https://www.netflix.com/",
"companyId": null "companyId": "netflix",
"source": "AdGuard"
}, },
"netletix": { "netletix": {
"name": "Netletix", "name": "Netletix",
@ -17969,19 +17980,22 @@
"name": "Vk.com", "name": "Vk.com",
"categoryId": 7, "categoryId": 7,
"url": "https://vk.com/", "url": "https://vk.com/",
"companyId": "megafon" "companyId": "vk",
"source": "AdGuard"
}, },
"vkontakte": { "vkontakte": {
"name": "VKontakte", "name": "VKontakte",
"categoryId": 7, "categoryId": 7,
"url": "https://vk.com/", "url": "https://vk.com/",
"companyId": "megafon" "companyId": "vk",
"source": "AdGuard"
}, },
"vkontakte_widgets": { "vkontakte_widgets": {
"name": "VKontakte Widgets", "name": "VKontakte Widgets",
"categoryId": 7, "categoryId": 7,
"url": "http://vk.com/developers.php", "url": "https://dev.vk.com/",
"companyId": "megafon" "companyId": "vk",
"source": "AdGuard"
}, },
"vntsm.com": { "vntsm.com": {
"name": "Venatus Media", "name": "Venatus Media",
@ -19200,6 +19214,160 @@
"categoryId": 4, "categoryId": 4,
"url": "http://www.zypmedia.com/", "url": "http://www.zypmedia.com/",
"companyId": "zypmedia" "companyId": "zypmedia"
},
"slack": {
"name": "Slack",
"categoryId": 8,
"url": "https://www.slack.com/",
"companyId": "salesforce",
"source": "AdGuard"
},
"apple": {
"name": "Apple",
"categoryId": 8,
"url": "https://www.apple.com/",
"companyId": "apple",
"source": "AdGuard"
},
"apple_ads": {
"name": "Apple Search Ads",
"categoryId": 4,
"url": "https://searchads.apple.com/",
"companyId": "apple",
"source": "AdGuard"
},
"facebook_audience": {
"name": "Facebook Audience Network",
"categoryId": 4,
"url": "https://www.facebook.com/business/products/audience-network",
"companyId": "facebook",
"source": "AdGuard"
},
"crashlytics": {
"name": "Crashlytics",
"categoryId": 101,
"url": "https://crashlytics.com/",
"companyId": null,
"source": "AdGuard"
},
"showrss": {
"name": "showRSS",
"categoryId": 8,
"url": "https://showrss.info/",
"companyId": "showrss",
"source": "AdGuard"
},
"hockeyapp": {
"name": "HockeyApp",
"categoryId": 101,
"url": "https://hockeyapp.net/",
"companyId": null,
"source": "AdGuard"
},
"google_trust_services": {
"name": "Google Trust Services",
"categoryId": 5,
"url": "https://pki.goog/",
"companyId": "google",
"source": "AdGuard"
},
"firebase": {
"name": "Firebase",
"categoryId": 101,
"url": "https://firebase.google.com/",
"companyId": "google",
"source": "AdGuard"
},
"yandex_appmetrica": {
"name": "Yandex AppMetrica",
"categoryId": 101,
"url": "https://appmetrica.yandex.com/",
"companyId": "yandex",
"source": "AdGuard"
},
"branch": {
"name": "Branch.io",
"categoryId": 101,
"url": "https://branch.io/",
"companyId": "branch_metrics_inc",
"source": "AdGuard"
},
"qualcomm": {
"name": "Qualcomm",
"categoryId": 8,
"url": "https://www.qualcomm.com/",
"companyId": "qualcomm",
"source": "AdGuard"
},
"solaredge": {
"name": "SolarEdge Technologies, Inc.",
"categoryId": 8,
"url": "https://www.solaredge.com/",
"companyId": "solaredge",
"source": "AdGuard"
},
"element": {
"name": "Element",
"categoryId": 7,
"url": "https://element.io/",
"companyId": "element",
"source": "AdGuard"
},
"outlook": {
"name": "Microsoft Outlook",
"categoryId": 13,
"url": "https://outlook.live.com/",
"companyId": "microsoft",
"source": "AdGuard"
},
"appcenter": {
"name": "Microsoft App Center",
"categoryId": 5,
"url": "https://appcenter.ms/",
"companyId": null,
"source": "AdGuard"
},
"unity_ads": {
"name": "Unity Ads",
"categoryId": 4,
"url": "https://unity.com/solutions/mobile-business/monetize-your-game",
"companyId": null,
"source": "AdGuard"
},
"azure": {
"name": "Microsoft Azure",
"categoryId": 10,
"url": "https://azure.microsoft.com/",
"companyId": "microsoft",
"source": "AdGuard"
},
"button": {
"name": "Button",
"categoryId": 4,
"url": "https://www.usebutton.com/",
"companyId": null,
"source": "AdGuard"
},
"lets_encrypt": {
"name": "Let's Encrypt",
"categoryId": 5,
"url": "https://letsencrypt.org/",
"companyId": "lets_encrypt",
"source": "AdGuard"
},
"plex": {
"name": "Plex",
"categoryId": 0,
"url": "https://www.plex.tv/",
"companyId": "plex",
"source": "AdGuard"
},
"matrix": {
"name": "Matrix",
"categoryId": 5,
"url": "https://matrix.org/",
"companyId": "matrix",
"source": "AdGuard"
} }
}, },
"trackerDomains": { "trackerDomains": {
@ -20740,7 +20908,7 @@
"facebook.net": "facebook", "facebook.net": "facebook",
"fbcdn.net": "facebook_cdn", "fbcdn.net": "facebook_cdn",
"fbsbx.com": "facebook_cdn", "fbsbx.com": "facebook_cdn",
"graph.facebook.com": "facebook_graph", "graph.facebook.com": "facebook_audience",
"facetz.net": "facetz.dca", "facetz.net": "facetz.dca",
"adsfac.eu": "facilitate_digital", "adsfac.eu": "facilitate_digital",
"adsfac.net": "facilitate_digital", "adsfac.net": "facilitate_digital",
@ -21273,7 +21441,7 @@
"isocket.com": "isocket", "isocket.com": "isocket",
"ispot.tv": "ispot.tv", "ispot.tv": "ispot.tv",
"itineraire.info": "itineraire.info", "itineraire.info": "itineraire.info",
"apple.com": "itunes_link_maker", "apple.com": "apple",
"autolinkmaker.itunes.apple.com": "itunes_link_maker", "autolinkmaker.itunes.apple.com": "itunes_link_maker",
"ity.im": "ity.im", "ity.im": "ity.im",
"iubenda.com": "iubenda.com", "iubenda.com": "iubenda.com",
@ -21727,7 +21895,7 @@
"azurewebsites.net": "microsoft", "azurewebsites.net": "microsoft",
"cloudapp.net": "microsoft", "cloudapp.net": "microsoft",
"gfx.ms": "microsoft", "gfx.ms": "microsoft",
"live.com": "microsoft", "live.com": "outlook",
"microsoft.com": "microsoft", "microsoft.com": "microsoft",
"microsoftonline-p.com": "microsoft", "microsoftonline-p.com": "microsoft",
"microsoftonline.com": "microsoft", "microsoftonline.com": "microsoft",
@ -23529,6 +23697,54 @@
"zukxd6fkxqn.com": "zukxd6fkxqn.com", "zukxd6fkxqn.com": "zukxd6fkxqn.com",
"zwaar.net": "zwaar", "zwaar.net": "zwaar",
"zwaar.org": "zwaar", "zwaar.org": "zwaar",
"extend.tv": "zypmedia" "extend.tv": "zypmedia",
"slack.com": "slack",
"slackb.com": "slack",
"slack-edge.com": "slack",
"slack-imgs.com": "slack",
"adguard.app": "adguard",
"adguard.io": "adguard",
"adguard.org": "adguard",
"adguard-dns.com": "adguard",
"adguard-dns.io": "adguard",
"adguard-vpn.com": "adguard",
"adguardvpn.com": "adguard",
"adguard-vpn.online": "adguard",
"nflximg.com": "netflix",
"element.io": "element",
"riot.im": "element",
"akadns.net": "akamai_technologies",
"akamaiedge.net": "akamai_technologies",
"apple.news": "apple",
"apple-dns.net": "apple",
"aaplimg.com": "apple",
"icloud.com": "apple",
"icloud-content.com": "apple",
"mzstatic.com": "apple",
"matrix.org": "matrix",
"l-msedge.net": "microsoft",
"iadsdk.apple.com": "apple_ads",
"showrss.info": "showrss",
"solaredge.com": "solaredge",
"crashlytics.com": "crashlytics",
"flurry.com": "flurry",
"hockeyapp.net": "hockeyapp",
"app-measurement.com": "firebase",
"appmetrica.yandex.com": "yandex_appmetrica",
"letsencrypt.org": "lets_encrypt",
"lencr.org": "lets_encrypt",
"mobileapptracking.com": "branch",
"plex.tv": "plex",
"plex.direct": "plex",
"edgecastcdn.net": "markmonitor",
"appcenter.ms": "appcenter",
"unityads.unity3d.com": "unity_ads",
"azure.com": "azure",
"trafficmanager.net": "azure",
"hotmail.com": "outlook",
"bttn.io": "button",
"pki.goog": "google_trust_services",
"xtracloud.net": "qualcomm",
"qualcomm.com": "qualcomm"
} }
} }

View File

@ -205,7 +205,7 @@ code from [the repo][companiesrepo].
### Usage ### Usage
```sh ```sh
( cd scripts/companiesdb && sh ./download.sh ) sh ./scripts/companiesdb/download.sh
``` ```
[companiesrepo]: https://github.com/AdguardTeam/companiesdb [companiesrepo]: https://github.com/AdguardTeam/companiesdb

View File

@ -4,11 +4,9 @@ set -e -f -u -x
# This script syncs companies DB that we bundle with AdGuard Home. The source # This script syncs companies DB that we bundle with AdGuard Home. The source
# for this database is https://github.com/AdguardTeam/companiesdb. # for this database is https://github.com/AdguardTeam/companiesdb.
#
trackers_url='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/trackers.json'
output='./client/src/helpers/trackers/trackers.json'
readonly trackers_url output
whotracksme='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/whotracksme.json' curl -o "$output" -v "$trackers_url"
adguard='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/adguard.json'
base_path='../../client/src/helpers/trackers'
readonly whotracksme adguard base_path
curl -o "${base_path}/whotracksme.json" -v "$whotracksme"
curl -o "${base_path}/adguard.json" -v "$adguard"