From c71e9a898390ad6e9588474704b8bec998279d4a Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 27 Dec 2024 21:34:25 +0800 Subject: [PATCH 1/5] Replace outdated domains --- Source/domainset/cdn.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index 2f26583a1..6c561b410 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -3054,7 +3054,7 @@ cdn.safecharge.com unlimited-cdn.waifu2x.net .lowendbox.com cdn-static.talent.com -static.frostming.com +webp.frostming.com .fiverrcdn.com cdn.ashbyprd.com cdn.faire.com From 1df5ede2ea1b377fa2256f4030df71faf30bcc8b Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 27 Dec 2024 21:49:27 +0800 Subject: [PATCH 2/5] Validate domain alive with domestic DoH servers as well --- Build/validate-domain-alive.ts | 74 +++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/Build/validate-domain-alive.ts b/Build/validate-domain-alive.ts index 40c911768..d850e7316 100644 --- a/Build/validate-domain-alive.ts +++ b/Build/validate-domain-alive.ts @@ -50,6 +50,20 @@ const dohServers: Array<[string, DNS2.DnsResolver]> = ([ }) ] as const); +const domesticDohServers: Array<[string, DNS2.DnsResolver]> = ([ + '223.5.5.5', + '223.6.6.6', + '120.53.53.53', + '1.12.12.12' +] as const).map(dns => [ + dns, + DNS2.DOHClient({ + dns, + http: false + // get: (url: string) => undici.request(url).then(r => r.body) + }) +] as const); + const queue = newQueue(32); const mutex = new Map>(); function keyedAsyncMutexWithQueue(key: string, fn: () => Promise) { @@ -72,26 +86,31 @@ interface DnsResponse extends DNS2.$DnsResponse { dns: string } -const resolve: DNS2.DnsResolver = async (...args) => { - try { - return await asyncRetry(async () => { - const [dohServer, dohClient] = dohServers[Math.floor(Math.random() * dohServers.length)]; - - try { - return { - ...await dohClient(...args), - dns: dohServer - } satisfies DnsResponse; - } catch (e) { - console.error(e); - throw new DnsError((e as Error).message, dohServer); - } - }, { retries: 5 }); - } catch (e) { - console.log('[doh error]', ...args, e); - throw e; - } -}; +function createResolve(server: Array<[string, DNS2.DnsResolver]>): DNS2.DnsResolver { + return async (...args) => { + try { + return await asyncRetry(async () => { + const [dohServer, dohClient] = server[Math.floor(Math.random() * server.length)]; + + try { + return { + ...await dohClient(...args), + dns: dohServer + } satisfies DnsResponse; + } catch (e) { + console.error(e); + throw new DnsError((e as Error).message, dohServer); + } + }, { retries: 5 }); + } catch (e) { + console.log('[doh error]', ...args, e); + throw e; + } + }; +} + +const resolve = createResolve(dohServers); +const domesticResolve = createResolve(domesticDohServers); async function getWhois(domain: string) { return asyncRetry(() => whoiser.domain(domain), { retries: 5 }); @@ -209,6 +228,21 @@ export async function isDomainAlive(domain: string, isSuffix: boolean): Promise< aaaaDns.push(aaaaRecords.dns); } + // only then, let's test once with domesticDohServers + const aRecords = (await domesticResolve($domain, 'A')); + if (aRecords.answers.length !== 0) { + domainAliveMap.set(domain, true); + return [domain, true] as const; + } + aDns.push(aRecords.dns); + + const aaaaRecords = (await domesticResolve($domain, 'AAAA')); + if (aaaaRecords.answers.length !== 0) { + domainAliveMap.set(domain, true); + return [domain, true] as const; + } + aaaaDns.push(aaaaRecords.dns); + console.log(picocolors.red('[domain dead]'), 'no A/AAAA records', { domain, a: aDns, aaaa: aaaaDns }); } From b63aae756e4b065b288015e23d3a879782e6dcfe Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sun, 29 Dec 2024 12:48:29 +0800 Subject: [PATCH 3/5] Perf: optimize filter debugger --- Build/lib/parse-filter.ts | 43 ++++++++++++++++++++++-------------- Build/lib/rules/domainset.ts | 7 +----- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/Build/lib/parse-filter.ts b/Build/lib/parse-filter.ts index 3ecfb3d4b..5d862465c 100644 --- a/Build/lib/parse-filter.ts +++ b/Build/lib/parse-filter.ts @@ -10,10 +10,29 @@ import { createAhoCorasick as createKeywordFilter } from 'foxts/ahocorasick'; import { looseTldtsOpt } from '../constants/loose-tldts-opt'; import { identity } from 'foxts/identity'; import { DEBUG_DOMAIN_TO_FIND } from '../constants/reject-data-source'; +import { noop } from 'foxts/noop'; let foundDebugDomain = false; const temporaryBypass = typeof DEBUG_DOMAIN_TO_FIND === 'string'; +const onBlackFound = DEBUG_DOMAIN_TO_FIND + ? (line: string, meta: string) => { + if (line.includes(DEBUG_DOMAIN_TO_FIND!)) { + console.warn(picocolors.red(meta), '(black)', line.replaceAll(DEBUG_DOMAIN_TO_FIND!, picocolors.bold(DEBUG_DOMAIN_TO_FIND))); + foundDebugDomain = true; + } + } + : noop; + +const onWhiteFound = DEBUG_DOMAIN_TO_FIND + ? (line: string, meta: string) => { + if (line.includes(DEBUG_DOMAIN_TO_FIND!)) { + console.warn(picocolors.red(meta), '(white)', line.replaceAll(DEBUG_DOMAIN_TO_FIND!, picocolors.bold(DEBUG_DOMAIN_TO_FIND))); + foundDebugDomain = true; + } + } + : noop; + function domainListLineCb(l: string, set: string[], includeAllSubDomain: boolean, meta: string) { let line = processLine(l); if (!line) return; @@ -32,10 +51,7 @@ function domainListLineCb(l: string, set: string[], includeAllSubDomain: boolean return; } - if (DEBUG_DOMAIN_TO_FIND && line.includes(DEBUG_DOMAIN_TO_FIND)) { - console.warn(picocolors.red(meta), '(black)', line.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND))); - foundDebugDomain = true; - } + onBlackFound(domain, meta); set.push(includeAllSubDomain ? `.${line}` : line); } @@ -84,10 +100,8 @@ function hostsLineCb(l: string, set: string[], includeAllSubDomain: boolean, met if (!domain) { return; } - if (DEBUG_DOMAIN_TO_FIND && domain.includes(DEBUG_DOMAIN_TO_FIND)) { - console.warn(picocolors.red(meta), '(black)', domain.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND))); - foundDebugDomain = true; - } + + onBlackFound(domain, meta); set.push(includeAllSubDomain ? `.${domain}` : domain); } @@ -169,15 +183,10 @@ export async function processFilterRules( const hostname = result[0]; - if (DEBUG_DOMAIN_TO_FIND && hostname.includes(DEBUG_DOMAIN_TO_FIND)) { - console.warn( - picocolors.red(filterRulesUrl), - flag === ParseType.WhiteIncludeSubdomain || flag === ParseType.WhiteAbsolute - ? '(white)' - : '(black)', - hostname.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND)) - ); - foundDebugDomain = true; + if (flag === ParseType.WhiteIncludeSubdomain || flag === ParseType.WhiteAbsolute) { + onWhiteFound(hostname, filterRulesUrl); + } else { + onBlackFound(hostname, filterRulesUrl); } switch (flag) { diff --git a/Build/lib/rules/domainset.ts b/Build/lib/rules/domainset.ts index ed2a74d82..6868bdd78 100644 --- a/Build/lib/rules/domainset.ts +++ b/Build/lib/rules/domainset.ts @@ -27,12 +27,7 @@ export class DomainsetOutput extends RuleOutput { this.$surge.push(subdomain ? '.' + domain : domain); this.$clash.push(subdomain ? `+.${domain}` : domain); (subdomain ? this.$singbox_domains_suffixes : this.$singbox_domains).push(domain); - - if (subdomain) { - this.$adguardhome.push(`||${domain}^`); - } else { - this.$adguardhome.push(`|${domain}^`); - } + this.$adguardhome.push(subdomain ? `||${domain}^` : `|${domain}^`); }, true); return this.$surge; From 589e1f5727c0f1096e43ab76d7846f655c6fff34 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sun, 29 Dec 2024 12:33:48 +0800 Subject: [PATCH 4/5] Update CDN Hosts --- Source/domainset/cdn.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index 6c561b410..838219289 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -2664,6 +2664,7 @@ cdn.pkg.svc.ui.com assets.unifi-ai.com assets.ecomm.ui.com cdn.ecomm.ui.com +static-secure-uploads.ui.com # StatusPage hosted on CloudFront status.ui.com @@ -3805,3 +3806,5 @@ cdn.lemonsqueezy.com # AWS S3 + AWS CloudFront static.captcha-delivery.com cdn.thenewstack.io +.hommec.com +cdn.wikiwiki.jp From 9af0c4f8359f46c5050944816749d36e114686e5 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Mon, 30 Dec 2024 20:15:13 +0800 Subject: [PATCH 5/5] Update CDN Hosts --- Source/domainset/cdn.conf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index 838219289..ae487d8c0 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -468,10 +468,13 @@ cms.typenetwork.com assets.aboutamazon.com # AWS .awsstatic.com -cdn.assets.as2.amazonaws.com +.assets.as2.amazonaws.com ip-ranges.amazonaws.com .marketing.aws.dev -prod.us-east-1.ui.gcr-chat.marketing.aws.dev +.ui.gcr-chat.marketing.aws.dev +.assets.shortbread.aws.dev +.cdn.chrome.marketplace.aws.dev +.widgets.marketplace.aws.dev # >> SB.SB ooo.0o0.ooo @@ -3808,3 +3811,6 @@ static.captcha-delivery.com cdn.thenewstack.io .hommec.com cdn.wikiwiki.jp +static.pingcap.com +p.depot.dev +k-cdn.depot.dev