Skip to content

Commit

Permalink
Merge branch 'SukkaW:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@FYLSen
FYLSen authored Dec 31, 2024
2 parents bef73fa + 9af0c4f commit 14b2140
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 46 deletions.
43 changes: 26 additions & 17 deletions Build/lib/parse-filter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,29 @@ import { createAhoCorasick as createKeywordFilter } from 'foxts/ahocorasick';
import { looseTldtsOpt } from '../constants/loose-tldts-opt';
import { identity } from 'foxts/identity';
import { DEBUG_DOMAIN_TO_FIND } from '../constants/reject-data-source';
import { noop } from 'foxts/noop';

let foundDebugDomain = false;
const temporaryBypass = typeof DEBUG_DOMAIN_TO_FIND === 'string';

const onBlackFound = DEBUG_DOMAIN_TO_FIND
? (line: string, meta: string) => {
if (line.includes(DEBUG_DOMAIN_TO_FIND!)) {
console.warn(picocolors.red(meta), '(black)', line.replaceAll(DEBUG_DOMAIN_TO_FIND!, picocolors.bold(DEBUG_DOMAIN_TO_FIND)));
foundDebugDomain = true;
}
}
: noop;

const onWhiteFound = DEBUG_DOMAIN_TO_FIND
? (line: string, meta: string) => {
if (line.includes(DEBUG_DOMAIN_TO_FIND!)) {
console.warn(picocolors.red(meta), '(white)', line.replaceAll(DEBUG_DOMAIN_TO_FIND!, picocolors.bold(DEBUG_DOMAIN_TO_FIND)));
foundDebugDomain = true;
}
}
: noop;

function domainListLineCb(l: string, set: string[], includeAllSubDomain: boolean, meta: string) {
let line = processLine(l);
if (!line) return;
Expand All @@ -32,10 +51,7 @@ function domainListLineCb(l: string, set: string[], includeAllSubDomain: boolean
return;
}

if (DEBUG_DOMAIN_TO_FIND && line.includes(DEBUG_DOMAIN_TO_FIND)) {
console.warn(picocolors.red(meta), '(black)', line.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND)));
foundDebugDomain = true;
}
onBlackFound(domain, meta);

set.push(includeAllSubDomain ? `.${line}` : line);
}
Expand Down Expand Up @@ -84,10 +100,8 @@ function hostsLineCb(l: string, set: string[], includeAllSubDomain: boolean, met
if (!domain) {
return;
}
if (DEBUG_DOMAIN_TO_FIND && domain.includes(DEBUG_DOMAIN_TO_FIND)) {
console.warn(picocolors.red(meta), '(black)', domain.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND)));
foundDebugDomain = true;
}

onBlackFound(domain, meta);

set.push(includeAllSubDomain ? `.${domain}` : domain);
}
Expand Down Expand Up @@ -169,15 +183,10 @@ export async function processFilterRules(

const hostname = result[0];

if (DEBUG_DOMAIN_TO_FIND && hostname.includes(DEBUG_DOMAIN_TO_FIND)) {
console.warn(
picocolors.red(filterRulesUrl),
flag === ParseType.WhiteIncludeSubdomain || flag === ParseType.WhiteAbsolute
? '(white)'
: '(black)',
hostname.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND))
);
foundDebugDomain = true;
if (flag === ParseType.WhiteIncludeSubdomain || flag === ParseType.WhiteAbsolute) {
onWhiteFound(hostname, filterRulesUrl);
} else {
onBlackFound(hostname, filterRulesUrl);
}

switch (flag) {
Expand Down
7 changes: 1 addition & 6 deletions Build/lib/rules/domainset.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,7 @@ export class DomainsetOutput extends RuleOutput<string[]> {
this.$surge.push(subdomain ? '.' + domain : domain);
this.$clash.push(subdomain ? `+.${domain}` : domain);
(subdomain ? this.$singbox_domains_suffixes : this.$singbox_domains).push(domain);

if (subdomain) {
this.$adguardhome.push(`||${domain}^`);
} else {
this.$adguardhome.push(`|${domain}^`);
}
this.$adguardhome.push(subdomain ? `||${domain}^` : `|${domain}^`);
}, true);

return this.$surge;
Expand Down
74 changes: 54 additions & 20 deletions Build/validate-domain-alive.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,20 @@ const dohServers: Array<[string, DNS2.DnsResolver]> = ([
})
] as const);

const domesticDohServers: Array<[string, DNS2.DnsResolver]> = ([
'223.5.5.5',
'223.6.6.6',
'120.53.53.53',
'1.12.12.12'
] as const).map(dns => [
dns,
DNS2.DOHClient({
dns,
http: false
// get: (url: string) => undici.request(url).then(r => r.body)
})
] as const);

const queue = newQueue(32);
const mutex = new Map<string, Promise<unknown>>();
function keyedAsyncMutexWithQueue<T>(key: string, fn: () => Promise<T>) {
Expand All @@ -72,26 +86,31 @@ interface DnsResponse extends DNS2.$DnsResponse {
dns: string
}

const resolve: DNS2.DnsResolver<DnsResponse> = async (...args) => {
try {
return await asyncRetry(async () => {
const [dohServer, dohClient] = dohServers[Math.floor(Math.random() * dohServers.length)];

try {
return {
...await dohClient(...args),
dns: dohServer
} satisfies DnsResponse;
} catch (e) {
console.error(e);
throw new DnsError((e as Error).message, dohServer);
}
}, { retries: 5 });
} catch (e) {
console.log('[doh error]', ...args, e);
throw e;
}
};
function createResolve(server: Array<[string, DNS2.DnsResolver]>): DNS2.DnsResolver<DnsResponse> {
return async (...args) => {
try {
return await asyncRetry(async () => {
const [dohServer, dohClient] = server[Math.floor(Math.random() * server.length)];

try {
return {
...await dohClient(...args),
dns: dohServer
} satisfies DnsResponse;
} catch (e) {
console.error(e);
throw new DnsError((e as Error).message, dohServer);
}
}, { retries: 5 });
} catch (e) {
console.log('[doh error]', ...args, e);
throw e;
}
};
}

const resolve = createResolve(dohServers);
const domesticResolve = createResolve(domesticDohServers);

async function getWhois(domain: string) {
return asyncRetry(() => whoiser.domain(domain), { retries: 5 });
Expand Down Expand Up @@ -209,6 +228,21 @@ export async function isDomainAlive(domain: string, isSuffix: boolean): Promise<
aaaaDns.push(aaaaRecords.dns);
}

// only then, let's test once with domesticDohServers
const aRecords = (await domesticResolve($domain, 'A'));
if (aRecords.answers.length !== 0) {
domainAliveMap.set(domain, true);
return [domain, true] as const;
}
aDns.push(aRecords.dns);

const aaaaRecords = (await domesticResolve($domain, 'AAAA'));
if (aaaaRecords.answers.length !== 0) {
domainAliveMap.set(domain, true);
return [domain, true] as const;
}
aaaaDns.push(aaaaRecords.dns);

console.log(picocolors.red('[domain dead]'), 'no A/AAAA records', { domain, a: aDns, aaaa: aaaaDns });
}

Expand Down
15 changes: 12 additions & 3 deletions Source/domainset/cdn.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -468,10 +468,13 @@ cms.typenetwork.com
assets.aboutamazon.com
# AWS
.awsstatic.com
cdn.assets.as2.amazonaws.com
.assets.as2.amazonaws.com
ip-ranges.amazonaws.com
.marketing.aws.dev
prod.us-east-1.ui.gcr-chat.marketing.aws.dev
.ui.gcr-chat.marketing.aws.dev
.assets.shortbread.aws.dev
.cdn.chrome.marketplace.aws.dev
.widgets.marketplace.aws.dev

# >> SB.SB
ooo.0o0.ooo
Expand Down Expand Up @@ -2664,6 +2667,7 @@ cdn.pkg.svc.ui.com
assets.unifi-ai.com
assets.ecomm.ui.com
cdn.ecomm.ui.com
static-secure-uploads.ui.com
# StatusPage hosted on CloudFront
status.ui.com

Expand Down Expand Up @@ -3054,7 +3058,7 @@ cdn.safecharge.com
unlimited-cdn.waifu2x.net
.lowendbox.com
cdn-static.talent.com
static.frostming.com
webp.frostming.com
.fiverrcdn.com
cdn.ashbyprd.com
cdn.faire.com
Expand Down Expand Up @@ -3805,3 +3809,8 @@ cdn.lemonsqueezy.com
# AWS S3 + AWS CloudFront
static.captcha-delivery.com
cdn.thenewstack.io
.hommec.com
cdn.wikiwiki.jp
static.pingcap.com
p.depot.dev
k-cdn.depot.dev

0 comments on commit 14b2140

Please sign in to comment.