diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e7f278123..6c7f6d58b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,6 +31,7 @@ jobs: - name: Cache cache.db uses: actions/cache@v4 with: + save-always: true path: | .cache key: ${{ runner.os }}-v1-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}:${{ steps.date.outputs.second }} diff --git a/.gitignore b/.gitignore index fdfee4e01..7f0ebf70e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ Mock/www-google-analytics-com_ga.js Mock/www-googletagservices-com_gpt.js Mock/www-google-analytics-com_analytics.js Mock/www-googlesyndication-com_adsbygoogle.js +Mock/amazon-adsystem-com_amazon-apstag.js diff --git a/Build/build-chn-cidr.ts b/Build/build-chn-cidr.ts index e14bdf6f0..476ee1bf2 100644 --- a/Build/build-chn-cidr.ts +++ b/Build/build-chn-cidr.ts @@ -2,7 +2,6 @@ import { fetchRemoteTextByLine } from './lib/fetch-text-by-line'; import { resolve as pathResolve } from 'path'; import { compareAndWriteFile, withBannerArray } from './lib/create-file'; import { processLineFromReadline } from './lib/process-line'; -import { traceSync } from './lib/trace-runner'; import { task } from './trace'; import { exclude } from 'fast-cidr-tools'; @@ -21,11 +20,7 @@ const INCLUDE_CIDRS = [ export const getChnCidrPromise = createMemoizedPromise(async () => { const cidr = await processLineFromReadline(await fetchRemoteTextByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')); - return traceSync( - picocolors.gray('processing chnroutes2'), - () => exclude([...cidr, ...INCLUDE_CIDRS], EXCLUDE_CIDRS, true), - picocolors.gray - ); + return exclude([...cidr, ...INCLUDE_CIDRS], EXCLUDE_CIDRS, true); }); export const buildChnCidr = task(import.meta.path, async (span) => { diff --git a/Build/build-microsoft-cdn.ts b/Build/build-microsoft-cdn.ts index dff4023f7..033aeb830 100644 --- a/Build/build-microsoft-cdn.ts +++ b/Build/build-microsoft-cdn.ts @@ -1,5 +1,4 @@ import path from 'path'; -import { traceAsync } from './lib/trace-runner'; import { task } from './trace'; import { createRuleset } from './lib/create-file'; import { fetchRemoteTextByLine } from './lib/fetch-text-by-line'; diff --git a/Build/download-mock-assets.ts b/Build/download-mock-assets.ts index 026258896..3711c4f8c 100644 --- a/Build/download-mock-assets.ts +++ b/Build/download-mock-assets.ts @@ -6,7 +6,8 @@ const ASSETS_LIST = { 'www-google-analytics-com_ga.js': 'https://raw.githubusercontent.com/AdguardTeam/Scriptlets/master/dist/redirect-files/google-analytics-ga.js', 'www-googletagservices-com_gpt.js': 'https://raw.githubusercontent.com/AdguardTeam/Scriptlets/master/dist/redirect-files/googletagservices-gpt.js', 'www-google-analytics-com_analytics.js': 'https://raw.githubusercontent.com/AdguardTeam/Scriptlets/master/dist/redirect-files/google-analytics.js', - 'www-googlesyndication-com_adsbygoogle.js': 'https://raw.githubusercontent.com/AdguardTeam/Scriptlets/master/dist/redirect-files/googlesyndication-adsbygoogle.js' + 'www-googlesyndication-com_adsbygoogle.js': 'https://raw.githubusercontent.com/AdguardTeam/Scriptlets/master/dist/redirect-files/googlesyndication-adsbygoogle.js', + 'amazon-adsystem-com_amazon-apstag.js': 'https://raw.githubusercontent.com/AdguardTeam/Scriptlets/master/dist/redirect-files/amazon-apstag.js' } as const; const mockDir = path.resolve(import.meta.dir, '../Mock'); diff --git a/Build/download-publicsuffixlist.ts b/Build/download-publicsuffixlist.ts index 272760b6e..72fbd9b29 100644 --- a/Build/download-publicsuffixlist.ts +++ b/Build/download-publicsuffixlist.ts @@ -1,18 +1,14 @@ import { TTL, fsFetchCache } from './lib/cache-filesystem'; import { defaultRequestInit, fetchWithRetry } from './lib/fetch-retry'; import { createMemoizedPromise } from './lib/memo-promise'; -import { traceAsync } from './lib/trace-runner'; -export const getPublicSuffixListTextPromise = createMemoizedPromise(() => traceAsync( - 'obtain public_suffix_list', - () => fsFetchCache.apply( - 'https://publicsuffix.org/list/public_suffix_list.dat', - () => fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat', defaultRequestInit).then(r => r.text()), - { - // https://github.com/publicsuffix/list/blob/master/.github/workflows/tld-update.yml - // Though the action runs every 24 hours, the IANA list is updated every 7 days. - // So a 3 day TTL should be enough. - ttl: TTL.THREE_DAYS() - } - ) +export const getPublicSuffixListTextPromise = createMemoizedPromise(() => fsFetchCache.apply( + 'https://publicsuffix.org/list/public_suffix_list.dat', + () => fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat', defaultRequestInit).then(r => r.text()), + { + // https://github.com/publicsuffix/list/blob/master/.github/workflows/tld-update.yml + // Though the action runs every 24 hours, the IANA list is updated every 7 days. + // So a 3 day TTL should be enough. + ttl: TTL.THREE_DAYS() + } )); diff --git a/Build/lib/cache-filesystem.ts b/Build/lib/cache-filesystem.ts index 8f1d566de..c0387c320 100644 --- a/Build/lib/cache-filesystem.ts +++ b/Build/lib/cache-filesystem.ts @@ -4,7 +4,6 @@ import os from 'os'; import path from 'path'; import { mkdirSync } from 'fs'; import picocolors from 'picocolors'; -import { traceSync } from './trace-runner'; const identity = (x: any) => x; @@ -72,7 +71,13 @@ export class Cache { type: S extends string ? 'string' : 'buffer'; constructor({ - cachePath = path.join(os.tmpdir() || '/tmp', 'hdc'), tbd, tableName = 'cache', type }: CacheOptions = {}) { + cachePath = path.join(os.tmpdir() || '/tmp', 'hdc'), + tbd, + tableName = 'cache', + type + }: CacheOptions = {}) { + const start = Bun.nanoseconds(); + this.cachePath = cachePath; mkdirSync(this.cachePath, { recursive: true }); if (tbd != null) this.tbd = tbd; @@ -111,6 +116,9 @@ export class Cache { this.set('__LAST_VACUUM', dateString, 10 * 365 * 60 * 60 * 24 * 1000); this.db.exec('VACUUM;'); } + + const end = Bun.nanoseconds(); + console.log(`${picocolors.gray(`[${((end - start) / 1e6).toFixed(3)}ms]`)} cache initialized from ${this.cachePath}`); } set(key: string, value: string, ttl = 60 * 1000): void { @@ -172,7 +180,6 @@ export class Cache { if (peeked === promise) { return promise.then((value) => { - const serializer = 'serializer' in opt ? opt.serializer : identity; this.set(key, serializer(value), ttl); return value; }); @@ -195,7 +202,7 @@ export class Cache { } } -export const fsFetchCache = traceSync('initializing filesystem cache for fetch', () => new Cache({ cachePath: path.resolve(import.meta.dir, '../../.cache') })); +export const fsFetchCache = new Cache({ cachePath: path.resolve(import.meta.dir, '../../.cache') }); // process.on('exit', () => { // fsFetchCache.destroy(); // }); diff --git a/Build/lib/cached-tld-parse.ts b/Build/lib/cached-tld-parse.ts index 8acbf3d31..a09fa8090 100644 --- a/Build/lib/cached-tld-parse.ts +++ b/Build/lib/cached-tld-parse.ts @@ -1,11 +1,9 @@ import { createCache } from './cache-apply'; import type { PublicSuffixList } from '@gorhill/publicsuffixlist'; -let gothillGetDomainCache: ReturnType | null = null; +let gorhillGetDomainCache: ReturnType | null = null; export const createCachedGorhillGetDomain = (gorhill: PublicSuffixList) => { - gothillGetDomainCache ??= createCache('cached-gorhill-get-domain', true); - return (domain: string) => { - // we do know gothillGetDomainCache exists here - return gothillGetDomainCache!.sync(domain, () => gorhill.getDomain(domain[0] === '.' ? domain.slice(1) : domain)); - }; + gorhillGetDomainCache ??= createCache('cached-gorhill-get-domain', true); + return (domain: string) => gorhillGetDomainCache! // we do know gothillGetDomainCache exists here + .sync(domain, () => gorhill.getDomain(domain[0] === '.' ? domain.slice(1) : domain)); }; diff --git a/Build/lib/clash.ts b/Build/lib/clash.ts index 3dade26d6..819ecd17b 100644 --- a/Build/lib/clash.ts +++ b/Build/lib/clash.ts @@ -24,15 +24,9 @@ const REQUIRE_REWRITE = { export const surgeRulesetToClashClassicalTextRuleset = (rules: string[] | Set) => { const trie = Trie.from(rules); - return CLASH_SUPPORTED_RULE_TYPE.flatMap( - type => trie.find(`${type},`) - ).concat( - Object.keys(REQUIRE_REWRITE).flatMap( - (type) => { - const found = trie.find(`${type},`); - return found.map(line => `${REQUIRE_REWRITE[type as keyof typeof REQUIRE_REWRITE]}${line.slice(type.length)}`); - } - ) + return CLASH_SUPPORTED_RULE_TYPE.flatMap(type => trie.find(`${type},`)).concat( + Object.keys(REQUIRE_REWRITE).flatMap((type) => trie.find(`${type},`) + .map(line => `${REQUIRE_REWRITE[type as keyof typeof REQUIRE_REWRITE]}${line.slice(type.length)}`)) ); }; diff --git a/Build/lib/fetch-assets.ts b/Build/lib/fetch-assets.ts index 09600c29e..f1ef4aa7c 100644 --- a/Build/lib/fetch-assets.ts +++ b/Build/lib/fetch-assets.ts @@ -12,20 +12,10 @@ const sleepWithAbort = (ms: number, signal: AbortSignal) => new Promise((r return; } - signal.addEventListener('abort', stop); - Bun.sleep(ms).then(done).catch(doReject); + function stop(this: AbortSignal) { reject(this.reason); } - function done() { - signal.removeEventListener('abort', stop); - resolve(); - } - function stop(this: AbortSignal) { - reject(this.reason); - } - function doReject(reason: unknown) { - signal.removeEventListener('abort', stop); - reject(reason); - } + signal.addEventListener('abort', stop, { once: true }); + Bun.sleep(ms).then(resolve).catch(reject).finally(() => signal.removeEventListener('abort', stop)); }); export async function fetchAssets(url: string, fallbackUrls: string[] | readonly string[]) { diff --git a/Build/lib/get-gorhill-publicsuffix.ts b/Build/lib/get-gorhill-publicsuffix.ts index 20f0b7633..6092f652c 100644 --- a/Build/lib/get-gorhill-publicsuffix.ts +++ b/Build/lib/get-gorhill-publicsuffix.ts @@ -1,11 +1,10 @@ import { toASCII } from 'punycode'; -import { traceAsync } from './trace-runner'; import { createMemoizedPromise } from './memo-promise'; import { getPublicSuffixListTextPromise } from '../download-publicsuffixlist'; const customFetch = (url: string | URL): Promise => Promise.resolve(Bun.file(url)); -export const getGorhillPublicSuffixPromise = createMemoizedPromise(() => traceAsync('create gorhill public suffix instance', async () => { +export const getGorhillPublicSuffixPromise = createMemoizedPromise(async () => { const [publicSuffixListDat, { default: gorhill }] = await Promise.all([ getPublicSuffixListTextPromise(), import('@gorhill/publicsuffixlist') @@ -15,4 +14,4 @@ export const getGorhillPublicSuffixPromise = createMemoizedPromise(() => traceAs await gorhill.enableWASM({ customFetch }); return gorhill; -})); +}); diff --git a/Build/lib/memo-promise.ts b/Build/lib/memo-promise.ts index 0c7011139..5e26332a4 100644 --- a/Build/lib/memo-promise.ts +++ b/Build/lib/memo-promise.ts @@ -1,9 +1,5 @@ export const createMemoizedPromise = (fn: () => Promise, preload = true): () => Promise => { - let promise: Promise | null = null; - - if (preload) { - promise = fn(); - } + let promise: Promise | null = preload ? fn() : null; return () => { promise ??= fn(); diff --git a/Build/lib/trace-runner.ts b/Build/lib/trace-runner.ts deleted file mode 100644 index d1eed852e..000000000 --- a/Build/lib/trace-runner.ts +++ /dev/null @@ -1,20 +0,0 @@ -import picocolors from 'picocolors'; - -type Formatter = (result: string) => string; - -export function traceSync(prefix: string, fn: () => T, timeFormatter: Formatter = picocolors.blue): T { - const start = Bun.nanoseconds(); - const result = fn(); - const end = Bun.nanoseconds(); - console.log(`${timeFormatter(`[${((end - start) / 1e6).toFixed(3)}ms]`)} ${prefix}`); - return result; -} -// traceSync.skip = (_prefix: string, fn: () => T): T => fn(); - -export const traceAsync = async (prefix: string, fn: () => Promise, timeFormatter: Formatter = picocolors.blue): Promise => { - const start = Bun.nanoseconds(); - const result = await fn(); - const end = Bun.nanoseconds(); - console.log(`${timeFormatter(`[${((end - start) / 1e6).toFixed(3)}ms]`)} ${prefix}`); - return result; -}; diff --git a/Build/lib/trie.ts b/Build/lib/trie.ts index 1f96800f7..cc6690675 100644 --- a/Build/lib/trie.ts +++ b/Build/lib/trie.ts @@ -22,10 +22,14 @@ const deepTrieNodeToJSON = (node: TrieNode) => { return obj; }; +function trieNodeInspectCustom(this: TrieNode) { + return JSON.stringify(deepTrieNodeToJSON(this), null, 2); +} + const createNode = (): TrieNode => { const node = new Map() as TrieNode; node[SENTINEL] = false; - node[Bun.inspect.custom] = () => JSON.stringify(deepTrieNodeToJSON(node), null, 2); + node[Bun.inspect.custom] = trieNodeInspectCustom; return node; }; @@ -70,9 +74,7 @@ export const createTrie = (from?: string[] | Set | null) => { token = suffix[i]; node = node.get(token); - if (!node) { - return false; - } + if (!node) return false; } return true; @@ -88,9 +90,7 @@ export const createTrie = (from?: string[] | Set | null) => { token = inputSuffix[i]; node = node.get(token); - if (!node) { - return []; - } + if (!node) return []; } const matches: string[] = []; @@ -130,9 +130,7 @@ export const createTrie = (from?: string[] | Set | null) => { token = inputSuffix[i]; node = node.get(token); - if (!node) { - return; - } + if (!node) return; } // Performing DFS from prefix @@ -213,11 +211,11 @@ export const createTrie = (from?: string[] | Set | null) => { for (let i = suffix.length - 1; i >= 0; i--) { const token = suffix[i]; - if (node.has(token)) { - node = node.get(token)!; - } else { + if (!node.has(token)) { return false; } + + node = node.get(token)!; } return node[SENTINEL]; diff --git a/Modules/google_cn_307.sgmodule b/Modules/google_cn_307.sgmodule index 2ef9cc968..c55d55f33 100644 --- a/Modules/google_cn_307.sgmodule +++ b/Modules/google_cn_307.sgmodule @@ -2,7 +2,7 @@ #!desc=将 google.cn 的请求重定向到 google.com [URL Rewrite] -^(http|https):\/\/(www.)?(g|google)\.cn https://www.google.com 307 +^https?://(www.)?(g|google)\.cn https://www.google.com 307 [MITM] hostname = %APPEND% g.cn,www.g.cn,google.cn,www.google.cn diff --git a/Modules/sukka_enhance_adblock.sgmodule b/Modules/sukka_enhance_adblock.sgmodule index 08a98d4cb..ae3bd1997 100644 --- a/Modules/sukka_enhance_adblock.sgmodule +++ b/Modules/sukka_enhance_adblock.sgmodule @@ -15,3 +15,4 @@ hostname = %APPEND% *.google-analytics.com, *.googletagmanager.com, *.googlesynd ^https?://.*\.doubleclick\.net/instream/ad_status\.js https://ruleset.skk.moe/Mock/doubleclick-net_instream_ad_status.js 302 ^https?://cdn\.ampproject\.org/v0.js https://ruleset.skk.moe/Mock/ampproject-org_v0.js 302 ^https?://.*\.addthis\.com/addthis_widget.js https://ruleset.skk.moe/Mock/addthis-com_addthis_widget.js 302 +^https?://.*\.amazon-adsystem.com/aax2/apstag.js https://ruleset.skk.moe/Mock/amazon-adsystem-com_amazon-apstag.js 302 diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index fa6aae9a5..fa9c7b9ae 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -611,6 +611,7 @@ i.imgflip.com .yourimageshare.com .iili.io img.hcloud.lat +.imglink.win # imgix custom domain www.datocms-assets.com images.pexels.com @@ -1348,6 +1349,7 @@ images.opencollective.com static.figma.com s3-alpha-sig.figma.com s3-alpha.figma.com +s3-figma-plugin-images-production-sig.figma.com cdn.sprig.com # yande.re assets.yande.re @@ -2379,3 +2381,6 @@ redircdn.com images7.design-editor.com images8.design-editor.com images.design-editor.com +assets.st-note.com +filestore.fortinet.com +.daumcdn.net diff --git a/Source/domainset/reject_sukka.conf b/Source/domainset/reject_sukka.conf index 372a64010..d02a29fb2 100644 --- a/Source/domainset/reject_sukka.conf +++ b/Source/domainset/reject_sukka.conf @@ -20,6 +20,37 @@ optimus-ads.amap.com .taio.app .headcaptcha.live .coinzilla.io +.p2perrorlog.douyucdn.cn +.114la.com +.365key.com +.51.la +.adnyg.com +.baifendian.com +.crazyegg.com +.demdex.net +.donews.com +.duomai.com +.growingio.com +.kuaizhan.com +.lapin365.com +.miaozhen.com +.omtrdc.net +.pv.com.cn +.scorecardresearch.com +.sensorsdata.cn +.stargame.com +.taboola.com.cn +.taboola.com +.tagtic.cn +.testin.cn +.tiqcdn.cn +.tiqcdn.com +.ugdtimg.com +.woopra.com +.xlingdi.com +.zampdsp.com +.zhugeio.com +.telemetry.coder.com # fake pirate bay .thepiratebay3.com # malware / scam @@ -661,6 +692,7 @@ analytics.slashdotmedia.com .bkrtx.com .ads.pro-market.net .stat.xtom.com +.stat.cat.net .analytics.archive.org .stats.yinyuetai.com .collect.yinyuetai.com @@ -672,7 +704,8 @@ analytics.infomaniak.com p.cloudcraft.co collect.foresightnews.pro stats.snipaste.com -.plausible.cyberciti. +.plausible.cyberciti.com +umami.rss3.io .adjust.io .airbrake.io @@ -835,6 +868,8 @@ bingads.microsoft.com .googletraveladservices.com # >> Huawei +.nebula-collector.huawei.com + .data.hicloud.com logservice.hicloud.com logservice1.hicloud.com @@ -1091,6 +1126,7 @@ videopush.baidu.com .tracker.baidu.com .tuijian.baidu.com .ubmcmm.baidustatic.com +.cbjs.baidustatic.com # >> Qihoo 360 .stat.360safe.com @@ -1661,6 +1697,7 @@ log.ycapp.yiche.com thirdparty.biliapi.com thirdparty.biliapi.net cm.bilibili.com +.data.bilibili.com # >> CNTV galaxy.bjcathay.com