diff --git a/Build/build-chn-cidr.ts b/Build/build-chn-cidr.ts index 0ada565e1..6295f9950 100644 --- a/Build/build-chn-cidr.ts +++ b/Build/build-chn-cidr.ts @@ -11,12 +11,12 @@ const EXCLUDE_CIDRS = [ ]; export const buildChnCidr = task(__filename, async () => { - const [{ exclude: excludeCidrs }, cidr] = await Promise.all([ + const [{ exclude }, cidr] = await Promise.all([ import('cidr-tools-wasm'), processLineFromReadline(await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')) ]); - const filteredCidr = excludeCidrs(cidr, EXCLUDE_CIDRS, true); + const filteredCidr = exclude(cidr, EXCLUDE_CIDRS, true); const description = [ 'License: CC BY-SA 2.0', diff --git a/Build/build-speedtest-domainset.ts b/Build/build-speedtest-domainset.ts index 9298f99d9..f0d45e627 100644 --- a/Build/build-speedtest-domainset.ts +++ b/Build/build-speedtest-domainset.ts @@ -6,18 +6,29 @@ import domainSorter from './lib/stable-sort-domain'; import { Sema } from 'async-sema'; import * as tldts from 'tldts'; import { task } from './lib/trace-runner'; -const s = new Sema(2); +const s = new Sema(3); + +const latestTopUserAgentsPromise = fetch('https://unpkg.com/top-user-agents@latest/index.json') + .then(res => res.json() as Promise); const querySpeedtestApi = async (keyword: string): Promise<(string | null)[]> => { - await s.acquire(); + const [topUserAgents] = await Promise.all([ + latestTopUserAgentsPromise, + s.acquire() + ]); + + const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)]; try { + const key = `fetch speedtest endpoints: ${keyword}`; + console.time(key); + const res = await fetch(`https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`, { headers: { dnt: '1', Referer: 'https://www.speedtest.net/', accept: 'application/json, text/plain, */*', - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', + 'User-Agent': randomUserAgent, 'Accept-Language': 'en-US,en;q=0.9', 'Sec-Ch-Ua': '"Not/A)Brand";v="99", "Google Chrome";v="115", "Chromium";v="115"', 'Sec-Ch-Ua-Mobile': '?0', @@ -28,12 +39,14 @@ const querySpeedtestApi = async (keyword: string): Promise<(string | null)[]> => } }); if (!res.ok) { - const text = await res.text(); - throw new Error(text); + throw new Error(res.statusText + '\n' + await res.text()); } const json = await res.json() as { url: string; }[]; s.release(); + + console.timeEnd(key); + return json.map(({ url }) => tldts.getHostname(url, { detectIp: false })); } catch (e) { s.release(); diff --git a/Build/download-previous-build.ts b/Build/download-previous-build.ts index 65da50cb2..7369d057b 100644 --- a/Build/download-previous-build.ts +++ b/Build/download-previous-build.ts @@ -75,8 +75,6 @@ export const downloadPreviousBuild = task(__filename, async () => { ); } })); - - // return fsp.unlink(extractedPath).catch(() => { }); }); export const downloadPublicSuffixList = task(__filename, async () => { diff --git a/Build/index.ts b/Build/index.ts index ac15fbc05..df85b5f08 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -16,39 +16,12 @@ import { buildRedirectModule } from './build-redirect-module'; import { validate } from './validate-domainset'; import { buildPublicHtml } from './build-public'; - -import { Worker } from 'jest-worker'; - -type WithWorker = import('jest-worker').Worker & { __sukka_worker_name: string } & T - -const requireWorker = (path: string, exposedMethods?: (keyof T & string)[]): WithWorker => { - const _worker = new Worker( - import.meta.require.resolve(path), - { - numWorkers: 1, - maxRetries: 0, - enableWorkerThreads: true, - exposedMethods - } - ) as WithWorker; - _worker.getStderr().pipe(process.stderr); - _worker.getStdout().pipe(process.stdout); - _worker.__sukka_worker_name = path; - return _worker; -}; - -const endWorker = async (worker: WithWorker) => { - const { forceExited } = await worker.end(); - if (forceExited && worker.__sukka_worker_name) { - console.log(worker.__sukka_worker_name, 'forceExited'); - } -}; +import { TaskResult } from './lib/trace-runner'; (async () => { - const buildInternalReverseChnCIDRWorker: WithWorker = requireWorker('./build-internal-reverse-chn-cidr', ['buildInternalReverseChnCIDR']); - try { - const { buildInternalReverseChnCIDR } = buildInternalReverseChnCIDRWorker; + console.log('Bun version:', Bun.version); + try { const downloadPreviousBuildPromise = downloadPreviousBuild(); const downloadPublicSuffixListPromise = downloadPublicSuffixList(); const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon()); @@ -75,7 +48,18 @@ const endWorker = async (worker: WithWorker) => { buildCommonPromise, buildCdnConfPromise ]).then(() => buildInternalCDNDomains()); - const buildInternalReverseChnCIDRPromise = buildInternalReverseChnCIDR(); + + const buildInternalReverseChnCIDRPromise = new Promise(resolve => { + const buildInternalReverseChnCIDRWorker = new Worker(new URL('./workers/build-internal-reverse-chn-cidr-worker.ts', import.meta.url)); + + const handleMessage = (e: MessageEvent) => { + buildInternalReverseChnCIDRWorker.terminate(); + resolve(e.data); + } + + buildInternalReverseChnCIDRWorker.addEventListener('message', handleMessage); + }); + const buildInternalChnDomainsPromise = buildInternalChnDomains(); const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset()); @@ -109,9 +93,7 @@ const endWorker = async (worker: WithWorker) => { printStats(stats); } catch (e) { - console.error(e) - } finally { - await endWorker(buildInternalReverseChnCIDRWorker) + console.error(e); } })(); diff --git a/Build/lib/fetch-remote-text-by-line.ts b/Build/lib/fetch-remote-text-by-line.ts index 3e93dd168..2c5225ee9 100644 --- a/Build/lib/fetch-remote-text-by-line.ts +++ b/Build/lib/fetch-remote-text-by-line.ts @@ -56,7 +56,6 @@ export async function* createReadlineInterfaceFromResponse(resp: Response): Asyn } } -export async function fetchRemoteTextAndCreateReadlineInterface(url: string | URL, opt?: RequestInit): Promise> { - const resp = await fetchWithRetry(url, opt); - return createReadlineInterfaceFromResponse(resp); +export function fetchRemoteTextAndCreateReadlineInterface(url: string | URL, opt?: RequestInit): Promise> { + return fetchWithRetry(url, opt).then(res => createReadlineInterfaceFromResponse(res)); } diff --git a/Build/lib/parse-filter.ts b/Build/lib/parse-filter.ts index 89550c735..98df18d02 100644 --- a/Build/lib/parse-filter.ts +++ b/Build/lib/parse-filter.ts @@ -68,7 +68,7 @@ export async function processDomainLists(domainListsUrl: string | URL) { } export async function processHosts(hostsUrl: string | URL, includeAllSubDomain = false) { - console.time(` - processHosts: ${hostsUrl}`); + console.time(`- processHosts: ${hostsUrl}`); if (typeof hostsUrl === 'string') { hostsUrl = new URL(hostsUrl); diff --git a/Build/lib/trace-runner.ts b/Build/lib/trace-runner.ts index 3a0920a50..886ac177f 100644 --- a/Build/lib/trace-runner.ts +++ b/Build/lib/trace-runner.ts @@ -18,6 +18,12 @@ const traceAsync = async (prefix: string, fn: () => Promise): Promise = }; export { traceAsync }; +export interface TaskResult { + readonly start: number; + readonly end: number; + readonly taskName: string; +} + const task = (__filename: string, fn: () => Promise, customname: string | null = null) => { const taskName = customname ?? path.basename(__filename, path.extname(__filename)); return async () => { @@ -27,7 +33,7 @@ const task = (__filename: string, fn: () => Promise, customname: string | const end = performance.now(); console.log(`✅ [${taskName}] Executed successfully: ${(end - start).toFixed(3)}ms`); - return { start, end, taskName } as const; + return { start, end, taskName } as TaskResult; }; }; export { task }; diff --git a/Build/workers/build-internal-reverse-chn-cidr-worker.ts b/Build/workers/build-internal-reverse-chn-cidr-worker.ts new file mode 100644 index 000000000..562557e0c --- /dev/null +++ b/Build/workers/build-internal-reverse-chn-cidr-worker.ts @@ -0,0 +1,5 @@ +(async () => { + const { buildInternalReverseChnCIDR } = await import('../build-internal-reverse-chn-cidr'); + const stat = await buildInternalReverseChnCIDR(); + postMessage(stat); +})(); diff --git a/bun.lockb b/bun.lockb index c38d15c9e..d69e15cd1 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index cf75aa711..4b6594587 100644 --- a/package.json +++ b/package.json @@ -19,10 +19,9 @@ "@vercel/fetch-retry": "^5.1.3", "async-sema": "^3.1.1", "ci-info": "^4.0.0", - "cidr-tools-wasm": "^0.0.11", + "cidr-tools-wasm": "^0.0.13", "eslint": "^8.53.0", "gorhill-publicsuffixlist": "github:gorhill/publicsuffixlist.js", - "jest-worker": "^29.7.0", "mnemonist": "^0.39.5", "path-scurry": "^1.10.1", "picocolors": "^1.0.0",