diff --git a/Build/build-cloudmounter-rules.ts b/Build/build-cloudmounter-rules.ts index a69dcd874..2ca886980 100644 --- a/Build/build-cloudmounter-rules.ts +++ b/Build/build-cloudmounter-rules.ts @@ -12,7 +12,7 @@ export const buildCloudMounterRules = task(require.main === module, __filename)( `AND,((${domain}),(PROCESS-NAME,${process}))`, ...[ '10.0.0.0/8', - '127.0.0.0/8', + // '127.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16' ].map(cidr => `AND,((${domain}),(SRC-IP,${cidr}))`) diff --git a/Build/build-internal-reverse-chn-cidr.ts b/Build/build-internal-reverse-chn-cidr.ts index f2b2245bd..3ba1c10b2 100644 --- a/Build/build-internal-reverse-chn-cidr.ts +++ b/Build/build-internal-reverse-chn-cidr.ts @@ -5,8 +5,10 @@ import { exclude, merge } from 'fast-cidr-tools'; import { getChnCidrPromise } from './build-chn-cidr'; import { NON_CN_CIDR_INCLUDED_IN_CHNROUTE, RESERVED_IPV4_CIDR } from './constants/cidr'; -import { writeFile } from './lib/misc'; +import fs from 'node:fs'; import { OUTPUT_INTERNAL_DIR } from './constants/dir'; +import { asyncWriteToStream } from './lib/async-write-to-stream'; +import { mkdirp } from './lib/misc'; export const buildInternalReverseChnCIDR = task(require.main === module, __filename)(async () => { const [cidr] = await getChnCidrPromise(); @@ -23,9 +25,17 @@ export const buildInternalReverseChnCIDR = task(require.main === module, __filen ); const outputFile = path.join(OUTPUT_INTERNAL_DIR, 'reversed-chn-cidr.txt'); + await mkdirp(OUTPUT_INTERNAL_DIR); - return writeFile( - outputFile, - reversedCidr.join('\n') + '\n' - ); + const writeStream = fs.createWriteStream(outputFile); + for (const line of reversedCidr) { + const p = asyncWriteToStream(writeStream, line + '\n'); + if (p) { + // eslint-disable-next-line no-await-in-loop -- stream high water mark + await p; + } + } + await asyncWriteToStream(writeStream, '\n'); + + writeStream.end(); }); diff --git a/Build/build-public.ts b/Build/build-public.ts index 308051524..61dca6695 100644 --- a/Build/build-public.ts +++ b/Build/build-public.ts @@ -7,7 +7,7 @@ import { treeDir } from './lib/tree-dir'; import type { TreeType, TreeTypeArray } from './lib/tree-dir'; import { OUTPUT_MOCK_DIR, OUTPUT_MODULES_DIR, PUBLIC_DIR, ROOT_DIR } from './constants/dir'; -import { writeFile } from './lib/misc'; +import { mkdirp, writeFile } from './lib/misc'; import picocolors from 'picocolors'; const mockDir = path.join(ROOT_DIR, 'Mock'); @@ -32,8 +32,8 @@ const copyDirContents = async (srcDir: string, destDir: string) => { export const buildPublic = task(require.main === module, __filename)(async (span) => { await span.traceChildAsync('copy rest of the files', async () => { await Promise.all([ - fsp.mkdir(OUTPUT_MODULES_DIR, { recursive: true }), - fsp.mkdir(OUTPUT_MOCK_DIR, { recursive: true }) + mkdirp(OUTPUT_MODULES_DIR), + mkdirp(OUTPUT_MOCK_DIR) ]); await Promise.all([ diff --git a/Build/download-mock-assets.ts b/Build/download-mock-assets.ts index 2fd074a79..6a9e34589 100644 --- a/Build/download-mock-assets.ts +++ b/Build/download-mock-assets.ts @@ -1,11 +1,11 @@ import { task } from './trace'; import path from 'node:path'; import fs from 'node:fs'; -import fsp from 'node:fs/promises'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { fetchWithRetry } from './lib/fetch-retry'; import { OUTPUT_MOCK_DIR } from './constants/dir'; +import { mkdirp } from './lib/misc'; const ASSETS_LIST = { 'www-google-analytics-com_ga.js': 'https://raw.githubusercontent.com/AdguardTeam/Scriptlets/master/dist/redirect-files/google-analytics-ga.js', @@ -25,7 +25,8 @@ export const downloadMockAssets = task(require.main === module, __filename)((spa throw new Error(`Empty body from ${url}`); } - await fsp.mkdir(OUTPUT_MOCK_DIR, { recursive: true }); + await mkdirp(OUTPUT_MOCK_DIR); + return pipeline( Readable.fromWeb(res.body), fs.createWriteStream(src, 'utf-8') diff --git a/Build/lib/async-write-to-stream.ts b/Build/lib/async-write-to-stream.ts new file mode 100644 index 000000000..0c067ec59 --- /dev/null +++ b/Build/lib/async-write-to-stream.ts @@ -0,0 +1,9 @@ +import type { Writable } from 'node:stream'; +import { once } from 'node:events'; + +export const asyncWriteToStream = (stream: Writable, chunk: T) => { + const res = stream.write(chunk); + if (!res) { + return once(stream, 'drain'); // returns a promise only if needed + } +}; diff --git a/Build/lib/convert-clash-meta-mrs.ts b/Build/lib/convert-clash-meta-mrs.ts index 5e18a2f11..b4ec81226 100644 --- a/Build/lib/convert-clash-meta-mrs.ts +++ b/Build/lib/convert-clash-meta-mrs.ts @@ -7,6 +7,7 @@ import zlib from 'node:zlib'; import process from 'node:process'; import { async as ezspawn } from '@jsdevtools/ez-spawn'; +import { mkdirp } from './misc'; const mihomoBinaryDir = path.join(__dirname, '../../node_modules/.cache/mihomo'); const mihomoBinaryPath = path.join(mihomoBinaryDir, 'mihomo'); @@ -22,7 +23,7 @@ const mihomoBinaryUrl: Partial { - await fsp.mkdir(mihomoBinaryDir, { recursive: true }); + await mkdirp(mihomoBinaryDir); if (!fs.existsSync(mihomoBinaryPath)) { const writeStream = fs.createWriteStream(mihomoBinaryPath); diff --git a/Build/lib/create-file.ts b/Build/lib/create-file.ts index a501722eb..a8790d41c 100644 --- a/Build/lib/create-file.ts +++ b/Build/lib/create-file.ts @@ -10,6 +10,7 @@ import stringify from 'json-stringify-pretty-compact'; import { ipCidrListToSingbox, surgeDomainsetToSingbox, surgeRulesetToSingbox } from './singbox'; import { createTrie } from './trie'; import { pack, unpackFirst, unpackSecond } from './bitwise'; +import { asyncWriteToStream } from './async-write-to-stream'; export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) { let isEqual = true; @@ -67,17 +68,24 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath } await span.traceChildAsync(`writing ${filePath}`, async () => { - // if (linesALen < 10000) { - return writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n'); - // } - // const writer = file.writer(); - - // for (let i = 0; i < linesALen; i++) { - // writer.write(linesA[i]); - // writer.write('\n'); - // } + // The default highwater mark is normally 16384, + // So we make sure direct write to file if the content is + // most likely less than 500 lines + if (linesALen < 500) { + return writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n'); + } - // return writer.end(); + const writeStream = fs.createWriteStream(filePath); + for (let i = 0; i < linesALen; i++) { + let p = asyncWriteToStream(writeStream, linesA[i]); + // eslint-disable-next-line no-await-in-loop -- stream high water mark + if (p) await p; + p = asyncWriteToStream(writeStream, '\n'); + // eslint-disable-next-line no-await-in-loop -- stream high water mark + if (p) await p; + } + await asyncWriteToStream(writeStream, '\n'); + writeStream.end(); }); } diff --git a/Build/lib/misc.ts b/Build/lib/misc.ts index 182e6bbff..723d9e1d1 100644 --- a/Build/lib/misc.ts +++ b/Build/lib/misc.ts @@ -23,9 +23,17 @@ interface Write { ): Promise } +export const mkdirp = (dir: string) => { + if (fs.existsSync(dir)) { + return; + } + return fsp.mkdir(dir, { recursive: true }); +}; + export const writeFile: Write = async (destination: string, input, dir = dirname(destination)) => { - if (!fs.existsSync(dir)) { - await fsp.mkdir(dir, { recursive: true }); + const p = mkdirp(dir); + if (p) { + await p; } return fsp.writeFile(destination, input, { encoding: 'utf-8' }); }; diff --git a/Source/domainset/download.conf b/Source/domainset/download.conf index 0c5e1ceb6..b55afeebf 100644 --- a/Source/domainset/download.conf +++ b/Source/domainset/download.conf @@ -356,3 +356,4 @@ ciscobinary.openh264.org update.sleazyfork.org updates.istatmenus.app download.maxmind.com +downloader.cursor.sh