Merge branch 'SukkaW:master' into master

FYLSen · Jan 23, 2025 · 336cf56 · 336cf56
2 parents 2931191 + 3c2b49d
commit 336cf56
Show file tree

Hide file tree

Showing 19 changed files with 143 additions and 256 deletions.
diff --git a/Build/build-common.ts b/Build/build-common.ts
@@ -16,8 +16,6 @@ const MAGIC_COMMAND_TITLE = '# $ meta_title ';
 const MAGIC_COMMAND_DESCRIPTION = '# $ meta_description ';
 const MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES = '# $ sgmodule_mitm_hostnames ';
 
-const domainsetSrcFolder = 'domainset' + path.sep;
-
 const clawSourceDirPromise = new Fdir()
   .withRelativePaths()
   .filter((filepath, isDirectory) => {
@@ -39,15 +37,11 @@ export const buildCommon = task(require.main === module, __filename)(async (span
     const relativePath = paths[i];
     const fullPath = SOURCE_DIR + path.sep + relativePath;
 
-    if (relativePath.startsWith(domainsetSrcFolder)) {
-      promises.push(transformDomainset(span, fullPath));
-      continue;
-    }
     // if (
     //   relativePath.startsWith('ip/')
     //   || relativePath.startsWith('non_ip/')
     // ) {
-    promises.push(transformRuleset(span, fullPath, relativePath));
+    promises.push(transform(span, fullPath, relativePath));
     // continue;
     // }
 
@@ -102,71 +96,44 @@ function processFile(span: Span, sourcePath: string) {
   });
 }
 
-function transformDomainset(parentSpan: Span, sourcePath: string) {
-  const extname = path.extname(sourcePath);
-  const basename = path.basename(sourcePath, extname);
-  return parentSpan
-    .traceChildAsync(
-      `transform domainset: ${basename}`,
-      async (span) => {
-        const res = await processFile(span, sourcePath);
-        if (res === $skip) return;
-
-        const id = basename;
-        const [title, incomingDescriptions, lines] = res;
-
-        let finalDescriptions: string[];
-        if (incomingDescriptions.length) {
-          finalDescriptions = SHARED_DESCRIPTION.slice();
-          finalDescriptions.push('');
-          appendArrayInPlace(finalDescriptions, incomingDescriptions);
-        } else {
-          finalDescriptions = SHARED_DESCRIPTION;
-        }
-
-        return new DomainsetOutput(span, id)
-          .withTitle(title)
-          .withDescription(finalDescriptions)
-          .addFromDomainset(lines)
-          .write();
-      }
-    );
-}
-
-/**
- * Output Surge RULE-SET and Clash classical text format
- */
-async function transformRuleset(parentSpan: Span, sourcePath: string, relativePath: string) {
+async function transform(parentSpan: Span, sourcePath: string, relativePath: string) {
   const extname = path.extname(sourcePath);
-  const basename = path.basename(sourcePath, extname);
+  const id = path.basename(sourcePath, extname);
 
   return parentSpan
-    .traceChild(`transform ruleset: ${basename}`)
+    .traceChild(`transform ruleset: ${id}`)
     .traceAsyncFn(async (span) => {
-      const res = await processFile(span, sourcePath);
-      if (res === $skip) return;
+      const type = relativePath.split(path.sep)[0];
 
-      const id = basename;
-      const type = relativePath.slice(0, -extname.length).split(path.sep)[0];
-
-      if (type !== 'ip' && type !== 'non_ip') {
+      if (type !== 'ip' && type !== 'non_ip' && type !== 'domainset') {
         throw new TypeError(`Invalid type: ${type}`);
       }
 
+      const res = await processFile(span, sourcePath);
+      if (res === $skip) return;
+
       const [title, descriptions, lines, sgmodulePathname] = res;
 
-      let description: string[];
+      let finalDescriptions: string[];
       if (descriptions.length) {
-        description = SHARED_DESCRIPTION.slice();
-        description.push('');
-        appendArrayInPlace(description, descriptions);
+        finalDescriptions = SHARED_DESCRIPTION.slice();
+        finalDescriptions.push('');
+        appendArrayInPlace(finalDescriptions, descriptions);
       } else {
-        description = SHARED_DESCRIPTION;
+        finalDescriptions = SHARED_DESCRIPTION;
+      }
+
+      if (type === 'domainset') {
+        return new DomainsetOutput(span, id)
+          .withTitle(title)
+          .withDescription(finalDescriptions)
+          .addFromDomainset(lines)
+          .write();
       }
 
       return new RulesetOutput(span, id, type)
         .withTitle(title)
-        .withDescription(description)
+        .withDescription(finalDescriptions)
         .withMitmSgmodulePath(sgmodulePathname)
         .addFromRuleset(lines)
         .write();

diff --git a/Build/build-reject-ip-list.ts b/Build/build-reject-ip-list.ts
@@ -3,12 +3,11 @@ import path from 'node:path';
 import { createReadlineInterfaceFromResponse, readFileIntoProcessedArray } from './lib/fetch-text-by-line';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './constants/description';
-import { isProbablyIpv4, isProbablyIpv6 } from 'foxts/is-probably-ip';
-import { processLine } from './lib/process-line';
 import { RulesetOutput } from './lib/create-file';
 import { SOURCE_DIR } from './constants/dir';
 import { $$fetch } from './lib/fetch-retry';
 import { fetchAssets } from './lib/fetch-assets';
+import { fastIpVersion } from './lib/misc';
 
 const BOGUS_NXDOMAIN_URL = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf';
 const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = $$fetch(BOGUS_NXDOMAIN_URL).then(async (resp) => {
@@ -18,9 +17,10 @@ const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = $$
   for await (const line of createReadlineInterfaceFromResponse(resp, true)) {
     if (line.startsWith('bogus-nxdomain=')) {
       const ip = line.slice(15).trim();
-      if (isProbablyIpv4(ip)) {
+      const v = fastIpVersion(ip);
+      if (v === 4) {
         ipv4.push(ip);
-      } else if (isProbablyIpv6(ip)) {
+      } else if (v === 6) {
         ipv6.push(ip);
       }
     }
@@ -37,14 +37,12 @@ const BOTNET_FILTER_MIRROR_URL = [
   // https://curbengh.github.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt
 ];
 
-const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets(BOTNET_FILTER_URL, BOTNET_FILTER_MIRROR_URL).then(text => text.split('\n').reduce<[ipv4: string[], ipv6: string[]]>((acc, cur) => {
-  const ip = processLine(cur);
-  if (ip) {
-    if (isProbablyIpv4(ip)) {
-      acc[0].push(ip);
-    } else if (isProbablyIpv6(ip)) {
-      acc[1].push(ip);
-    }
+const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets(BOTNET_FILTER_URL, BOTNET_FILTER_MIRROR_URL, true).then(arr => arr.reduce<[ipv4: string[], ipv6: string[]]>((acc, ip) => {
+  const v = fastIpVersion(ip);
+  if (v === 4) {
+    acc[0].push(ip);
+  } else if (v === 6) {
+    acc[1].push(ip);
   }
   return acc;
 }, [[], []]));

diff --git a/Build/build-telegram-cidr.ts b/Build/build-telegram-cidr.ts
@@ -1,11 +1,11 @@
 // @ts-check
 import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line';
-import { isProbablyIpv4, isProbablyIpv6 } from 'foxts/is-probably-ip';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './constants/description';
 import { createMemoizedPromise } from './lib/memo-promise';
 import { RulesetOutput } from './lib/create-file';
 import { $$fetch } from './lib/fetch-retry';
+import { fastIpVersion } from './lib/misc';
 
 export const getTelegramCIDRPromise = createMemoizedPromise(async () => {
   const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt');
@@ -20,11 +20,10 @@ export const getTelegramCIDRPromise = createMemoizedPromise(async () => {
   const ipcidr6: string[] = [];
 
   for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) {
-    const [subnet] = cidr.split('/');
-    if (isProbablyIpv4(subnet)) {
+    const v = fastIpVersion(cidr);
+    if (v === 4) {
       ipcidr.push(cidr);
-    }
-    if (isProbablyIpv6(subnet)) {
+    } else if (v === 6) {
       ipcidr6.push(cidr);
     }
   }

diff --git a/Build/lib/cache-filesystem.ts b/Build/lib/cache-filesystem.ts
@@ -199,7 +199,7 @@ export const deserializeSet = (str: string) => new Set(str.split(separator));
 export const serializeArray = (arr: string[]) => fastStringArrayJoin(arr, separator);
 export const deserializeArray = (str: string) => str.split(separator);
 
-export const getFileContentHash = (filename: string) => simpleStringHash(fs.readFileSync(filename, 'utf-8'));
+const getFileContentHash = (filename: string) => simpleStringHash(fs.readFileSync(filename, 'utf-8'));
 export function createCacheKey(filename: string) {
   const fileHash = getFileContentHash(filename);
   return (key: string) => key + '$' + fileHash + '$';

diff --git a/Build/lib/fetch-assets.ts b/Build/lib/fetch-assets.ts
@@ -1,32 +1,19 @@
 import picocolors from 'picocolors';
 import { $$fetch, defaultRequestInit, ResponseError } from './fetch-retry';
 import { waitWithAbort } from 'foxts/wait';
+import { nullthrow } from 'foxts/guard';
+import { TextLineStream } from './text-line-transform-stream';
+import { ProcessLineStream } from './process-line';
 
 // eslint-disable-next-line sukka/unicorn/custom-error-definition -- typescript is better
 export class CustomAbortError extends Error {
   public readonly name = 'AbortError';
   public readonly digest = 'AbortError';
 }
 
-export class Custom304NotModifiedError extends Error {
-  public readonly name = 'Custom304NotModifiedError';
-  public readonly digest = 'Custom304NotModifiedError';
+const reusedCustomAbortError = new CustomAbortError();
 
-  constructor(public readonly url: string, public readonly data: string) {
-    super('304 Not Modified');
-  }
-}
-
-export class CustomNoETagFallbackError extends Error {
-  public readonly name = 'CustomNoETagFallbackError';
-  public readonly digest = 'CustomNoETagFallbackError';
-
-  constructor(public readonly data: string) {
-    super('No ETag Fallback');
-  }
-}
-
-export async function fetchAssets(url: string, fallbackUrls: null | undefined | string[] | readonly string[]) {
+export async function fetchAssets(url: string, fallbackUrls: null | undefined | string[] | readonly string[], processLine = false) {
   const controller = new AbortController();
 
   const createFetchFallbackPromise = async (url: string, index: number) => {
@@ -36,22 +23,27 @@ export async function fetchAssets(url: string, fallbackUrls: null | undefined |
         await waitWithAbort(50 + (index + 1) * 100, controller.signal);
       } catch {
         console.log(picocolors.gray('[fetch cancelled early]'), picocolors.gray(url));
-        throw new CustomAbortError();
+        throw reusedCustomAbortError;
       }
     }
     if (controller.signal.aborted) {
       console.log(picocolors.gray('[fetch cancelled]'), picocolors.gray(url));
-      throw new CustomAbortError();
+      throw reusedCustomAbortError;
     }
     const res = await $$fetch(url, { signal: controller.signal, ...defaultRequestInit });
-    const text = await res.text();
 
-    if (text.length < 2) {
+    let stream = nullthrow(res.body).pipeThrough(new TextDecoderStream()).pipeThrough(new TextLineStream());
+    if (processLine) {
+      stream = stream.pipeThrough(new ProcessLineStream());
+    }
+    const arr = await Array.fromAsync(stream);
+
+    if (arr.length < 1) {
       throw new ResponseError(res, url, 'empty response w/o 304');
     }
 
     controller.abort();
-    return text;
+    return arr;
   };
 
   if (!fallbackUrls || fallbackUrls.length === 0) {

diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts
@@ -34,11 +34,7 @@ setGlobalDispatcher(agent.compose(
     // TODO: this part of code is only for allow more errors to be retried by default
     // This should be removed once https://github.com/nodejs/undici/issues/3728 is implemented
     retry(err, { state, opts }, cb) {
-      const statusCode = 'statusCode' in err && typeof err.statusCode === 'number' ? err.statusCode : null;
       const errorCode = 'code' in err ? (err as NodeJS.ErrnoException).code : undefined;
-      const headers = ('headers' in err && typeof err.headers === 'object') ? err.headers : undefined;
-
-      const { counter } = state;
 
       // Any code that is not a Undici's originated and allowed to retry
       if (
@@ -49,42 +45,44 @@ setGlobalDispatcher(agent.compose(
         return cb(err);
       }
 
+      const statusCode = 'statusCode' in err && typeof err.statusCode === 'number' ? err.statusCode : null;
+
+      // bail out if the status code matches one of the following
+      if (
+        statusCode != null
+        && (
+          statusCode === 401 // Unauthorized, should check credentials instead of retrying
+          || statusCode === 403 // Forbidden, should check permissions instead of retrying
+          || statusCode === 404 // Not Found, should check URL instead of retrying
+          || statusCode === 405 // Method Not Allowed, should check method instead of retrying
+        )
+      ) {
+        return cb(err);
+      }
+
       // if (errorCode === 'UND_ERR_REQ_RETRY') {
       //   return cb(err);
       // }
 
-      const { method, retryOptions = {} } = opts;
-
       const {
         maxRetries = 5,
         minTimeout = 500,
         maxTimeout = 10 * 1000,
         timeoutFactor = 2,
         methods = ['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE', 'TRACE']
-      } = retryOptions;
+      } = opts.retryOptions || {};
 
       // If we reached the max number of retries
-      if (counter > maxRetries) {
+      if (state.counter > maxRetries) {
         return cb(err);
       }
 
       // If a set of method are provided and the current method is not in the list
-      if (Array.isArray(methods) && !methods.includes(method)) {
+      if (Array.isArray(methods) && !methods.includes(opts.method)) {
         return cb(err);
       }
 
-      // bail out if the status code matches one of the following
-      if (
-        statusCode != null
-        && (
-          statusCode === 401 // Unauthorized, should check credentials instead of retrying
-          || statusCode === 403 // Forbidden, should check permissions instead of retrying
-          || statusCode === 404 // Not Found, should check URL instead of retrying
-          || statusCode === 405 // Method Not Allowed, should check method instead of retrying
-        )
-      ) {
-        return cb(err);
-      }
+      const headers = ('headers' in err && typeof err.headers === 'object') ? err.headers : undefined;
 
       const retryAfterHeader = (headers as Record<string, string> | null | undefined)?.['retry-after'];
       let retryAfter = -1;
@@ -97,7 +95,7 @@ setGlobalDispatcher(agent.compose(
 
       const retryTimeout = retryAfter > 0
         ? Math.min(retryAfter, maxTimeout)
-        : Math.min(minTimeout * (timeoutFactor ** (counter - 1)), maxTimeout);
+        : Math.min(minTimeout * (timeoutFactor ** (state.counter - 1)), maxTimeout);
 
       console.log('[fetch retry]', 'schedule retry', { statusCode, retryTimeout, errorCode, url: opts.origin });
       // eslint-disable-next-line sukka/prefer-timer-id -- won't leak

diff --git a/Build/lib/fetch-text-by-line.bench.ts b/Build/lib/fetch-text-by-line.bench.ts
@@ -1,4 +1,4 @@
-import { readFileByLine, readFileByLineLegacy, readFileByLineNew } from './fetch-text-by-line';
+import { readFileByLine, readFileByLineNew } from './fetch-text-by-line';
 import path from 'node:path';
 import fsp from 'node:fs/promises';
 import { OUTPUT_SURGE_DIR } from '../constants/dir';
@@ -10,7 +10,6 @@ const file = path.join(OUTPUT_SURGE_DIR, 'domainset/reject_extra.conf');
 
   group(() => {
     bench('readFileByLine', () => Array.fromAsync(readFileByLine(file)));
-    bench('readFileByLineLegacy', () => Array.fromAsync(readFileByLineLegacy(file)));
     bench('readFileByLineNew', async () => Array.fromAsync(await readFileByLineNew(file)));
     bench('fsp.readFile', () => fsp.readFile(file, 'utf-8').then((content) => content.split('\n')));
   });