Skip to content

Commit

Permalink
Merge branch 'SukkaW:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
FYLSen authored Aug 29, 2024
2 parents 87cc11e + 00abcdd commit 4326c8c
Show file tree
Hide file tree
Showing 47 changed files with 623 additions and 459 deletions.
2 changes: 1 addition & 1 deletion Build/build-cdn-download-conf.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import path from 'path';
import path from 'node:path';
import { createRuleset } from './lib/create-file';
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { createTrie } from './lib/trie';
Expand Down
2 changes: 1 addition & 1 deletion Build/build-cloudmounter-rules.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import path from 'path';
import path from 'node:path';
import { DOMAINS, PROCESS_NAMES } from '../Source/non_ip/cloudmounter';
import { SHARED_DESCRIPTION } from './lib/constants';
import { createRuleset } from './lib/create-file';
Expand Down
2 changes: 1 addition & 1 deletion Build/build-common.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// @ts-check

import * as path from 'path';
import * as path from 'node:path';
import { readFileByLine } from './lib/fetch-text-by-line';
import { processLine } from './lib/process-line';
import { createRuleset } from './lib/create-file';
Expand Down
2 changes: 1 addition & 1 deletion Build/build-deprecate-files.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { compareAndWriteFile } from './lib/create-file';
import { task } from './trace';
import path from 'path';
import path from 'node:path';

const DEPRECATED_FILES = [
['non_ip/global_plus', 'This file has been merged with non_ip/global'],
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// @ts-check
import path from 'path';
import path from 'node:path';
import { DOMESTICS } from '../Source/non_ip/domestic';
import { DIRECTS, LANS } from '../Source/non_ip/direct';
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
Expand Down
2 changes: 1 addition & 1 deletion Build/build-internal-reverse-chn-cidr.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import path from 'path';
import path from 'node:path';
import { task } from './trace';

import { exclude, merge } from 'fast-cidr-tools';
Expand Down
2 changes: 1 addition & 1 deletion Build/build-mitm-hostname.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { readFileByLine } from './lib/fetch-text-by-line';
import pathFn from 'path';
import pathFn from 'node:path';
import table from 'table';
import { fdir as Fdir } from 'fdir';
import { green, yellow } from 'picocolors';
Expand Down
6 changes: 3 additions & 3 deletions Build/build-public.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import path from 'path';
import fs from 'fs';
import fsp from 'fs/promises';
import path from 'node:path';
import fs from 'node:fs';
import fsp from 'node:fs/promises';
import { task } from './trace';
import { treeDir } from './lib/tree-dir';
import type { TreeType, TreeTypeArray } from './lib/tree-dir';
Expand Down
3 changes: 2 additions & 1 deletion Build/build-reject-domainset.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// @ts-check
import path from 'path';
import path from 'node:path';
import process from 'node:process';

import { processHosts, processFilterRules, processDomainLists } from './lib/parse-filter';
import { createTrie } from './lib/trie';
Expand Down
2 changes: 1 addition & 1 deletion Build/build-reject-ip-list.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// @ts-check
import path from 'path';
import path from 'node:path';
import { createRuleset } from './lib/create-file';
import { fetchRemoteTextByLine, readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { task } from './trace';
Expand Down
2 changes: 1 addition & 1 deletion Build/build-sgmodule-always-realip.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import path from 'path';
import path from 'node:path';
import { task } from './trace';
import { compareAndWriteFile } from './lib/create-file';
import { DIRECTS, LANS } from '../Source/non_ip/direct';
Expand Down
4 changes: 3 additions & 1 deletion Build/build-sgmodule-redirect.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import path from 'path';
import path from 'node:path';
import { task } from './trace';
import { compareAndWriteFile } from './lib/create-file';
import { getHostname } from 'tldts';
Expand Down Expand Up @@ -68,6 +68,8 @@ const REDIRECT_MIRROR = [
['raw.kkgithub.com/', 'https://raw.githubusercontent.com/'],
// Polyfill
['polyfill.io/', 'https://cdnjs.cloudflare.com/polyfill/'],
['polyfill.top/', 'https://cdnjs.cloudflare.com/polyfill/'],
['polyfill-js.cn/', 'https://cdnjs.cloudflare.com/polyfill/'],
['cdn.polyfill.io/', 'https://cdnjs.cloudflare.com/polyfill/'],
['fastly-polyfill.io/', 'https://cdnjs.cloudflare.com/polyfill/'],
['fastly-polyfill.net/', 'https://cdnjs.cloudflare.com/polyfill/'],
Expand Down
2 changes: 1 addition & 1 deletion Build/build-speedtest-domainset.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { domainDeduper } from './lib/domain-deduper';
import path from 'path';
import path from 'node:path';
import { createRuleset } from './lib/create-file';
import { sortDomains } from './lib/stable-sort-domain';

Expand Down
2 changes: 1 addition & 1 deletion Build/build-sspanel-appprofile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { getDomesticAndDirectDomainsRulesetPromise } from './build-domestic-dire
import { surgeRulesetToClashClassicalTextRuleset, surgeDomainsetToClashRuleset } from './lib/clash';
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { task } from './trace';
import path from 'path';
import path from 'node:path';

import { ALL as AllStreamServices } from '../Source/stream';
import { getChnCidrPromise } from './build-chn-cidr';
Expand Down
8 changes: 4 additions & 4 deletions Build/download-mock-assets.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { task } from './trace';
import path from 'path';
import fs from 'fs';
import { Readable } from 'stream';
import { pipeline } from 'stream/promises';
import path from 'node:path';
import fs from 'node:fs';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { fetchWithRetry } from './lib/fetch-retry';

const ASSETS_LIST = {
Expand Down
18 changes: 8 additions & 10 deletions Build/download-previous-build.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
import { existsSync, createWriteStream } from 'fs';
import { mkdir } from 'fs/promises';
import path from 'path';
import { pipeline } from 'stream/promises';
import { existsSync, createWriteStream } from 'node:fs';
import { mkdir } from 'node:fs/promises';
import path from 'node:path';
import { pipeline } from 'node:stream/promises';
import { readFileByLine } from './lib/fetch-text-by-line';
import { isCI } from 'ci-info';
import { task } from './trace';
import { defaultRequestInit, fetchWithRetry } from './lib/fetch-retry';
import tarStream from 'tar-stream';
import zlib from 'zlib';
import { Readable } from 'stream';
import zlib from 'node:zlib';
import { Readable } from 'node:stream';

const IS_READING_BUILD_OUTPUT = 1 << 2;
const ALL_FILES_EXISTS = 1 << 3;
Expand All @@ -32,10 +32,8 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a

buildOutputList.push(line);

if (!isCI) {
if (!existsSync(path.join(__dirname, '..', line))) {
flag = flag & ~ALL_FILES_EXISTS;
}
if (!isCI && !existsSync(path.join(__dirname, '..', line))) {
flag = flag & ~ALL_FILES_EXISTS;
}
}
});
Expand Down
2 changes: 2 additions & 0 deletions Build/index.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import process from 'node:process';

console.log('Version:', process.version);

import { downloadPreviousBuild } from './download-previous-build';
Expand Down
2 changes: 2 additions & 0 deletions Build/lib/cache-apply.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import process from 'node:process';

export const createCache = (namespace?: string, printStats = false) => {
const cache = new Map();

Expand Down
13 changes: 6 additions & 7 deletions Build/lib/cache-filesystem.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,16 @@
import createDb from 'better-sqlite3';
import type { Database } from 'better-sqlite3';
import os from 'os';
import path from 'path';
import { mkdirSync } from 'fs';
import os from 'node:os';
import path from 'node:path';
import { mkdirSync } from 'node:fs';
import picocolors from 'picocolors';
import { fastStringArrayJoin } from './misc';
import { performance } from 'perf_hooks';
import fs from 'fs';
import { performance } from 'node:perf_hooks';
import fs from 'node:fs';
import { stringHash } from './string-hash';

const identity = (x: any) => x;

// eslint-disable-next-line sukka-ts/no-const-enum -- bun is smart, right?
const enum CacheStatus {
Hit = 'hit',
Stale = 'stale',
Expand Down Expand Up @@ -156,7 +155,7 @@ export class Cache<S = string> {
const now = Date.now();
const rv = this.db.prepare<string, { ttl: number }>(`SELECT ttl FROM ${this.tableName} WHERE key = ?`).get(key);

return !rv ? CacheStatus.Miss : (rv.ttl > now ? CacheStatus.Hit : CacheStatus.Stale);
return rv ? (rv.ttl > now ? CacheStatus.Hit : CacheStatus.Stale) : CacheStatus.Miss;
}

del(key: string): void {
Expand Down
14 changes: 8 additions & 6 deletions Build/lib/convert-clash-meta-mrs.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import path from 'path';
import fs from 'fs';
import fsp from 'fs/promises';
import { Readable } from 'stream';
import { pipeline } from 'stream/promises';
import zlib from 'zlib';
import path from 'node:path';
import fs from 'node:fs';
import fsp from 'node:fs/promises';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import zlib from 'node:zlib';
import process from 'node:process';

import { async as ezspawn } from '@jsdevtools/ez-spawn';

const mihomoBinaryDir = path.join(__dirname, '../../node_modules/.cache/mihomo');
Expand Down
4 changes: 2 additions & 2 deletions Build/lib/create-file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash';
import picocolors from 'picocolors';
import type { Span } from '../trace';
import path from 'path';
import fs from 'fs';
import path from 'node:path';
import fs from 'node:fs';
import { fastStringArrayJoin, writeFile } from './misc';
import { readFileByLine } from './fetch-text-by-line';
import stringify from 'json-stringify-pretty-compact';
Expand Down
7 changes: 6 additions & 1 deletion Build/lib/fetch-assets.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
import picocolors from 'picocolors';
import { defaultRequestInit, fetchWithRetry } from './fetch-retry';
import { setTimeout } from 'timers/promises';
import { setTimeout } from 'node:timers/promises';

class CustomAbortError extends Error {
constructor() {
super();
this.name = 'CustomAbortError';
}

public readonly name = 'AbortError';
public readonly digest = 'AbortError';
}
Expand Down
19 changes: 8 additions & 11 deletions Build/lib/fetch-retry.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import retry from 'async-retry';
import picocolors from 'picocolors';
import { setTimeout } from 'timers/promises';
import { setTimeout } from 'node:timers/promises';

function isClientError(err: unknown): err is NodeJS.ErrnoException {
if (!err || typeof err !== 'object') return false;
Expand All @@ -24,6 +24,7 @@ export class ResponseError extends Error {
Error.captureStackTrace(this, ResponseError);
}

// eslint-disable-next-line sukka/unicorn/custom-error-definition -- deliberatly use previous name
this.name = this.constructor.name;
this.res = res;
this.code = res.status;
Expand Down Expand Up @@ -90,16 +91,12 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
return res;
}
} catch (err: unknown) {
if (err instanceof Error) {
if (
err.name === 'AbortError'
|| ('digest' in err && err.digest === 'AbortError')
) {
if (!retryOpts.retryOnAborted) {
console.log(picocolors.gray('[fetch abort]'), url);
return bail(err) as never;
}
}
if (err instanceof Error && (
err.name === 'AbortError'
|| ('digest' in err && err.digest === 'AbortError')
) && !retryOpts.retryOnAborted) {
console.log(picocolors.gray('[fetch abort]'), url);
return bail(err) as never;
}
if (isClientError(err)) {
return bail(err) as never;
Expand Down
4 changes: 2 additions & 2 deletions Build/lib/fetch-text-by-line.bench.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { bench, group, run } from 'mitata';
import { processLine, processLineFromReadline } from './process-line';
import { readFileByLine } from './fetch-text-by-line';
import path from 'path';
import fsp from 'fs/promises';
import path from 'node:path';
import fsp from 'node:fs/promises';

const file = path.resolve(__dirname, '../../Source/domainset/cdn.conf');

Expand Down
10 changes: 5 additions & 5 deletions Build/lib/fetch-text-by-line.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import fs from 'fs';
import { Readable } from 'stream';
import fs from 'node:fs';
import { Readable } from 'node:stream';
import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
import type { FileHandle } from 'fs/promises';
import type { FileHandle } from 'node:fs/promises';

import { TextLineStream } from './text-line-transform-stream';
import type { ReadableStream } from 'stream/web';
import { TextDecoderStream } from 'stream/web';
import type { ReadableStream } from 'node:stream/web';
import { TextDecoderStream } from 'node:stream/web';
import { processLine } from './process-line';

const getReadableStream = (file: string | FileHandle): ReadableStream => {
Expand Down
6 changes: 3 additions & 3 deletions Build/lib/misc.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import path, { dirname } from 'path';
import fs from 'fs';
import fsp from 'fs/promises';
import path, { dirname } from 'node:path';
import fs from 'node:fs';
import fsp from 'node:fs/promises';

export const isTruthy = <T>(i: T | 0 | '' | false | null | undefined): i is T => !!i;

Expand Down
33 changes: 15 additions & 18 deletions Build/lib/parse-filter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,6 @@ export function processHosts(span: Span, hostsUrl: string, mirrors: string[] | n
));
}

// eslint-disable-next-line sukka-ts/no-const-enum -- bun bundler is smart, maybe?
const enum ParseType {
WhiteIncludeSubdomain = 0,
WhiteAbsolute = -1,
Expand Down Expand Up @@ -174,25 +173,23 @@ export async function processFilterRules(

const hostname = result[0];

if (DEBUG_DOMAIN_TO_FIND) {
if (hostname.includes(DEBUG_DOMAIN_TO_FIND)) {
console.warn(
picocolors.red(filterRulesUrl),
flag === ParseType.WhiteIncludeSubdomain || flag === ParseType.WhiteAbsolute
? '(white)'
: '(black)',
hostname.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND))
);
foundDebugDomain = true;
}
if (DEBUG_DOMAIN_TO_FIND && hostname.includes(DEBUG_DOMAIN_TO_FIND)) {
console.warn(
picocolors.red(filterRulesUrl),
flag === ParseType.WhiteIncludeSubdomain || flag === ParseType.WhiteAbsolute
? '(white)'
: '(black)',
hostname.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND))
);
foundDebugDomain = true;
}

switch (flag) {
case ParseType.WhiteIncludeSubdomain:
if (hostname[0] !== '.') {
whitelistDomainSets.add(`.${hostname}`);
} else {
if (hostname[0] === '.') {
whitelistDomainSets.add(hostname);
} else {
whitelistDomainSets.add(`.${hostname}`);
}
break;
case ParseType.WhiteAbsolute:
Expand All @@ -202,10 +199,10 @@ export async function processFilterRules(
blacklistDomainSets.add(hostname);
break;
case ParseType.BlackIncludeSubdomain:
if (hostname[0] !== '.') {
blacklistDomainSets.add(`.${hostname}`);
} else {
if (hostname[0] === '.') {
blacklistDomainSets.add(hostname);
} else {
blacklistDomainSets.add(`.${hostname}`);
}
break;
case ParseType.ErrorMessage:
Expand Down
2 changes: 1 addition & 1 deletion Build/lib/text-line-transform-stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// This module is browser compatible.
// Modified by Sukka (https://skk.moe) to increase compatibility and performance with Bun.

import { TransformStream } from 'stream/web';
import { TransformStream } from 'node:stream/web';

interface TextLineStreamOptions {
/** Allow splitting by solo \r */
Expand Down
4 changes: 2 additions & 2 deletions Build/lib/tree-dir.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import fsp from 'fs/promises';
import { sep } from 'path';
import fsp from 'node:fs/promises';
import { sep } from 'node:path';

interface TreeFileType {
type: 'file',
Expand Down
Loading

0 comments on commit 4326c8c

Please sign in to comment.