Merge branch 'SukkaW:master' into master

Build/build-anti-bogus-domain.ts

@@ -37,7 +37,7 @@ export const buildAntiBogusDomain = task(import.meta.path, async (span) => {
 
   const peeked = Bun.peek(getBogusNxDomainIPsPromise);
   const bogusNxDomainIPs = peeked === getBogusNxDomainIPsPromise
-    ? await span.traceChild('get bogus nxdomain ips').traceAsyncFn(() => getBogusNxDomainIPsPromise)
+    ? await span.traceChildPromise('get bogus nxdomain ips', getBogusNxDomainIPsPromise)
     : (peeked as string[]);
 
   result.push(...bogusNxDomainIPs);

Build/build-apple-cdn.ts

@@ -21,7 +21,7 @@ export const buildAppleCdn = task(import.meta.path, async (span) => {
   const promise = getAppleCdnDomainsPromise();
   const peeked = Bun.peek(promise);
   const res: string[] = peeked === promise
-    ? await span.traceChild('get apple cdn domains').traceAsyncFn(() => promise)
+    ? await span.traceChildPromise('get apple cdn domains', promise)
     : (peeked as string[]);
 
   const description = [

Build/build-chn-cidr.ts

@@ -32,7 +32,7 @@ export const buildChnCidr = task(import.meta.path, async (span) => {
   const cidrPromise = getChnCidrPromise();
   const peeked = Bun.peek(cidrPromise);
   const filteredCidr: string[] = peeked === cidrPromise
-    ? await span.traceChild('download chnroutes2').tracePromise(cidrPromise)
+    ? await span.traceChildPromise('download chnroutes2', cidrPromise)
     : (peeked as string[]);
 
   // Can not use SHARED_DESCRIPTION here as different license

Build/build-common.ts

@@ -55,7 +55,7 @@ if (import.meta.main) {
 
 const processFile = (span: Span, sourcePath: string) => {
   // console.log('Processing', sourcePath);
-  return span.traceChild(`process file: ${sourcePath}`).traceAsyncFn(async () => {
+  return span.traceChildAsync(`process file: ${sourcePath}`, async () => {
     const lines: string[] = [];
 
     let title = '';
@@ -93,34 +93,36 @@ const processFile = (span: Span, sourcePath: string) => {
 
 function transformDomainset(parentSpan: Span, sourcePath: string, relativePath: string) {
   return parentSpan
-    .traceChild(`transform domainset: ${path.basename(sourcePath, path.extname(sourcePath))}`)
-    .traceAsyncFn(async (span) => {
-      const res = await processFile(span, sourcePath);
-      if (!res) return;
-
-      const [title, descriptions, lines] = res;
-
-      const deduped = domainDeduper(lines);
-      const description = [
-        ...SHARED_DESCRIPTION,
-        ...(
-          descriptions.length
-            ? ['', ...descriptions]
-            : []
-        )
-      ];
-
-      return createRuleset(
-        span,
-        title,
-        description,
-        new Date(),
-        deduped,
-        'domainset',
-        path.resolve(outputSurgeDir, relativePath),
-        path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
-      );
-    });
+    .traceChildAsync(
+      `transform domainset: ${path.basename(sourcePath, path.extname(sourcePath))}`,
+      async (span) => {
+        const res = await processFile(span, sourcePath);
+        if (!res) return;
+
+        const [title, descriptions, lines] = res;
+
+        const deduped = domainDeduper(lines);
+        const description = [
+          ...SHARED_DESCRIPTION,
+          ...(
+            descriptions.length
+              ? ['', ...descriptions]
+              : []
+          )
+        ];
+
+        return createRuleset(
+          span,
+          title,
+          description,
+          new Date(),
+          deduped,
+          'domainset',
+          path.resolve(outputSurgeDir, relativePath),
+          path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
+        );
+      }
+    );
 }
 
 /**

Build/build-microsoft-cdn.ts

@@ -56,7 +56,7 @@ export const buildMicrosoftCdn = task(import.meta.path, async (span) => {
   const promise = getMicrosoftCdnRulesetPromise();
   const peeked = Bun.peek(promise);
   const res: string[] = peeked === promise
-    ? await span.traceChild('get microsoft cdn domains').tracePromise(promise)
+    ? await span.traceChildPromise('get microsoft cdn domains', promise)
     : (peeked as string[]);
 
   return createRuleset(

Build/build-reject-domainset.ts

@@ -68,7 +68,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
           SetHelpers.add(domainSets, fullPhishingDomainSet);
           setAddFromArray(domainSets, purePhishingDomains);
         }),
-        childSpan.traceChild('process reject_sukka.conf').traceAsyncFn(async () => {
+        childSpan.traceChildAsync('process reject_sukka.conf', async () => {
           setAddFromArray(domainSets, await readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf')));
         })
       ]);
@@ -84,13 +84,13 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
   console.log(`Import ${previousSize} rules from Hosts / AdBlock Filter Rules & reject_sukka.conf!`);
 
   // Dedupe domainSets
-  await span.traceChild('dedupe from black keywords/suffixes').traceAsyncFn(async (childSpan) => {
+  await span.traceChildAsync('dedupe from black keywords/suffixes', async (childSpan) => {
     /** Collect DOMAIN-SUFFIX from non_ip/reject.conf for deduplication */
     const domainSuffixSet = new Set<string>();
     /** Collect DOMAIN-KEYWORD from non_ip/reject.conf for deduplication */
     const domainKeywordsSet = new Set<string>();
 
-    await childSpan.traceChild('collect keywords/suffixes').traceAsyncFn(async () => {
+    await childSpan.traceChildAsync('collect keywords/suffixes', async () => {
       for await (const line of readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/reject.conf'))) {
         const [type, value] = line.split(',');
 
@@ -106,7 +106,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
     SetHelpers.subtract(domainSets, domainSuffixSet);
     SetHelpers.subtract(domainSets, filterRuleWhitelistDomainSets);
 
-    childSpan.traceChild('dedupe from white/suffixes').traceSyncFn(() => {
+    childSpan.traceChildSync('dedupe from white/suffixes', () => {
       const trie = createTrie(domainSets);
 
       domainSuffixSet.forEach(suffix => {
@@ -126,7 +126,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
       });
     });
 
-    childSpan.traceChild('dedupe from black keywords').traceSyncFn(() => {
+    childSpan.traceChildSync('dedupe from black keywords', () => {
       const kwfilter = createKeywordFilter(domainKeywordsSet);
 
       for (const domain of domainSets) {
@@ -142,7 +142,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
   previousSize = domainSets.size;
 
   // Dedupe domainSets
-  const dudupedDominArray = span.traceChild('dedupe from covered subdomain').traceSyncFn(() => domainDeduper(Array.from(domainSets)));
+  const dudupedDominArray = span.traceChildSync('dedupe from covered subdomain', () => domainDeduper(Array.from(domainSets)));
 
   console.log(`Deduped ${previousSize - dudupedDominArray.length} rules from covered subdomain!`);
   console.log(`Final size ${dudupedDominArray.length}`);
@@ -186,7 +186,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
       'Sukka\'s Ruleset - Reject Base',
       description,
       new Date(),
-      span.traceChild('sort reject domainset').traceSyncFn(() => sortDomains(dudupedDominArray, gorhill)),
+      span.traceChildSync('sort reject domainset', () => sortDomains(dudupedDominArray, gorhill)),
       'domainset',
       path.resolve(import.meta.dir, '../List/domainset/reject.conf'),
       path.resolve(import.meta.dir, '../Clash/domainset/reject.txt')

Build/build-speedtest-domainset.ts

@@ -175,7 +175,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
     '.backend.librespeed.org'
   ]);
 
-  await span.traceChild('fetch previous speedtest domainset').traceAsyncFn(async () => {
+  await span.traceChildAsync('fetch previous speedtest domainset', async () => {
     SetHelpers.add(domains, await getPreviousSpeedtestDomainsPromise());
   });
 
@@ -190,6 +190,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
       'Tokyo',
       'Singapore',
       'Korea',
+      'Seoul',
       'Canada',
       'Toronto',
       'Montreal',
@@ -210,7 +211,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
       'Brazil',
       'Turkey'
     ]).reduce<Record<string, Promise<void>>>((pMap, keyword) => {
-      pMap[keyword] = span.traceChild(`fetch speedtest endpoints: ${keyword}`).traceAsyncFn(() => querySpeedtestApi(keyword)).then(hostnameGroup => {
+      pMap[keyword] = span.traceChildAsync(`fetch speedtest endpoints: ${keyword}`, () => querySpeedtestApi(keyword)).then(hostnameGroup => {
         hostnameGroup.forEach(hostname => {
           if (hostname) {
             domains.add(hostname);
@@ -237,7 +238,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
   });
 
   const gorhill = await getGorhillPublicSuffixPromise();
-  const deduped = span.traceChild('sort result').traceSyncFn(() => sortDomains(domainDeduper(Array.from(domains)), gorhill));
+  const deduped = span.traceChildSync('sort result', () => sortDomains(domainDeduper(Array.from(domains)), gorhill));
 
   const description = [
     ...SHARED_DESCRIPTION,

Build/build-stream-service.ts

@@ -9,7 +9,7 @@ import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream';
 import { SHARED_DESCRIPTION } from './lib/constants';
 
 export const createRulesetForStreamService = (span: Span, fileId: string, title: string, streamServices: Array<import('../Source/stream').StreamService>) => {
-  return span.traceChild(fileId).traceAsyncFn(async (childSpan) => Promise.all([
+  return span.traceChildAsync(fileId, async (childSpan) => Promise.all([
     // Domains
     createRuleset(
       childSpan,

Build/build-telegram-cidr.ts

@@ -36,7 +36,7 @@ export const buildTelegramCIDR = task(import.meta.path, async (span) => {
   const promise = getTelegramCIDRPromise();
   const peeked = Bun.peek(promise);
   const { date, results } = peeked === promise
-    ? await span.traceChild('get telegram cidr').tracePromise(promise)
+    ? await span.traceChildPromise('get telegram cidr', promise)
     : (peeked as { date: Date, results: string[] });
 
   if (results.length === 0) {

Build/lib/cache-filesystem.ts

@@ -15,28 +15,27 @@ const enum CacheStatus {
   Miss = 'miss'
 }
 
-export interface CacheOptions {
+export interface CacheOptions<S = string> {
   /** Path to sqlite file dir */
   cachePath?: string,
   /** Time before deletion */
   tbd?: number,
   /** Cache table name */
-  tableName?: string
+  tableName?: string,
+  type?: S extends string ? 'string' : 'buffer'
 }
 
-interface CacheApplyNonStringOption<T> {
+interface CacheApplyRawOption {
   ttl?: number | null,
-  serializer: (value: T) => string,
-  deserializer: (cached: string) => T,
   temporaryBypass?: boolean
 }
 
-interface CacheApplyStringOption {
-  ttl?: number | null,
-  temporaryBypass?: boolean
+interface CacheApplyNonRawOption<T, S> extends CacheApplyRawOption {
+  serializer: (value: T) => S,
+  deserializer: (cached: S) => T
 }
 
-type CacheApplyOption<T> = T extends string ? CacheApplyStringOption : CacheApplyNonStringOption<T>;
+type CacheApplyOption<T, S> = T extends S ? CacheApplyRawOption : CacheApplyNonRawOption<T, S>;
 
 const randomInt = (min: number, max: number) => Math.floor(Math.random() * (max - min + 1)) + min;
 
@@ -62,20 +61,27 @@ export const TTL = {
   TWO_WEEKS: () => randomInt(10, 14) * ONE_DAY
 };
 
-export class Cache {
+export class Cache<S = string> {
   db: Database;
   /** Time before deletion */
   tbd = 60 * 1000;
   /** SQLite file path */
   cachePath: string;
   /** Table name */
   tableName: string;
+  type: S extends string ? 'string' : 'buffer';
 
-  constructor({ cachePath = path.join(os.tmpdir() || '/tmp', 'hdc'), tbd, tableName = 'cache' }: CacheOptions = {}) {
+  constructor({
+    cachePath = path.join(os.tmpdir() || '/tmp', 'hdc'), tbd, tableName = 'cache', type }: CacheOptions<S> = {}) {
     this.cachePath = cachePath;
     mkdirSync(this.cachePath, { recursive: true });
     if (tbd != null) this.tbd = tbd;
     this.tableName = tableName;
+    if (type) {
+      this.type = type;
+    } else {
+      this.type = 'string' as any;
+    }
 
     const db = new Database(path.join(this.cachePath, 'cache.db'));
 
@@ -84,7 +90,7 @@ export class Cache {
     db.exec('PRAGMA temp_store = memory;');
     db.exec('PRAGMA optimize;');
 
-    db.prepare(`CREATE TABLE IF NOT EXISTS ${this.tableName} (key TEXT PRIMARY KEY, value TEXT, ttl REAL NOT NULL);`).run();
+    db.prepare(`CREATE TABLE IF NOT EXISTS ${this.tableName} (key TEXT PRIMARY KEY, value ${this.type === 'string' ? 'TEXT' : 'BLOB'}, ttl REAL NOT NULL);`).run();
     db.prepare(`CREATE INDEX IF NOT EXISTS cache_ttl ON ${this.tableName} (ttl);`).run();
 
     const date = new Date();
@@ -119,9 +125,9 @@ export class Cache {
     });
   }
 
-  get(key: string, defaultValue?: string): string | undefined {
-    const rv = this.db.prepare<{ value: string }, string>(
-      `SELECT value FROM ${this.tableName} WHERE key = ?`
+  get(key: string, defaultValue?: S): S | undefined {
+    const rv = this.db.prepare<{ value: S }, string>(
+      `SELECT value FROM ${this.tableName} WHERE key = ? LIMIT 1`
     ).get(key);
 
     if (!rv) return defaultValue;
@@ -142,7 +148,7 @@ export class Cache {
   async apply<T>(
     key: string,
     fn: () => Promise<T>,
-    opt: CacheApplyOption<T>
+    opt: CacheApplyOption<T, S>
   ): Promise<T> {
     const { ttl, temporaryBypass } = opt;
 
@@ -194,6 +200,8 @@ export const fsFetchCache = traceSync('initializing filesystem cache for fetch',
 //   fsFetchCache.destroy();
 // });
 
+// export const fsCache = traceSync('initializing filesystem cache', () => new Cache<Uint8Array>({ cachePath: path.resolve(import.meta.dir, '../../.cache'), type: 'buffer' }));
+
 const separator = '\u0000';
 // const textEncoder = new TextEncoder();
 // const textDecoder = new TextDecoder();

Build/lib/create-file.ts

@@ -18,7 +18,7 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
     console.log(`Nothing to write to ${filePath}...`);
     isEqual = false;
   } else {
-    isEqual = await span.traceChild(`comparing ${filePath}`).traceAsyncFn(async () => {
+    isEqual = await span.traceChildAsync(`comparing ${filePath}`, async () => {
       let index = 0;
 
       for await (const lineB of readFileByLine(file)) {
@@ -63,7 +63,7 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
     return;
   }
 
-  await span.traceChild(`writing ${filePath}`).traceAsyncFn(async () => {
+  await span.traceChildAsync(`writing ${filePath}`, async () => {
     if (linesALen < 10000) {
       return Bun.write(file, `${linesA.join('\n')}\n`);
     }
@@ -98,7 +98,7 @@ export const createRuleset = (
   type: 'ruleset' | 'domainset', surgePath: string, clashPath: string
 ) => parentSpan.traceChild(`create ruleset: ${path.basename(surgePath, path.extname(surgePath))}`).traceAsyncFn((childSpan) => {
   const surgeContent = withBannerArray(title, description, date, content);
-  const clashContent = childSpan.traceChild('convert incoming ruleset to clash').traceSyncFn(() => {
+  const clashContent = childSpan.traceChildSync('convert incoming ruleset to clash', () => {
     let _clashContent;
     switch (type) {
       case 'domainset':

Build/lib/get-phishing-domains.ts

@@ -98,8 +98,8 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
     SetAdd(domainSet, domainSet2);
   }
 
-  span.traceChild('whitelisting phishing domains').traceSyncFn((parentSpan) => {
-    const trieForRemovingWhiteListed = parentSpan.traceChild('create trie for whitelisting').traceSyncFn(() => createTrie(domainSet));
+  span.traceChildSync('whitelisting phishing domains', (parentSpan) => {
+    const trieForRemovingWhiteListed = parentSpan.traceChildSync('create trie for whitelisting', () => createTrie(domainSet));
 
     return parentSpan.traceChild('delete whitelisted from domainset').traceSyncFn(() => {
       for (let i = 0, len = WHITELIST_DOMAIN.length; i < len; i++) {
@@ -115,7 +115,7 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
   const domainCountMap: Record<string, number> = {};
   const getDomain = createCachedGorhillGetDomain(gorhill);
 
-  span.traceChild('process phishing domain set').traceSyncFn(() => {
+  span.traceChildSync('process phishing domain set', () => {
     const domainArr = Array.from(domainSet);
 
     for (let i = 0, len = domainArr.length; i < len; i++) {
@@ -177,14 +177,14 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
     }
   });
 
-  const results = span.traceChild('get final phishing results').traceSyncFn(() => {
-    const results: string[] = [];
+  const results = span.traceChildSync('get final phishing results', () => {
+    const res: string[] = [];
     for (const domain in domainCountMap) {
       if (domainCountMap[domain] >= 5) {
-        results.push(`.${domain}`);
+        res.push(`.${domain}`);
       }
     }
-    return results;
+    return res;
   });
 
   return [results, domainSet] as const;