Merge branch 'SukkaW:master' into master

.github/workflows/main.yml

@@ -4,10 +4,15 @@ on:
     branches:
       - master
   schedule:
-    - cron: '0 12 * * *'
+    - cron: "0 12 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
+    name: Build
     runs-on: ubuntu-latest
 
     steps:
@@ -18,12 +23,12 @@ jobs:
       - uses: pnpm/action-setup@v4
         name: Install pnpm
         with:
-            run_install: false
+          run_install: false
       - name: Use Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: '.node-version'
-          cache: 'pnpm'
+          node-version-file: ".node-version"
+          cache: "pnpm"
       - name: Get current date
         id: date
         run: |
@@ -64,23 +69,49 @@ jobs:
             echo "public directory is empty"
             exit 1
           fi
+      - uses: actions/upload-artifact@v4
+        with:
+          name: build-artifact-${{ github. ref_name }}
+          path: public
+          if-no-files-found: error
+          retention-days: 1
+          compression-level: 4
+          include-hidden-files: false
+  deploy_to_cloudflare_pages:
+    needs:
+      - build
+    name: Deploy to Cloudflare Pages
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: build-artifact-${{ github. ref_name }}
+          path: public
       - name: Deploy to Cloudflare Pages
         uses: cloudflare/wrangler-action@v3
         with:
           apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
           command: pages deploy public --project-name=sukkaw-ruleset --commit-dirty=true --branch=main
+  deploy_to_github_gitlab:
+    needs:
+      - build
+    name: Deploy to GitHub and GitLab
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: build-artifact-${{ github.ref_name }}
+          path: public
       - name: Upload Dist to GitLab
         run: |
-          mkdir ./deploy-git
+          git clone --filter=blob:none https://${GITLAB_TOKEN_NAME}:${GITLAB_TOKEN}@gitlab.com/SukkaW/ruleset.skk.moe.git ./deploy-git
           cd ./deploy-git
-          git config --global init.defaultBranch master
-          git init
           git config --global push.default matching
           git config --global user.email "${GITLAB_EMAIL}"
           git config --global user.name "${GITLAB_USER}"
-          git remote add origin https://${GITLAB_TOKEN_NAME}:${GITLAB_TOKEN}@gitlab.com/SukkaW/ruleset.skk.moe.git
-          git pull origin master
           rm -rf ./*
           cp -rf ../public/* ./
           git add --all .

Build/build-apple-cdn.ts

@@ -32,27 +32,15 @@ export const buildAppleCdn = task(require.main === module, __filename)(async (sp
     ' - https://github.com/felixonmars/dnsmasq-china-list'
   ];
 
-  const ruleset = res.map(domain => `DOMAIN-SUFFIX,${domain}`);
   const domainset = res.map(i => `.${i}`);
 
-  return Promise.all([
-    createRuleset(
-      span,
-      'Sukka\'s Ruleset - Apple CDN',
-      description,
-      new Date(),
-      ruleset,
-      'ruleset',
-      output('apple_cdn', 'non_ip')
-    ),
-    createRuleset(
-      span,
-      'Sukka\'s Ruleset - Apple CDN',
-      description,
-      new Date(),
-      domainset,
-      'domainset',
-      output('apple_cdn', 'domainset')
-    )
-  ]);
+  return createRuleset(
+    span,
+    'Sukka\'s Ruleset - Apple CDN',
+    description,
+    new Date(),
+    domainset,
+    'domainset',
+    output('apple_cdn', 'domainset')
+  );
 });

Build/build-cdn-download-conf.ts

@@ -5,7 +5,7 @@ import { createTrie } from './lib/trie';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { getPublicSuffixListTextPromise } from './lib/download-publicsuffixlist';
-import { domainDeduper } from './lib/domain-deduper';
+import { domainsetDeduper } from './lib/domain-deduper';
 import { appendArrayInPlace } from './lib/append-array-in-place';
 import { sortDomains } from './lib/stable-sort-domain';
 import { output } from './lib/misc';
@@ -76,7 +76,7 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as
         'This file contains object storage and static assets CDN domains.'
       ],
       new Date(),
-      sortDomains(domainDeduper(cdnDomainsList)),
+      sortDomains(domainsetDeduper(cdnDomainsList)),
       'domainset',
       output('cdn', 'domainset')
     ),
@@ -89,7 +89,7 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as
         'This file contains domains for software updating & large file hosting.'
       ],
       new Date(),
-      sortDomains(domainDeduper(downloadDomainSet)),
+      sortDomains(domainsetDeduper(downloadDomainSet)),
       'domainset',
       output('download', 'domainset')
     )

Build/build-common.ts

@@ -4,17 +4,15 @@ import * as path from 'node:path';
 import { readFileByLine } from './lib/fetch-text-by-line';
 import { processLine } from './lib/process-line';
 import { createRuleset } from './lib/create-file';
-import { domainDeduper } from './lib/domain-deduper';
+import { domainsetDeduper } from './lib/domain-deduper';
 import type { Span } from './trace';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { fdir as Fdir } from 'fdir';
 import { appendArrayInPlace } from './lib/append-array-in-place';
-import { removeFiles } from './lib/misc';
 import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR, SOURCE_DIR } from './constants/dir';
 
 const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
-const MAGIC_COMMAND_RM = '# $ custom_no_output';
 const MAGIC_COMMAND_TITLE = '# $ meta_title ';
 const MAGIC_COMMAND_DESCRIPTION = '# $ meta_description ';
 
@@ -68,7 +66,6 @@ export const buildCommon = task(require.main === module, __filename)(async (span
 });
 
 const $skip = Symbol('skip');
-const $rm = Symbol('rm');
 
 const processFile = (span: Span, sourcePath: string) => {
   // console.log('Processing', sourcePath);
@@ -80,9 +77,6 @@ const processFile = (span: Span, sourcePath: string) => {
 
     try {
       for await (const line of readFileByLine(sourcePath)) {
-        if (line.startsWith(MAGIC_COMMAND_RM)) {
-          return $rm;
-        }
         if (line.startsWith(MAGIC_COMMAND_SKIP)) {
           return $skip;
         }
@@ -121,16 +115,8 @@ function transformDomainset(parentSpan: Span, sourcePath: string, relativePath:
 
         const clashFileBasename = relativePath.slice(0, -path.extname(relativePath).length);
 
-        if (res === $rm) {
-          return removeFiles([
-            path.resolve(OUTPUT_SURGE_DIR, relativePath),
-            path.resolve(OUTPUT_CLASH_DIR, `${clashFileBasename}.txt`),
-            path.resolve(OUTPUT_SINGBOX_DIR, `${clashFileBasename}.json`)
-          ]);
-        }
-
         const [title, descriptions, lines] = res;
-        const deduped = domainDeduper(lines);
+        const deduped = domainsetDeduper(lines);
 
         let description: string[];
         if (descriptions.length) {
@@ -170,14 +156,6 @@ async function transformRuleset(parentSpan: Span, sourcePath: string, relativePa
 
       const clashFileBasename = relativePath.slice(0, -path.extname(relativePath).length);
 
-      if (res === $rm) {
-        return removeFiles([
-          path.resolve(OUTPUT_SURGE_DIR, relativePath),
-          path.resolve(OUTPUT_CLASH_DIR, `${clashFileBasename}.txt`),
-          path.resolve(OUTPUT_SINGBOX_DIR, `${clashFileBasename}.json`)
-        ]);
-      }
-
       const [title, descriptions, lines] = res;
 
       let description: string[];

Build/build-deprecate-files.ts

@@ -1,16 +1,29 @@
-import { OUTPUT_CLASH_DIR, OUTPUT_SURGE_DIR } from './constants/dir';
+import { OUTPUT_CLASH_DIR, OUTPUT_SURGE_DIR, PUBLIC_DIR } from './constants/dir';
 import { compareAndWriteFile } from './lib/create-file';
 import { task } from './trace';
 import path from 'node:path';
+import fsp from 'node:fs/promises';
 
 const DEPRECATED_FILES = [
   ['non_ip/global_plus', 'This file has been merged with non_ip/global'],
   ['domainset/reject_sukka', 'This file has been merged with domainset/reject'],
   ['domainset/reject_phishing', 'This file has been merged with domainset/reject']
 ];
 
+const REMOVED_FILES = [
+  'Internal/cdn.txt',
+  'List/domainset/steam.conf',
+  'List/internal/appprofile.php',
+  'Clash/domainset/steam.txt',
+  'sing-box/domainset/steam.json'
+];
+
 export const buildDeprecateFiles = task(require.main === module, __filename)((span) => span.traceChildAsync('create deprecated files', async (childSpan) => {
-  const promises: Array<Promise<unknown>> = [];
+  const promises: Array<Promise<unknown>> = REMOVED_FILES
+    .map(f => fsp.rm(
+      path.join(PUBLIC_DIR, f),
+      { force: true, recursive: true }
+    ));
 
   for (const [filePath, description] of DEPRECATED_FILES) {
     const surgeFile = path.resolve(OUTPUT_SURGE_DIR, `${filePath}.conf`);

Build/build-reject-domainset.ts

@@ -7,7 +7,7 @@ import { createTrie } from './lib/trie';
 
 import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA } from './constants/reject-data-source';
 import { createRuleset, compareAndWriteFile } from './lib/create-file';
-import { domainDeduper } from './lib/domain-deduper';
+import { domainsetDeduper } from './lib/domain-deduper';
 import createKeywordFilter from './lib/aho-corasick';
 import { readFileByLine, readFileIntoProcessedArray } from './lib/fetch-text-by-line';
 import { buildParseDomainMap, sortDomains } from './lib/stable-sort-domain';
@@ -149,8 +149,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
   });
 
   // Dedupe domainSets
-  const dudupedDominArray = span.traceChildSync('dedupe from covered subdomain (base)', () => domainDeduper(baseTrie));
-  const dudupedDominArrayExtra = span.traceChildSync('dedupe from covered subdomain (extra)', () => domainDeduper(extraTrie));
+  const dudupedDominArray = span.traceChildSync('dedupe from covered subdomain (base)', () => domainsetDeduper(baseTrie));
+  const dudupedDominArrayExtra = span.traceChildSync('dedupe from covered subdomain (extra)', () => domainsetDeduper(extraTrie));
 
   console.log(`Final size ${dudupedDominArray.length} + ${dudupedDominArrayExtra.length}`);
 

Build/build-speedtest-domainset.ts

@@ -1,4 +1,4 @@
-import { domainDeduper } from './lib/domain-deduper';
+import { domainsetDeduper } from './lib/domain-deduper';
 import path from 'node:path';
 import { createRuleset } from './lib/create-file';
 import { sortDomains } from './lib/stable-sort-domain';
@@ -235,7 +235,7 @@ export const buildSpeedtestDomainSet = task(require.main === module, __filename)
     }
   }))));
 
-  const deduped = span.traceChildSync('sort result', () => sortDomains(domainDeduper(domainTrie)));
+  const deduped = span.traceChildSync('sort result', () => sortDomains(domainsetDeduper(domainTrie)));
 
   const description = [
     ...SHARED_DESCRIPTION,

Build/constants/loose-tldts-opt.ts

@@ -12,3 +12,8 @@ export const loosTldOptWithPrivateDomains: Parameters<typeof tldts.getSubdomain>
   ...looseTldtsOpt,
   allowPrivateDomains: true
 };
+
+export const normalizeTldtsOpt: Parameters<typeof tldts.getSubdomain>[1] = {
+  allowPrivateDomains: true
+  // detectIp: true
+};

Build/download-previous-build.ts

@@ -1,10 +1,8 @@
-import { createWriteStream } from 'node:fs';
-import { mkdir } from 'node:fs/promises';
 import path from 'node:path';
 import { pipeline } from 'node:stream/promises';
 import { task } from './trace';
 import { defaultRequestInit, fetchWithRetry } from './lib/fetch-retry';
-import tarStream from 'tar-stream';
+import { extract as tarExtract, type Headers as TarEntryHeaders } from 'tar-fs';
 import zlib from 'node:zlib';
 import { Readable } from 'node:stream';
 
@@ -58,28 +56,39 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a
       throw new Error('Download previous build failed! No body found');
     }
 
+    const pathPrefix = 'ruleset.skk.moe-master/';
+
     const gunzip = zlib.createGunzip();
-    const extract = tarStream.extract();
+    const extract = tarExtract(
+      publicDir,
+      {
+        ignore: tarOnIgnore,
+        map(header) {
+          header.name = header.name.replace(pathPrefix, '');
+          return header;
+        }
+      }
+    );
 
-    pipeline(
+    return pipeline(
       Readable.fromWeb(resp.body),
       gunzip,
       extract
     );
+  });
+});
 
-    const pathPrefix = 'ruleset.skk.moe-master/';
-
-    for await (const entry of extract) {
-      if (entry.header.type !== 'file') {
-        entry.resume(); // Drain the entry
-        continue;
-      }
-
-      const relativeEntryPath = entry.header.name.replace(pathPrefix, '');
-      const targetPath = path.join(publicDir, relativeEntryPath);
+function tarOnIgnore(_: string, header?: TarEntryHeaders) {
+  if (header) {
+    if (header.type !== 'file' && header.type !== 'directory') {
+      return true;
+    }
 
-      await mkdir(path.dirname(targetPath), { recursive: true });
-      await pipeline(entry, createWriteStream(targetPath));
+    const extname = path.extname(header.name);
+    if (extname === '.ts') {
+      return true;
     }
-  });
-});
+  }
+
+  return false;
+}