Merge branch 'SukkaW:master' into master

.github/workflows/main.yml

@@ -68,6 +68,10 @@ jobs:
             echo "public directory is empty"
             exit 1
           fi
+          if [ ! -f .BUILD_FINISHED ]; then
+            echo ".BUILD_FINISHED not found"
+            exit 1
+          fi
       - uses: actions/upload-artifact@v4
         with:
           name: build-artifact-${{ github. ref_name }}
@@ -124,6 +128,7 @@ jobs:
           name: build-artifact-${{ github.ref_name }}
           path: public
       - name: Upload Dist to GitLab
+        continue-on-error: true
         run: |
           git clone --filter=tree:0 https://${GITLAB_TOKEN_NAME}:${GITLAB_TOKEN}@gitlab.com/SukkaW/ruleset.skk.moe.git ./deploy-git
           cd ./deploy-git

.gitignore

@@ -5,3 +5,4 @@ node_modules
 .cache
 public
 tmp.*
+.BUILD_FINISHED

Build/build-reject-domainset.ts

@@ -6,7 +6,7 @@ import { processHosts, processFilterRules, processDomainLists } from './lib/pars
 
 import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_WHITELIST } from './constants/reject-data-source';
 import { compareAndWriteFile } from './lib/create-file';
-import { readFileByLine, readFileIntoProcessedArray } from './lib/fetch-text-by-line';
+import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
 import { task } from './trace';
 // tldts-experimental is way faster than tldts, but very little bit inaccurate
 // (since it is hashes based). But the result is still deterministic, which is
@@ -21,7 +21,8 @@ import { DomainsetOutput } from './lib/create-file';
 
 const readLocalRejectDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka.conf'));
 const readLocalRejectExtraDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka_extra.conf'));
-const readLocalRejectRulesetPromise = readFileByLine(path.join(SOURCE_DIR, 'non_ip/reject.conf'));
+const readLocalRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/reject.conf'));
+const readLocalMyRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/my_reject.conf'));
 
 export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => {
   const rejectBaseDescription = [
@@ -171,7 +172,15 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
           '! Description: The domainset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining',
           '!'
         ],
-        rejectOutput.adguardhome()
+        appendArrayInPlace(
+          rejectOutput.adguardhome(),
+          (
+            await new DomainsetOutput(span, 'my_reject')
+              .addFromRuleset(readLocalMyRejectRulesetPromise)
+              .addFromRuleset(readLocalRejectRulesetPromise)
+              .done()
+          ).adguardhome()
+        )
       ),
       path.join(OUTPUT_INTERNAL_DIR, 'reject-adguardhome.txt')
     )

Build/index.ts

@@ -1,5 +1,6 @@
 import process from 'node:process';
 import os from 'node:os';
+import fs from 'node:fs';
 
 import { downloadPreviousBuild } from './download-previous-build';
 import { buildCommon } from './build-common';
@@ -28,6 +29,7 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules';
 import { createSpan, printTraceResult, whyIsNodeRunning } from './trace';
 import { buildDeprecateFiles } from './build-deprecate-files';
 import { cacheGc } from './lib/make-fetch-happen';
+import path from 'node:path';
 
 process.on('uncaughtException', (error) => {
   console.error('Uncaught exception:', error);
@@ -38,6 +40,8 @@ process.on('unhandledRejection', (reason) => {
   process.exit(1);
 });
 
+const buildFinishedLock = path.join(__dirname, '../.BUILD_FINISHED');
+
 (async () => {
   console.log('Version:', process.version);
 
@@ -60,6 +64,10 @@ process.on('unhandledRejection', (reason) => {
 
   const rootSpan = createSpan('root');
 
+  if (fs.existsSync(buildFinishedLock)) {
+    fs.unlinkSync(buildFinishedLock);
+  }
+
   try {
     const downloadPreviousBuildPromise = downloadPreviousBuild(rootSpan);
     const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon(rootSpan));
@@ -97,6 +105,9 @@ process.on('unhandledRejection', (reason) => {
 
     printTraceResult(rootSpan.traceResult);
 
+    // write a file to demonstrate that the build is finished
+    fs.writeFileSync(buildFinishedLock, 'BUILD_FINISHED\n');
+
     // Finish the build to avoid leaking timer/fetch ref
     await whyIsNodeRunning();
     process.exit(0);

Build/lib/rules/base.ts

@@ -36,7 +36,7 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
   protected otherRules: string[] = [];
   protected abstract type: 'domainset' | 'non_ip' | 'ip';
 
-  private pendingPromise: Promise<void> | null = null;
+  private pendingPromise: Promise<any> | null = null;
 
   static readonly jsonToLines = (json: unknown): string[] => stringify(json).split('\n');
 
@@ -201,8 +201,13 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
     }
   }
 
-  addFromRuleset(source: AsyncIterable<string> | Iterable<string>) {
-    this.pendingPromise = (this.pendingPromise ||= Promise.resolve()).then(() => this.addFromRulesetPromise(source));
+  addFromRuleset(source: AsyncIterable<string> | Iterable<string> | Promise<Iterable<string>>) {
+    if (this.pendingPromise) {
+      this.pendingPromise = this.pendingPromise.then(() => source);
+    } else {
+      this.pendingPromise = Promise.resolve(source);
+    }
+    this.pendingPromise = this.pendingPromise.then((source) => this.addFromRulesetPromise(source));
     return this;
   }
 
@@ -247,6 +252,7 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
   async done() {
     await this.pendingPromise;
     this.pendingPromise = null;
+    return this;
   }
 
   private guardPendingPromise() {

Source/domainset/cdn.conf

@@ -158,6 +158,7 @@ vscode-sync-insiders.trafficmanager.net
 # https://learn.microsoft.com/en-us/visualstudio/install/install-and-use-visual-studio-behind-a-firewall-or-proxy-server?view=vs-2022
 az700632.vo.msecnd.net
 open-vsx.org
+.tunnels.api.visualstudio.com
 # NPM
 registry.npmjs.org
 npm.im
@@ -932,6 +933,8 @@ sentry.ainopara.com
 telemetry.individual.githubcopilot.com
 telemetry.business.githubcopilot.com
 .scorecardresearch.com
+analytics.twitter.com
+tracking.inflection.io
 # these domains pop up few months ago, dont block since we dunnno what they are
 # but they can definitely go through secondary policy
 .ogads-pa.googleapis.com
@@ -963,6 +966,7 @@ js.chilipiper.com
 .statuspage.io
 cdn.statuspage.io
 api.producthunt.com
+s3.producthunt.com
 app-api.vidjet.io
 freeshippingbar.herokuapp.com
 sdk.qikify.com
@@ -1153,6 +1157,7 @@ jssdk.opin.media
 us-js.zonka.co
 code.upscope.io
 js.upscope.io
+sdk.customfit.ai
 
 js.chargebee.com
 js1.chargebee.com
@@ -1174,6 +1179,7 @@ cdn-yotpo-images-production.yotpo.com
 img.announcekit.app
 cdn.announcekit.app
 
+cdn.jotfor.ms
 cdn01.jotfor.ms
 cdn02.jotfor.ms
 cdn03.jotfor.ms
@@ -1209,6 +1215,7 @@ f.convertkit.com
 
 static.scroll.com
 assets.scroll.com
+static-staging.scroll.com
 
 participants.evolv.ai
 media.evolv.ai
@@ -3847,3 +3854,14 @@ assets.dify.ai
 bookface-static.ycombinator.com
 .sthlstatic.com
 image.hkhl.hk
+cdn.joinhoney.com
+media.kit.com
+webstatic.chargebee.com
+images.typeform.com
+media-cms.onesignal.com
+import-cdn.default.com
+assets.embeddables.com
+netlify.apollo.io
+cdn.bullwhip.cloud
+cdn.coral.coralproject.net
+js.puter.com

Source/non_ip/global.conf

@@ -399,8 +399,7 @@ DOMAIN-SUFFIX,edgecompute.app
 DOMAIN-SUFFIX,fly.dev
 DOMAIN-SUFFIX,shw.io
 DOMAIN-SUFFIX,edgeapp.net
-
-# >> ccTLD
+# ccTLD
 DOMAIN-SUFFIX,ai
 DOMAIN-SUFFIX,at
 DOMAIN-SUFFIX,au
@@ -443,8 +442,7 @@ DOMAIN-SUFFIX,tw
 DOMAIN-SUFFIX,uk
 DOMAIN-SUFFIX,us
 DOMAIN-SUFFIX,ws
-
-# >> gTLD
+# gTLD
 DOMAIN-SUFFIX,app
 DOMAIN-SUFFIX,blog
 DOMAIN-SUFFIX,cafe
@@ -495,6 +493,7 @@ DOMAIN-SUFFIX,akamaihd.net
 DOMAIN-SUFFIX,amazon.com
 DOMAIN-SUFFIX,amazon.co.jp
 DOMAIN-SUFFIX,amazonaws.com
+DOMAIN-SUFFIX,androidpolice.com
 DOMAIN-SUFFIX,ant.design
 DOMAIN-SUFFIX,apernet.io
 DOMAIN-SUFFIX,apibay.org

Source/non_ip/my_reject.conf

@@ -38,18 +38,6 @@ DOMAIN-SUFFIX,uhabo.com
 DOMAIN-SUFFIX,xycdn.com
 
 # >> Misc
-DOMAIN-SUFFIX,parallels.com
-DOMAIN-SUFFIX,www.parallelskorea.com
-DOMAIN-SUFFIX,parallels.cn
-DOMAIN-SUFFIX,parallels.de
-DOMAIN-SUFFIX,parallels.es
-DOMAIN-SUFFIX,parallels.fr
-DOMAIN-SUFFIX,parallels.nl
-DOMAIN-SUFFIX,parallels.pt
-DOMAIN-SUFFIX,parallels.ru
-DOMAIN-SUFFIX,parallelskorea.com
-DOMAIN-SUFFIX,myparallels.com
-DOMAIN-SUFFIX,my.parallels.com
 
 DOMAIN-KEYWORD,bahoom
 DOMAIN,daisydiskapp.com

Source/non_ip/my_us.conf

@@ -7,3 +7,4 @@ DOMAIN-SUFFIX,mandaom.com
 DOMAIN-SUFFIX,crunchyroll.com
 DOMAIN,www.coinbase.com
 DOMAIN,javdb.com
+DOMAIN,cableav.tv

Source/non_ip/reject.conf

@@ -102,6 +102,8 @@ DOMAIN-SUFFIX,scorecardresearch.com
 DOMAIN-SUFFIX,omtrdc.net
 # Force Block histats.com
 DOMAIN-SUFFIX,histats.com
+# Force Block minexmr.com
+DOMAIN-SUFFIX,minexmr.com
 
 DOMAIN-KEYWORD,.net.daraz.
 DOMAIN-KEYWORD,.zooplus.

Source/stream.ts

@@ -607,7 +607,7 @@ const PORNHUB: StreamService = {
 const SOUNDCLOUD: StreamService = {
   name: 'SoundCloud',
   rules: [
-    'DOMAIN-SUFFIX,sndcdn.com',
+    // 'DOMAIN-SUFFIX,sndcdn.com',
     'DOMAIN-SUFFIX,soundcloud.com',
 
     'USER-AGENT,SoundCloud*'

package.json

@@ -23,7 +23,7 @@
     "@ghostery/adblocker": "^2.3.1",
     "@henrygd/queue": "^1.0.7",
     "async-retry": "^1.3.3",
-    "better-sqlite3": "^11.7.0",
+    "better-sqlite3": "^11.7.2",
     "cacache": "^19.0.1",
     "ci-info": "^4.1.0",
     "csv-parse": "^5.6.0",
@@ -38,8 +38,8 @@
     "picocolors": "^1.1.1",
     "tar-fs": "^3.0.6",
     "tinyexec": "^0.3.2",
-    "tldts": "^6.1.70",
-    "tldts-experimental": "^6.1.70",
+    "tldts": "^6.1.71",
+    "tldts-experimental": "^6.1.71",
     "undici": "^7.2.0",
     "undici-cache-store-better-sqlite3": "^0.1.1",
     "whoiser": "^1.18.0",
@@ -58,19 +58,19 @@
     "@types/fast-fifo": "^1.3.0",
     "@types/make-fetch-happen": "^10.0.4",
     "@types/mocha": "^10.0.10",
-    "@types/node": "^22.10.2",
+    "@types/node": "^22.10.5",
     "@types/node-fetch": "^2.6.12",
     "@types/tar-fs": "^2.0.4",
     "@types/tar-stream": "^3.1.3",
     "eslint": "^9.17.0",
     "eslint-config-sukka": "^6.13.0",
     "eslint-formatter-sukka": "^6.13.0",
     "expect": "^29.7.0",
-    "mitata": "^1.0.23",
+    "mitata": "^1.0.26",
     "mocha": "^11.0.1",
     "typescript": "^5.7.2"
   },
-  "packageManager": "[email protected].2",
+  "packageManager": "[email protected].3",
   "resolutions": {
     "has": "npm:@nolyfill/has@latest"
   }