From 96f223cf082cd7efc643dc10d52694534bd476ee Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 09:51:58 +0800 Subject: [PATCH 01/35] Update Global Hosts --- Source/non_ip/global.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/non_ip/global.conf b/Source/non_ip/global.conf index bafe0c434..7c1a983c4 100644 --- a/Source/non_ip/global.conf +++ b/Source/non_ip/global.conf @@ -733,6 +733,7 @@ DOMAIN-SUFFIX,mapbox.com DOMAIN-SUFFIX,mastodon.social DOMAIN-SUFFIX,macpaw.com DOMAIN-SUFFIX,matrix.org +DOMAIN-SUFFIX,matters.town DOMAIN-SUFFIX,mayoclinic.org DOMAIN-SUFFIX,meetup.com DOMAIN-SUFFIX,medallia.com From 8d5da9776acdca0dc8151aa2e15d29abe6a56712 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 10:02:38 +0800 Subject: [PATCH 02/35] Remove dead/outdated domains --- Source/domainset/reject_sukka_extra.conf | 6 ------ Source/non_ip/direct.conf | 1 - 2 files changed, 7 deletions(-) diff --git a/Source/domainset/reject_sukka_extra.conf b/Source/domainset/reject_sukka_extra.conf index 330462dac..30e4bfcdf 100644 --- a/Source/domainset/reject_sukka_extra.conf +++ b/Source/domainset/reject_sukka_extra.conf @@ -1004,7 +1004,6 @@ .elhereum.com .oggnalive.com .hashitmining.com -.equihashmining.com .tucanpool.com .erstweal.com .ethashpool.com @@ -1509,7 +1508,6 @@ .smartcoinpool.net .theminershaven.net .aevocoin.net -.cryptopool.net .cccpool.net .airfreedom.net .miningpeon.net @@ -2510,8 +2508,6 @@ .gabberpool.nl .123unlock.nl .stx.nl -.adcash.ga -.wordc.ga .adzjzewsma.cf .as.cf .bitcoin-server.cf @@ -2944,7 +2940,6 @@ .eos.blue .ape.capital .bitetimes.fishing -.diabase.zone .raptoreum.zone .bitmining.zone .candid.zone @@ -2969,7 +2964,6 @@ .cloudcdn.gdn .binance.click .gemini.click -.korbit.click .bloc.money .bitnet.money .iwanttoearn.money diff --git a/Source/non_ip/direct.conf b/Source/non_ip/direct.conf index 25b85c9b5..05fe9f02b 100644 --- a/Source/non_ip/direct.conf +++ b/Source/non_ip/direct.conf @@ -111,7 +111,6 @@ DOMAIN-SUFFIX,hdhome.org DOMAIN-SUFFIX,hdpost.top DOMAIN-SUFFIX,hdroute.org DOMAIN-SUFFIX,hdsky.me -DOMAIN-SUFFIX,hdstreet.club DOMAIN-SUFFIX,hdtime.org DOMAIN-SUFFIX,hdupt.com DOMAIN-SUFFIX,hdzone.me From 6858cd63b46aa66e3f40fe2e4092167c98145ab5 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 14:43:35 +0800 Subject: [PATCH 03/35] Feat: `sukka_local_dns_mapping.sgmodule` now uses new rule-set syntax --- ...c-direct-lan-ruleset-dns-mapping-module.ts | 76 ++++++++++++++----- Build/constants/dir.ts | 1 + Build/lib/rules/base.ts | 46 +++++++---- Build/lib/rules/ruleset.ts | 2 +- Source/non_ip/direct.ts | 6 ++ Source/non_ip/domestic.ts | 40 +++++----- 6 files changed, 119 insertions(+), 52 deletions(-) diff --git a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts index e501aded1..cc16a0712 100644 --- a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts +++ b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts @@ -10,7 +10,7 @@ import { SHARED_DESCRIPTION } from './constants/description'; import { createMemoizedPromise } from './lib/memo-promise'; import * as yaml from 'yaml'; import { appendArrayInPlace } from './lib/append-array-in-place'; -import { OUTPUT_INTERNAL_DIR, OUTPUT_MODULES_DIR, SOURCE_DIR } from './constants/dir'; +import { OUTPUT_INTERNAL_DIR, OUTPUT_MODULES_DIR, OUTPUT_MODULES_RULES_DIR, SOURCE_DIR } from './constants/dir'; import { RulesetOutput } from './lib/create-file'; export function createGetDnsMappingRule(allowWildcard: boolean) { @@ -78,7 +78,7 @@ export const getDomesticAndDirectDomainsRulesetPromise = createMemoizedPromise(a export const buildDomesticRuleset = task(require.main === module, __filename)(async (span) => { const [domestics, directs, lans] = await getDomesticAndDirectDomainsRulesetPromise(); - const dataset: DNSMapping[] = ([DOH_BOOTSTRAP, DOMESTICS, DIRECTS] as const).flatMap(Object.values); + const dataset: Array<[name: string, DNSMapping]> = ([DOH_BOOTSTRAP, DOMESTICS, DIRECTS, LAN] as const).flatMap(Object.entries); return Promise.all([ new RulesetOutput(span, 'domestic', 'non_ip') @@ -108,6 +108,41 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as ]) .addFromRuleset(lans) .write(), + + ...dataset.map(([name, { ruleset, domains }]) => { + if (!ruleset) { + return; + } + + const output = new RulesetOutput(span, name.toLowerCase(), 'sukka_local_dns_mapping').withTitle(`Sukka's Ruleset - Local DNS Mapping (${name})`).withDescription([ + ...SHARED_DESCRIPTION, + '', + 'This is an internal rule that is only referenced by sukka_local_dns_mapping.sgmodule', + 'Do not use this file in your Rule section, all rules are included in non_ip/domestic.conf already.' + ]); + + domains.forEach((domain) => { + switch (domain[0]) { + case '$': + output.addDomain(domain.slice(1)); + break; + case '+': + output.addDomainSuffix(domain.slice(1)); + break; + default: + output.addDomainSuffix(domain); + break; + } + }); + + return output.write({ + surge: true, + clash: false, + singbox: false, + surgeDir: OUTPUT_MODULES_RULES_DIR + }); + }), + compareAndWriteFile( span, [ @@ -119,26 +154,31 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as // I use an object to deduplicate the domains // Otherwise I could just construct an array directly dataset.reduce>((acc, cur) => { - const { domains, dns, hosts } = cur; + const ruleset_name = cur[0].toLowerCase(); + const { domains, dns, hosts, ruleset } = cur[1]; Object.entries(hosts).forEach(([dns, ips]) => { acc[dns] ||= ips.join(', '); }); - domains.forEach((domain) => { - switch (domain[0]) { - case '$': - acc[domain.slice(1)] ||= `server:${dns}`; - break; - case '+': - acc[`*.${domain.slice(1)}`] ||= `server:${dns}`; - break; - default: - acc[domain] ||= `server:${dns}`; - acc[`*.${domain}`] ||= `server:${dns}`; - break; - } - }); + if (ruleset) { + acc[`RULE-SET:https://ruleset.skk.moe/Modules/Rules/sukka_local_dns_mapping/${ruleset_name}.conf`] ||= `server:${dns}`; + } else { + domains.forEach((domain) => { + switch (domain[0]) { + case '$': + acc[domain.slice(1)] ||= `server:${dns}`; + break; + case '+': + acc[`*.${domain.slice(1)}`] ||= `server:${dns}`; + break; + default: + acc[domain] ||= `server:${dns}`; + acc[`*.${domain}`] ||= `server:${dns}`; + break; + } + }); + } return acc; }, {}) @@ -153,7 +193,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as dns: { 'nameserver-policy': Record }, hosts: Record }>((acc, cur) => { - const { domains, dns, ...rest } = cur; + const { domains, dns, ...rest } = cur[1]; domains.forEach((domain) => { let domainWildcard = domain; if (domain[0] === '$') { diff --git a/Build/constants/dir.ts b/Build/constants/dir.ts index 335bb5cdf..956154267 100644 --- a/Build/constants/dir.ts +++ b/Build/constants/dir.ts @@ -11,5 +11,6 @@ export const OUTPUT_SURGE_DIR = path.join(PUBLIC_DIR, 'List'); export const OUTPUT_CLASH_DIR = path.resolve(PUBLIC_DIR, 'Clash'); export const OUTPUT_SINGBOX_DIR = path.resolve(PUBLIC_DIR, 'sing-box'); export const OUTPUT_MODULES_DIR = path.resolve(PUBLIC_DIR, 'Modules'); +export const OUTPUT_MODULES_RULES_DIR = path.resolve(OUTPUT_MODULES_DIR, 'Rules'); export const OUTPUT_INTERNAL_DIR = path.resolve(PUBLIC_DIR, 'Internal'); export const OUTPUT_MOCK_DIR = path.resolve(PUBLIC_DIR, 'Mock'); diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index 1d681a382..b0ddf2899 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -34,7 +34,7 @@ export abstract class RuleOutput { protected destPort = new Set(); protected otherRules: string[] = []; - protected abstract type: 'domainset' | 'non_ip' | 'ip'; + protected abstract type: 'domainset' | 'non_ip' | 'ip' | (string & {}); private pendingPromise: Promise | null = null; @@ -295,13 +295,29 @@ export abstract class RuleOutput { ); } - write(): Promise { + write({ + surge = true, + clash = true, + singbox = true, + surgeDir = OUTPUT_SURGE_DIR, + clashDir = OUTPUT_CLASH_DIR, + singboxDir = OUTPUT_SINGBOX_DIR + }: { + surge?: boolean, + clash?: boolean, + singbox?: boolean, + surgeDir?: string, + clashDir?: string, + singboxDir?: string + } = {}): Promise { return this.done().then(() => this.span.traceChildAsync('write all', async () => { invariant(this.title, 'Missing title'); invariant(this.description, 'Missing description'); - const promises = [ - compareAndWriteFile( + const promises: Array> = []; + + if (surge) { + promises.push(compareAndWriteFile( this.span, withBannerArray( this.title, @@ -309,9 +325,11 @@ export abstract class RuleOutput { this.date, this.surge() ), - path.join(OUTPUT_SURGE_DIR, this.type, this.id + '.conf') - ), - compareAndWriteFile( + path.join(surgeDir, this.type, this.id + '.conf') + )); + } + if (clash) { + promises.push(compareAndWriteFile( this.span, withBannerArray( this.title, @@ -319,14 +337,16 @@ export abstract class RuleOutput { this.date, this.clash() ), - path.join(OUTPUT_CLASH_DIR, this.type, this.id + '.txt') - ), - compareAndWriteFile( + path.join(clashDir, this.type, this.id + '.txt') + )); + } + if (singbox) { + promises.push(compareAndWriteFile( this.span, this.singbox(), - path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json') - ) - ]; + path.join(singboxDir, this.type, this.id + '.json') + )); + } if (this.mitmSgmodule) { const sgmodule = this.mitmSgmodule(); diff --git a/Build/lib/rules/ruleset.ts b/Build/lib/rules/ruleset.ts index 87da0865b..c45e1cf51 100644 --- a/Build/lib/rules/ruleset.ts +++ b/Build/lib/rules/ruleset.ts @@ -12,7 +12,7 @@ import { isProbablyIpv4, isProbablyIpv6 } from 'foxts/is-probably-ip'; type Preprocessed = [domain: string[], domainSuffix: string[], sortedDomainRules: string[]]; export class RulesetOutput extends RuleOutput { - constructor(span: Span, id: string, protected type: 'non_ip' | 'ip') { + constructor(span: Span, id: string, protected type: 'non_ip' | 'ip' | (string & {})) { super(span, id); } diff --git a/Source/non_ip/direct.ts b/Source/non_ip/direct.ts index 1217d37c1..4b55af003 100644 --- a/Source/non_ip/direct.ts +++ b/Source/non_ip/direct.ts @@ -4,6 +4,8 @@ export interface DNSMapping { }, /** which also disallows wildcard */ realip: boolean, + /** should convert to ruleset */ + ruleset: boolean, dns: string, /** * domain[0] @@ -20,6 +22,7 @@ export const DIRECTS = { dns: 'system', hosts: {}, realip: false, + ruleset: false, domains: [ 'securelogin.com.cn', '$captive.apple.com', @@ -30,6 +33,7 @@ export const DIRECTS = { dns: 'system', hosts: {}, realip: true, + ruleset: false, domains: [ '+m2m', // '+ts.net', // TailScale Magic DNS @@ -47,6 +51,7 @@ export const LAN = { dns: 'system', hosts: {}, realip: false, + ruleset: true, domains: [ '+home', // 'zte.home', // ZTE CPE @@ -106,6 +111,7 @@ export const LAN = { localhost: ['127.0.0.1'] }, realip: true, + ruleset: true, domains: [ '+lan', // 'amplifi.lan', diff --git a/Source/non_ip/domestic.ts b/Source/non_ip/domestic.ts index 9e16f18ed..62fb25696 100644 --- a/Source/non_ip/domestic.ts +++ b/Source/non_ip/domestic.ts @@ -5,6 +5,7 @@ export const DOMESTICS: Record = { hosts: {}, dns: 'quic://dns.alidns.com:853', realip: false, + ruleset: true, domains: [ 'uc.cn', // 'ucweb.com', // UC International @@ -82,13 +83,18 @@ export const DOMESTICS: Record = { 'tanx.com', 'hellobike.com', '+hichina.com', - '+yunos.com' + '+yunos.com', + + // Bilibili Aliyun CDN + '$upos-sz-mirrorali.bilivideo.com', + '$upos-sz-estgoss.bilivideo.com' ] }, TENCENT: { hosts: {}, dns: 'https://doh.pub/dns-query', realip: false, + ruleset: true, domains: [ // 'dns.pub', // 'doh.pub', @@ -144,28 +150,11 @@ export const DOMESTICS: Record = { '+codehub.cn' ] }, - BILIBILI_ALI: { - dns: 'quic://dns.alidns.com:853', - hosts: {}, - realip: false, - domains: [ - '$upos-sz-mirrorali.bilivideo.com', - '$upos-sz-estgoss.bilivideo.com' - ] - }, - BILIBILI_BD: { - dns: '180.76.76.76', - hosts: {}, - realip: false, - domains: [ - '$upos-sz-mirrorbd.bilivideo.com', - '$upos-sz-mirrorbos.bilivideo.com' - ] - }, BILIBILI: { dns: 'https://doh.pub/dns-query', hosts: {}, realip: false, + ruleset: true, domains: [ // '$upos-sz-mirrorcoso1.bilivideo.com', // already included in bilivideo.com // '$upos-sz-estgcos.bilivideo.com', // already included in bilivideo.com, tencent cloud cdn @@ -196,6 +185,7 @@ export const DOMESTICS: Record = { dns: 'https://doh.pub/dns-query', hosts: {}, realip: false, + ruleset: true, domains: [ 'mi.com', 'duokan.com', @@ -218,6 +208,7 @@ export const DOMESTICS: Record = { dns: '180.184.2.2', hosts: {}, realip: false, + ruleset: true, domains: [ 'bytedance.com', '+bytecdn.cn', @@ -271,6 +262,7 @@ export const DOMESTICS: Record = { dns: '180.76.76.76', hosts: {}, realip: false, + ruleset: true, domains: [ '91.com', 'hao123.com', @@ -295,13 +287,18 @@ export const DOMESTICS: Record = { '+bdydns.com', '+jomoxc.com', // Baidu PCDN, of sort '+duapp.com', - '+antpcdn.com' // Baidu PCDN + '+antpcdn.com', // Baidu PCDN + + // Bilibili Baidu CDN + '$upos-sz-mirrorbd.bilivideo.com', + '$upos-sz-mirrorbos.bilivideo.com' ] }, QIHOO360: { hosts: {}, dns: 'https://doh.360.cn/dns-query', realip: false, + ruleset: true, domains: [ '+qhimg.com', '+qhimgs.com', @@ -350,6 +347,7 @@ export const DOH_BOOTSTRAP: Record = { 'dns.alidns.com': ['223.5.5.5', '223.6.6.6', '2400:3200:baba::1', '2400:3200::1'] }, realip: false, + ruleset: false, dns: 'quic://223.5.5.5:853', domains: [ '$dns.alidns.com' @@ -362,6 +360,7 @@ export const DOH_BOOTSTRAP: Record = { // 'dns.pub': ['120.53.53.53', '1.12.12.12'] }, realip: false, + ruleset: false, dns: 'https://1.12.12.12/dns-query', domains: [ // '$dot.pub', @@ -382,6 +381,7 @@ export const DOH_BOOTSTRAP: Record = { // dot.360.net CNAME dns.360.net }, realip: false, + ruleset: false, // Surge only supports UDP 53 or Hosts as the bootstrap server of domain DoH dns: '101.198.198.198', // 'https://101.198.198.198/dns-query', // https://101.198.199.200/dns-query domains: [ From 79c8e2ceef4a5eb618c25ae19ea50a39e5838129 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 17:36:38 +0800 Subject: [PATCH 04/35] Update CDN & Global Hosts --- Source/domainset/cdn.conf | 5 +++++ Source/non_ip/global.conf | 2 ++ 2 files changed, 7 insertions(+) diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index 56e6ef9f5..a4306db47 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -325,6 +325,9 @@ openapi.vercel.sh # >> Static.app .static.domains +# >> Fleek +.on-fleek.app + # >> Twitter .twimg.com cdn.cms-twdigitalassets.com @@ -478,6 +481,7 @@ ip-ranges.amazonaws.com .assets.shortbread.aws.dev .cdn.chrome.marketplace.aws.dev .widgets.marketplace.aws.dev +cdn.builderprofile.aws.dev # >> SB.SB ooo.0o0.ooo @@ -3892,3 +3896,4 @@ media.lantern.io image.vuukle.com cdn.vuukle.com touchgal-image.moyu.moe +libraries.hund.io diff --git a/Source/non_ip/global.conf b/Source/non_ip/global.conf index 7c1a983c4..6f9305bf8 100644 --- a/Source/non_ip/global.conf +++ b/Source/non_ip/global.conf @@ -679,6 +679,7 @@ DOMAIN-SUFFIX,iheart.com DOMAIN-SUFFIX,iloveimg.com DOMAIN-SUFFIX,imo.im DOMAIN-SUFFIX,influxdata.com +DOMAIN-SUFFIX,instatus.com DOMAIN-SUFFIX,imdb.com DOMAIN-SUFFIX,imgur.com DOMAIN-SUFFIX,intel.com @@ -932,6 +933,7 @@ DOMAIN-SUFFIX,vivaldi.com DOMAIN-SUFFIX,vk.com DOMAIN-SUFFIX,vmware.com DOMAIN-SUFFIX,voachinese.com +DOMAIN-SUFFIX,voanews.com DOMAIN-SUFFIX,wakatime.com DOMAIN-SUFFIX,w3schools.com DOMAIN-SUFFIX,waifu2x.net From 43085cc00e075ea5183a3913f39cbe32142fc9e2 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 20:43:56 +0800 Subject: [PATCH 05/35] Chore: drop reject stats generation --- Build/build-public.ts | 5 +++-- Build/build-reject-domainset.ts | 17 -------------- Build/lib/rules/domainset.ts | 40 --------------------------------- 3 files changed, 3 insertions(+), 59 deletions(-) diff --git a/Build/build-public.ts b/Build/build-public.ts index 68e21dc83..44a27373b 100644 --- a/Build/build-public.ts +++ b/Build/build-public.ts @@ -6,7 +6,7 @@ import { task } from './trace'; import { treeDir, TreeFileType } from './lib/tree-dir'; import type { TreeType, TreeTypeArray } from './lib/tree-dir'; -import { OUTPUT_MOCK_DIR, OUTPUT_MODULES_DIR, PUBLIC_DIR, ROOT_DIR } from './constants/dir'; +import { OUTPUT_MOCK_DIR, OUTPUT_MODULES_DIR, OUTPUT_MODULES_RULES_DIR, PUBLIC_DIR, ROOT_DIR } from './constants/dir'; import { fastStringCompare, mkdirp, writeFile } from './lib/misc'; import picocolors from 'picocolors'; import { tagged as html } from 'foxts/tagged'; @@ -34,7 +34,8 @@ async function copyDirContents(srcDir: string, destDir: string) { export const buildPublic = task(require.main === module, __filename)(async (span) => { await span.traceChildAsync('copy rest of the files', async () => { await Promise.all([ - mkdirp(OUTPUT_MODULES_DIR), + // mkdirp(OUTPUT_MODULES_DIR), + mkdirp(OUTPUT_MODULES_RULES_DIR), mkdirp(OUTPUT_MOCK_DIR) ]); diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index f837c3dcc..bd713292d 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -129,26 +129,9 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as } }); - // Create reject stats - const rejectDomainsStats: string[] = span - .traceChild('create reject stats') - .traceSyncFn(() => { - const results = []; - results.push('=== base ==='); - appendArrayInPlace(results, rejectOutput.getStatMap()); - results.push('=== extra ==='); - appendArrayInPlace(results, rejectExtraOutput.getStatMap()); - return results; - }); - return Promise.all([ rejectOutput.write(), rejectExtraOutput.write(), - compareAndWriteFile( - span, - rejectDomainsStats, - path.join(OUTPUT_INTERNAL_DIR, 'reject-stats.txt') - ), compareAndWriteFile( span, appendArrayInPlace( diff --git a/Build/lib/rules/domainset.ts b/Build/lib/rules/domainset.ts index 845de59bf..5643774f1 100644 --- a/Build/lib/rules/domainset.ts +++ b/Build/lib/rules/domainset.ts @@ -1,11 +1,7 @@ -import { invariant } from 'foxts/guard'; import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; import { RuleOutput } from './base'; import type { SingboxSourceFormat } from '../singbox'; -import * as tldts from 'tldts-experimental'; -import { looseTldtsOpt } from '../../constants/loose-tldts-opt'; -import { fastStringCompare } from '../misc'; import { escapeStringRegexp } from 'foxts/escape-string-regexp'; export class DomainsetOutput extends RuleOutput { @@ -55,42 +51,6 @@ export class DomainsetOutput extends RuleOutput { } satisfies SingboxSourceFormat); } - protected apexDomainMap: Map | null = null; - getStatMap() { - this.runPreprocess(); - - invariant(this.$preprocessed, 'Non dumped yet'); - - if (!this.apexDomainMap) { - const domainMap = new Map(); - - for (let i = 0, len = this.$preprocessed.length; i < len; i++) { - const cur = this.$preprocessed[i]; - if (!domainMap.has(cur)) { - const domain = tldts.getDomain(cur, looseTldtsOpt); - domainMap.set(cur, domain ?? cur); - } - } - this.apexDomainMap = domainMap; - } - - return Array.from(this.$preprocessed - .reduce>( - (acc, cur) => { - const suffix = this.apexDomainMap!.get(cur); - if (suffix) { - acc.set(suffix, (acc.get(suffix) ?? 0) + 1); - } - return acc; - }, - new Map() - ) - .entries()) - .filter(a => a[1] > 9) - .sort((a, b) => (b[1] - a[1]) || fastStringCompare(a[0], b[0])) - .map(([domain, count]) => `${domain}${' '.repeat(100 - domain.length)}${count}`); - } - mitmSgmodule = undefined; adguardhome(): string[] { From 9a4acbd3f179516cd923d5e0ec45d22c0834cdf8 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 20:44:03 +0800 Subject: [PATCH 06/35] CI: fix remote git upload --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 80fb9326b..27da8a960 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -130,7 +130,7 @@ jobs: - name: Upload Dist to GitLab continue-on-error: true run: | - git clone --filter=tree:0 --no-tags --prune https://${GITLAB_TOKEN_NAME}:${GITLAB_TOKEN}@gitlab.com/SukkaW/ruleset.skk.moe.git ./deploy-git + git clone --filter=tree:0 --no-tags https://${GITLAB_TOKEN_NAME}:${GITLAB_TOKEN}@gitlab.com/SukkaW/ruleset.skk.moe.git ./deploy-git cd ./deploy-git git config --global push.default matching git config --global user.email "${GITLAB_EMAIL}" @@ -150,7 +150,7 @@ jobs: - name: Upload Dist to GitHub continue-on-error: true run: | - git clone --filter=tree:0 --no-tags --prune https://${GH_USER}:${GH_TOKEN}@github.com/SukkaLab/ruleset.skk.moe.git ./deploy-git + git clone --filter=tree:0 --no-tags https://${GH_USER}:${GH_TOKEN}@github.com/SukkaLab/ruleset.skk.moe.git ./deploy-git cd ./deploy-git git config --global push.default matching git config --global user.email "${GH_EMAIL}" From 7d555d0ba9ec1d9a50cf8139b1018969ca5c563c Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 20:47:16 +0800 Subject: [PATCH 07/35] Chore: clean up `make-fetch-happen` --- Build/index.ts | 7 +- Build/lib/make-fetch-happen.ts | 87 -------------- package.json | 8 +- pnpm-lock.yaml | 214 --------------------------------- 4 files changed, 5 insertions(+), 311 deletions(-) delete mode 100644 Build/lib/make-fetch-happen.ts diff --git a/Build/index.ts b/Build/index.ts index 6972faeb4..40c95bdb5 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -28,7 +28,6 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules'; import { createSpan, printTraceResult, whyIsNodeRunning } from './trace'; import { buildDeprecateFiles } from './build-deprecate-files'; -import { cacheGc } from './lib/make-fetch-happen'; import path from 'node:path'; import { ROOT_DIR } from './constants/dir'; @@ -97,10 +96,8 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); downloadMockAssets(rootSpan) ]); - await Promise.all([ - buildDeprecateFiles(rootSpan).then(() => buildPublic(rootSpan)), - cacheGc(rootSpan) - ]); + await buildDeprecateFiles(rootSpan); + await buildPublic(rootSpan); rootSpan.stop(); diff --git a/Build/lib/make-fetch-happen.ts b/Build/lib/make-fetch-happen.ts deleted file mode 100644 index 7a6529b9b..000000000 --- a/Build/lib/make-fetch-happen.ts +++ /dev/null @@ -1,87 +0,0 @@ -import path from 'node:path'; -import fsp from 'node:fs/promises'; -// import makeFetchHappen from 'make-fetch-happen'; -// import type { FetchOptions } from 'make-fetch-happen'; -// import cacache from 'cacache'; -// import picocolors from 'picocolors'; - -import { task } from '../trace'; -import { ROOT_DIR } from '../constants/dir'; -// import { bytes } from 'xbits'; - -const cachePath = path.join(ROOT_DIR, '.cache/__make_fetch_happen__'); -// fs.mkdirSync(cachePath, { recursive: true }); - -// interface CacacheVerifyStats { -// startTime: Date, -// endTime: Date, -// runTime: { -// markStartTime: 0, -// fixPerms: number, -// garbageCollect: number, -// rebuildIndex: number, -// cleanTmp: number, -// writeVerifile: number, -// markEndTime: number, -// total: number -// }, -// verifiedContent: number, -// reclaimedCount: number, -// reclaimedSize: number, -// badContentCount: number, -// keptSize: number, -// missingContent: number, -// rejectedEntries: number, -// totalEntries: number -// } - -export const cacheGc = task(require.main === module, __filename)( - () => fsp.rm(cachePath, { recursive: true, force: true }) - // span - // .traceChildAsync('cacache gc', () => cacache.verify(cachePath, { concurrency: 64 })) - // .then((stats: CacacheVerifyStats) => { - // // console.log({ stats }); - // console.log(picocolors.green('[cacheGc] running gc on cache:'), cachePath); - // console.log(picocolors.green('[cacheGc] content verified:'), stats.verifiedContent, '(' + bytes(stats.keptSize) + ')'); - // console.log(picocolors.green('[cacheGc] reclaimed:'), stats.reclaimedCount, '(' + bytes(stats.reclaimedSize) + ')'); - // }); -); - -// const _fetch = makeFetchHappen.defaults({ -// cachePath, -// maxSockets: 32, /** -// * They said 15 is a good default that prevents knocking out others' routers, -// * I disagree. 32 is a good number. -// */ -// headers: { -// 'User-Agent': 'curl/8.9.1 (https://github.com/SukkaW/Surge)' -// }, -// retry: { -// retries: 5, -// randomize: true -// } -// }); - -// export function $fetch(uriOrRequest: string | Request, opts?: FetchOptions) { -// return _fetch(uriOrRequest, opts).then((resp) => { -// printResponseStatus(resp); -// return resp; -// }); -// } - -// export async function $delete(resp: NodeFetchResponse) { -// const cacheKey = resp.headers.get('X-Local-Cache-Key'); -// if (cacheKey) { -// await cacache.rm.entry(cachePath, cacheKey); -// await cacache.verify(cachePath, { concurrency: 64 }); -// } -// } - -// export function printResponseStatus(resp: NodeFetchResponse) { -// const status = resp.headers.get('X-Local-Cache-Status'); -// if (status) { -// console.log('[$fetch cache]', { status }, picocolors.gray(resp.url)); -// } -// } - -// export { type Response as NodeFetchResponse } from 'node-fetch'; diff --git a/package.json b/package.json index 74ec4f646..0d128db14 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,6 @@ "foxts": "^1.1.5", "hash-wasm": "^4.12.0", "json-stringify-pretty-compact": "^3.0.0", - "make-fetch-happen": "^14.0.3", "picocolors": "^1.1.1", "tar-fs": "^3.0.6", "tinyexec": "^0.3.2", @@ -57,7 +56,6 @@ "@types/cacache": "^17.0.2", "@types/dns2": "^2.0.9", "@types/fast-fifo": "^1.3.0", - "@types/make-fetch-happen": "^10.0.4", "@types/mocha": "^10.0.10", "@types/node": "^22.10.5", "@types/node-fetch": "^2.6.12", @@ -72,13 +70,13 @@ "typescript": "^5.7.3" }, "packageManager": "pnpm@9.15.3", - "resolutions": { - "has": "npm:@nolyfill/has@latest" - }, "pnpm": { "patchedDependencies": { "undici": "patches/undici.patch", "whoiser": "patches/whoiser.patch" } + }, + "resolutions": { + "has": "npm:@nolyfill/has@latest" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index beb43b2a1..40b008757 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -61,9 +61,6 @@ importers: json-stringify-pretty-compact: specifier: ^3.0.0 version: 3.0.0 - make-fetch-happen: - specifier: ^14.0.3 - version: 14.0.3 picocolors: specifier: ^1.1.1 version: 1.1.1 @@ -125,9 +122,6 @@ importers: '@types/fast-fifo': specifier: ^1.3.0 version: 1.3.0 - '@types/make-fetch-happen': - specifier: ^10.0.4 - version: 10.0.4 '@types/mocha': specifier: ^10.0.10 version: 10.0.10 @@ -311,10 +305,6 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} - '@npmcli/agent@3.0.0': - resolution: {integrity: sha512-S79NdEgDQd/NGCay6TCoVzXSj74skRZIKJcpJjC5lOq34SZzyI6MqtiiWoiVWoVrTcGjNeC4ipbh1VIHlpfF5Q==} - engines: {node: ^18.17.0 || >=20.5.0} - '@npmcli/fs@4.0.0': resolution: {integrity: sha512-/xGlezI6xfGO9NwuJlnwz/K14qD1kCSAGtacBHnGzeAIuJGazcp45KP5NuyARXoKb7cwulAGWVsbeSxdG/cb0Q==} engines: {node: ^18.17.0 || >=20.5.0} @@ -549,9 +539,6 @@ packages: '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} - '@types/make-fetch-happen@10.0.4': - resolution: {integrity: sha512-jKzweQaEMMAi55ehvR1z0JF6aSVQm/h1BXBhPLOJriaeQBctjw5YbpIGs7zAx9dN0Sa2OO5bcXwCkrlgenoPEA==} - '@types/mocha@10.0.10': resolution: {integrity: sha512-xPyYSz1cMPnJQhl0CLMH68j3gprKZaTjG3s5Vi+fDgx+uhG9NOXwbVt52eFS8ECyXhyKcjDLCBEqBExKuiZb7Q==} @@ -564,9 +551,6 @@ packages: '@types/retry@0.12.5': resolution: {integrity: sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==} - '@types/ssri@7.1.5': - resolution: {integrity: sha512-odD/56S3B51liILSk5aXJlnYt99S6Rt9EFDDqGtJM26rKHApHcwyU/UoYHrzKkdkHMAIquGWCuHtQTbes+FRQw==} - '@types/stack-utils@2.0.3': resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==} @@ -670,10 +654,6 @@ packages: engines: {node: '>=0.4.0'} hasBin: true - agent-base@7.1.1: - resolution: {integrity: sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==} - engines: {node: '>= 14'} - ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} @@ -903,9 +883,6 @@ packages: emoji-regex@9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} - encoding@0.1.13: - resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==} - end-of-stream@1.4.4: resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} @@ -913,9 +890,6 @@ packages: resolution: {integrity: sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==} engines: {node: '>=10.13.0'} - err-code@2.0.3: - resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} - escalade@3.2.0: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} @@ -1229,21 +1203,6 @@ packages: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} hasBin: true - http-cache-semantics@4.1.1: - resolution: {integrity: sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==} - - http-proxy-agent@7.0.2: - resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} - engines: {node: '>= 14'} - - https-proxy-agent@7.0.5: - resolution: {integrity: sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==} - engines: {node: '>= 14'} - - iconv-lite@0.6.3: - resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} - engines: {node: '>=0.10.0'} - ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} @@ -1265,10 +1224,6 @@ packages: ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} - ip-address@9.0.5: - resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==} - engines: {node: '>= 12'} - is-binary-path@2.1.0: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} engines: {node: '>=8'} @@ -1337,9 +1292,6 @@ packages: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} hasBin: true - jsbn@1.1.0: - resolution: {integrity: sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==} - jsdoc-type-pratt-parser@4.1.0: resolution: {integrity: sha512-Hicd6JK5Njt2QB6XYFS7ok9e37O8AYk3jTcppG4YVQnYjOemymvTcmc7OWsmq/Qqj5TdRFO5/x/tIPmBeRtGHg==} engines: {node: '>=12.0.0'} @@ -1381,10 +1333,6 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} - make-fetch-happen@14.0.3: - resolution: {integrity: sha512-QMjGbFTP0blj97EeidG5hk/QhKQ3T4ICckQGLgz38QF7Vgbk6e6FTARN8KhKxyBbWn8R0HU+bnw8aSoFPD4qtQ==} - engines: {node: ^18.17.0 || >=20.5.0} - merge2@1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -1423,10 +1371,6 @@ packages: resolution: {integrity: sha512-D7V8PO9oaz7PWGLbCACuI1qEOsq7UKfLotx/C0Aet43fCUB/wfQ7DYeq2oR/svFJGYDHPr38SHATeaj/ZoKHKw==} engines: {node: '>=16 || 14 >=14.17'} - minipass-fetch@4.0.0: - resolution: {integrity: sha512-2v6aXUXwLP1Epd/gc32HAMIWoczx+fZwEPRHm/VwtrJzRGwR1qGZXEYV3Zp8ZjjbwaZhMrM6uHV4KVkk+XCc2w==} - engines: {node: ^18.17.0 || >=20.5.0} - minipass-flush@1.0.5: resolution: {integrity: sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==} engines: {node: '>= 8'} @@ -1435,10 +1379,6 @@ packages: resolution: {integrity: sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==} engines: {node: '>=8'} - minipass-sized@1.0.3: - resolution: {integrity: sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==} - engines: {node: '>=8'} - minipass@3.3.6: resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} engines: {node: '>=8'} @@ -1476,10 +1416,6 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} - negotiator@1.0.0: - resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} - engines: {node: '>= 0.6'} - node-abi@3.71.0: resolution: {integrity: sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw==} engines: {node: '>=10'} @@ -1560,17 +1496,9 @@ packages: resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} - proc-log@5.0.0: - resolution: {integrity: sha512-Azwzvl90HaF0aCz1JrDdXQykFakSSNPaPoiZ9fm5qJIMHioDZEi7OAdRwSm6rSoPtY3Qutnm3L7ogmg3dc+wbQ==} - engines: {node: ^18.17.0 || >=20.5.0} - promise-make-naked@2.1.2: resolution: {integrity: sha512-y7s8ZuHIG56JYspB24be9GFkXA1zXL85Ur9u1DKrW/tvyUoPxWgBjnalK6Nc6l7wHBcAW0c3PO07+XOsWTRuhg==} - promise-retry@2.0.1: - resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==} - engines: {node: '>=10'} - pump@3.0.2: resolution: {integrity: sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==} @@ -1625,10 +1553,6 @@ packages: resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} hasBin: true - retry@0.12.0: - resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} - engines: {node: '>= 4'} - retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} @@ -1647,9 +1571,6 @@ packages: safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} - safer-buffer@2.1.2: - resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} - scslre@0.3.0: resolution: {integrity: sha512-3A6sD0WYP7+QrjbfNA2FN3FsOaGGFoekCVgTyypy53gPxhbkCIjtO6YWgdrfM+n/8sI8JeXZOIxsHjMTNxQ4nQ==} engines: {node: ^14.0.0 || >=16.0.0} @@ -1684,18 +1605,6 @@ packages: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} - smart-buffer@4.2.0: - resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} - engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} - - socks-proxy-agent@8.0.4: - resolution: {integrity: sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==} - engines: {node: '>= 14'} - - socks@2.8.3: - resolution: {integrity: sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==} - engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} - source-map-support@0.5.21: resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} @@ -1703,9 +1612,6 @@ packages: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} - sprintf-js@1.1.3: - resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} - ssri@12.0.0: resolution: {integrity: sha512-S7iGNosepx9RadX82oimUkvr0Ct7IjJbEbs4mJcTxst8um95J3sDYU1RBEOvdu6oL1Wek2ODI5i4MAw+dZ6cAQ==} engines: {node: ^18.17.0 || >=20.5.0} @@ -2117,16 +2023,6 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} - '@npmcli/agent@3.0.0': - dependencies: - agent-base: 7.1.1 - http-proxy-agent: 7.0.2 - https-proxy-agent: 7.0.5 - lru-cache: 10.4.3 - socks-proxy-agent: 8.0.4 - transitivePeerDependencies: - - supports-color - '@npmcli/fs@4.0.0': dependencies: semver: 7.6.3 @@ -2331,12 +2227,6 @@ snapshots: '@types/json-schema@7.0.15': {} - '@types/make-fetch-happen@10.0.4': - dependencies: - '@types/node-fetch': 2.6.12 - '@types/retry': 0.12.5 - '@types/ssri': 7.1.5 - '@types/mocha@10.0.10': {} '@types/node-fetch@2.6.12': @@ -2350,10 +2240,6 @@ snapshots: '@types/retry@0.12.5': {} - '@types/ssri@7.1.5': - dependencies: - '@types/node': 22.10.5 - '@types/stack-utils@2.0.3': {} '@types/tar-fs@2.0.4': @@ -2493,12 +2379,6 @@ snapshots: acorn@8.14.0: {} - agent-base@7.1.1: - dependencies: - debug: 4.3.7(supports-color@8.1.1) - transitivePeerDependencies: - - supports-color - ajv@6.12.6: dependencies: fast-deep-equal: 3.1.3 @@ -2717,11 +2597,6 @@ snapshots: emoji-regex@9.2.2: {} - encoding@0.1.13: - dependencies: - iconv-lite: 0.6.3 - optional: true - end-of-stream@1.4.4: dependencies: once: 1.4.0 @@ -2731,8 +2606,6 @@ snapshots: graceful-fs: 4.2.11 tapable: 2.2.1 - err-code@2.0.3: {} - escalade@3.2.0: {} escape-string-regexp@2.0.0: {} @@ -3115,27 +2988,6 @@ snapshots: he@1.2.0: {} - http-cache-semantics@4.1.1: {} - - http-proxy-agent@7.0.2: - dependencies: - agent-base: 7.1.1 - debug: 4.3.7(supports-color@8.1.1) - transitivePeerDependencies: - - supports-color - - https-proxy-agent@7.0.5: - dependencies: - agent-base: 7.1.1 - debug: 4.3.7(supports-color@8.1.1) - transitivePeerDependencies: - - supports-color - - iconv-lite@0.6.3: - dependencies: - safer-buffer: 2.1.2 - optional: true - ieee754@1.2.1: {} ignore@5.3.2: {} @@ -3151,11 +3003,6 @@ snapshots: ini@1.3.8: {} - ip-address@9.0.5: - dependencies: - jsbn: 1.1.0 - sprintf-js: 1.1.3 - is-binary-path@2.1.0: dependencies: binary-extensions: 2.3.0 @@ -3233,8 +3080,6 @@ snapshots: dependencies: argparse: 2.0.1 - jsbn@1.1.0: {} - jsdoc-type-pratt-parser@4.1.0: {} json-buffer@3.0.1: {} @@ -3274,22 +3119,6 @@ snapshots: lru-cache@10.4.3: {} - make-fetch-happen@14.0.3: - dependencies: - '@npmcli/agent': 3.0.0 - cacache: 19.0.1 - http-cache-semantics: 4.1.1 - minipass: 7.1.2 - minipass-fetch: 4.0.0 - minipass-flush: 1.0.5 - minipass-pipeline: 1.2.4 - negotiator: 1.0.0 - proc-log: 5.0.0 - promise-retry: 2.0.1 - ssri: 12.0.0 - transitivePeerDependencies: - - supports-color - merge2@1.4.1: {} micromatch@4.0.8: @@ -3323,14 +3152,6 @@ snapshots: dependencies: minipass: 7.1.2 - minipass-fetch@4.0.0: - dependencies: - minipass: 7.1.2 - minipass-sized: 1.0.3 - minizlib: 3.0.1 - optionalDependencies: - encoding: 0.1.13 - minipass-flush@1.0.5: dependencies: minipass: 3.3.6 @@ -3339,10 +3160,6 @@ snapshots: dependencies: minipass: 3.3.6 - minipass-sized@1.0.3: - dependencies: - minipass: 3.3.6 - minipass@3.3.6: dependencies: yallist: 4.0.0 @@ -3389,8 +3206,6 @@ snapshots: natural-compare@1.4.0: {} - negotiator@1.0.0: {} - node-abi@3.71.0: dependencies: semver: 7.6.3 @@ -3483,15 +3298,8 @@ snapshots: ansi-styles: 5.2.0 react-is: 18.3.1 - proc-log@5.0.0: {} - promise-make-naked@2.1.2: {} - promise-retry@2.0.1: - dependencies: - err-code: 2.0.3 - retry: 0.12.0 - pump@3.0.2: dependencies: end-of-stream: 1.4.4 @@ -3547,8 +3355,6 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 - retry@0.12.0: {} - retry@0.13.1: {} reusify@1.0.4: {} @@ -3563,9 +3369,6 @@ snapshots: safe-buffer@5.2.1: {} - safer-buffer@2.1.2: - optional: true - scslre@0.3.0: dependencies: '@eslint-community/regexpp': 4.12.1 @@ -3596,21 +3399,6 @@ snapshots: slash@3.0.0: {} - smart-buffer@4.2.0: {} - - socks-proxy-agent@8.0.4: - dependencies: - agent-base: 7.1.1 - debug: 4.3.7(supports-color@8.1.1) - socks: 2.8.3 - transitivePeerDependencies: - - supports-color - - socks@2.8.3: - dependencies: - ip-address: 9.0.5 - smart-buffer: 4.2.0 - source-map-support@0.5.21: dependencies: buffer-from: 1.1.2 @@ -3618,8 +3406,6 @@ snapshots: source-map@0.6.1: {} - sprintf-js@1.1.3: {} - ssri@12.0.0: dependencies: minipass: 7.1.2 From c8fb446bb5ed2215a7302e921832409326db7d0e Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 20:52:29 +0800 Subject: [PATCH 08/35] CI: test GitHub arm64 runner --- .github/workflows/check-source-domain.yml | 2 +- .github/workflows/main.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-source-domain.yml b/.github/workflows/check-source-domain.yml index fd5283115..03934ee55 100644 --- a/.github/workflows/check-source-domain.yml +++ b/.github/workflows/check-source-domain.yml @@ -6,7 +6,7 @@ on: jobs: check: name: Check - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: # - name: Tune GitHub-hosted runner network diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 27da8a960..48c4131d3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ concurrency: jobs: build: name: Build - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: # - name: Tune GitHub-hosted runner network @@ -92,7 +92,7 @@ jobs: - build name: Deploy to Cloudflare Pages if: github.ref == 'refs/heads/master' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: - name: Get NPM cache directory path id: npm_cache_path @@ -121,7 +121,7 @@ jobs: - build name: Deploy to GitHub and GitLab if: github.ref == 'refs/heads/master' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: - uses: actions/download-artifact@v4 with: From c07fe1dd56c79967d55694a1b6a050e64c31499c Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 20:56:34 +0800 Subject: [PATCH 09/35] CI: add arch to `npx wrangler` cache key --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 48c4131d3..136339b16 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -104,7 +104,7 @@ jobs: path: | ${{ steps.npm_cache_path.outputs.dir }} node_modules - key: ${{ runner.os }}-deploy-to-cloudflare-npm + key: ${{ runner.os }}-${{ runner.arch }}-deploy-to-cloudflare-npm - uses: actions/download-artifact@v4 with: name: build-artifact-${{ github.ref_name }} From c6bbbf9d4c7032e3d0d357e7546349714438f50f Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 21:28:28 +0800 Subject: [PATCH 10/35] Perf: reduce operations in `normalizeDomain` --- Build/lib/normalize-domain.ts | 28 +++++++++++++-------------- Build/lib/parse-filter/domainlists.ts | 4 ++-- Build/lib/parse-filter/filters.ts | 11 ++++++++--- Build/lib/parse-filter/hosts.ts | 4 ++-- Build/validate-gfwlist.ts | 4 ++-- Source/stream.ts | 2 +- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/Build/lib/normalize-domain.ts b/Build/lib/normalize-domain.ts index 1cefabc2b..093d415a3 100644 --- a/Build/lib/normalize-domain.ts +++ b/Build/lib/normalize-domain.ts @@ -6,28 +6,28 @@ import { normalizeTldtsOpt } from '../constants/loose-tldts-opt'; type TldTsParsed = ReturnType; +/** + * Skipped the input non-empty check, the `domain` should not be empty. + */ +export function fastNormalizeDomain(domain: string, parsed: TldTsParsed = tldts.parse(domain, normalizeTldtsOpt)) { + if (parsed.isIp) return null; + // Private invalid domain (things like .tor, .dn42, etc) + if (!parsed.isIcann && !parsed.isPrivate) return null; + + return parsed.hostname; +} + export function normalizeDomain(domain: string, parsed: TldTsParsed | null = null) { if (domain.length === 0) return null; parsed ??= tldts.parse(domain, normalizeTldtsOpt); if (parsed.isIp) return null; - - let h = parsed.hostname; - if (h === null) return null; // Private invalid domain (things like .tor, .dn42, etc) if (!parsed.isIcann && !parsed.isPrivate) return null; - let sliceStart = 0; - let sliceEnd = 0; - - if (h[0] === '.') sliceStart = 1; - // eslint-disable-next-line sukka/string/prefer-string-starts-ends-with -- performance - if (h[h.length - 1] === '.') sliceEnd = -1; - - if (sliceStart !== 0 || sliceEnd !== 0) { - h = h.slice(sliceStart, sliceEnd); - } + // const h = parsed.hostname; + // if (h === null) return null; - return h.length > 0 ? h : null; + return parsed.hostname; } diff --git a/Build/lib/parse-filter/domainlists.ts b/Build/lib/parse-filter/domainlists.ts index 0587f12df..52b776d89 100644 --- a/Build/lib/parse-filter/domainlists.ts +++ b/Build/lib/parse-filter/domainlists.ts @@ -1,5 +1,5 @@ import picocolors from 'picocolors'; -import { normalizeDomain } from '../normalize-domain'; +import { fastNormalizeDomain } from '../normalize-domain'; import { processLine } from '../process-line'; import { onBlackFound } from './shared'; import { fetchAssetsWithout304 } from '../fetch-assets'; @@ -9,7 +9,7 @@ function domainListLineCb(l: string, set: string[], includeAllSubDomain: boolean const line = processLine(l); if (!line) return; - const domain = normalizeDomain(line); + const domain = fastNormalizeDomain(line); if (!domain) return; if (domain !== line) { console.log( diff --git a/Build/lib/parse-filter/filters.ts b/Build/lib/parse-filter/filters.ts index eb5fc4434..18b14fe94 100644 --- a/Build/lib/parse-filter/filters.ts +++ b/Build/lib/parse-filter/filters.ts @@ -3,7 +3,7 @@ import type { Span } from '../../trace'; import { fetchAssetsWithout304 } from '../fetch-assets'; import { onBlackFound, onWhiteFound } from './shared'; import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; -import { normalizeDomain } from '../normalize-domain'; +import { fastNormalizeDomain } from '../normalize-domain'; import { looseTldtsOpt } from '../../constants/loose-tldts-opt'; import tldts from 'tldts-experimental'; import { NetworkFilter } from '@ghostery/adblocker'; @@ -227,7 +227,7 @@ export function parse($line: string, result: [string, ParseType], allowThirdPart && filter.isPlain() // isPlain() === !isRegex() && (!filter.isFullRegex()) ) { - const hostname = normalizeDomain(filter.hostname); + const hostname = fastNormalizeDomain(filter.hostname); if (!hostname) { result[1] = ParseType.Null; return result; @@ -421,6 +421,11 @@ export function parse($line: string, result: [string, ParseType], allowThirdPart } const sliced = (sliceStart > 0 || sliceEnd < 0) ? line.slice(sliceStart, sliceEnd === 0 ? undefined : sliceEnd) : line; + if (sliced.length === 0) { + result[1] = ParseType.Null; + return result; + } + if (sliced.charCodeAt(0) === 45 /* - */) { // line.startsWith('-') is not a valid domain result[1] = ParseType.ErrorMessage; @@ -437,7 +442,7 @@ export function parse($line: string, result: [string, ParseType], allowThirdPart return result; } - const domain = normalizeDomain(sliced); + const domain = fastNormalizeDomain(sliced); if (domain && domain === sliced) { result[0] = domain; diff --git a/Build/lib/parse-filter/hosts.ts b/Build/lib/parse-filter/hosts.ts index 8f0857b92..4d8fcec6d 100644 --- a/Build/lib/parse-filter/hosts.ts +++ b/Build/lib/parse-filter/hosts.ts @@ -1,6 +1,6 @@ import type { Span } from '../../trace'; import { fetchAssetsWithout304 } from '../fetch-assets'; -import { normalizeDomain } from '../normalize-domain'; +import { fastNormalizeDomain } from '../normalize-domain'; import { processLine } from '../process-line'; import { onBlackFound } from './shared'; @@ -14,7 +14,7 @@ function hostsLineCb(l: string, set: string[], includeAllSubDomain: boolean, met if (!_domain) { return; } - const domain = normalizeDomain(_domain); + const domain = fastNormalizeDomain(_domain); if (!domain) { return; } diff --git a/Build/validate-gfwlist.ts b/Build/validate-gfwlist.ts index 8b67fda65..3922b7242 100644 --- a/Build/validate-gfwlist.ts +++ b/Build/validate-gfwlist.ts @@ -1,5 +1,5 @@ import { processLine } from './lib/process-line'; -import { normalizeDomain } from './lib/normalize-domain'; +import { fastNormalizeDomain } from './lib/normalize-domain'; import { HostnameSmolTrie } from './lib/trie'; // import { Readable } from 'stream'; import { parse } from 'csv-parse/sync'; @@ -54,7 +54,7 @@ export async function parseGfwList() { trie.add(line); continue; } - const d = normalizeDomain(line); + const d = fastNormalizeDomain(line); if (d) { trie.add(d); continue; diff --git a/Source/stream.ts b/Source/stream.ts index ea6f77fee..19e365ec2 100644 --- a/Source/stream.ts +++ b/Source/stream.ts @@ -879,7 +879,7 @@ export const HK: StreamService[] = [ NOW_E, VIUTV, MYTV_SUPER, - HBO_ASIA, + HBO_ASIA // BILIBILI_INTL ]; From 6f8d515e7c61944c33c8e9549796b67482bfacc8 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 21:51:58 +0800 Subject: [PATCH 11/35] Perf: speed up file comparison --- Build/lib/rules/base.ts | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index b0ddf2899..d2e660507 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -381,32 +381,39 @@ export abstract class RuleOutput { abstract mitmSgmodule?(): string[] | null; } -export async function fileEqual(linesA: string[], source: AsyncIterable): Promise { +export async function fileEqual(linesA: string[], source: AsyncIterable | Iterable): Promise { if (linesA.length === 0) { return false; } + const linesABound = linesA.length - 1; + let index = -1; for await (const lineB of source) { index++; - if (index > linesA.length - 1) { - return (index === linesA.length && lineB === ''); + if (index > linesABound) { + return (index === linesA.length && lineB.length === 0); } const lineA = linesA[index]; - if (lineA[0] === '#' && lineB[0] === '#') { + const firstCharA = lineA.charCodeAt(0); + const firstCharB = lineB.charCodeAt(0); + + if (firstCharA === 35 /* # */ && firstCharB === 35 /* # */) { continue; } // adguard conf - if (lineA[0] === '!' && lineB[0] === '!') { + if (firstCharA === 33 /* ! */ && firstCharB === 33 /* ! */) { continue; } + if ( - lineA[0] === '/' + firstCharA === 47 /* / */ + && firstCharB === 47 /* / */ + && lineA[1] === '/' - && lineB[0] === '/' && lineB[1] === '/' && lineA[3] === '#' && lineB[3] === '#' @@ -420,7 +427,7 @@ export async function fileEqual(linesA: string[], source: AsyncIterable) } // The file becomes larger - return !(index < linesA.length - 1); + return !(index < linesABound); } export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) { From 6235d9fee0641076a42d9a6ed19a9c877e5b47c8 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 23:06:51 +0800 Subject: [PATCH 12/35] Chore: minor changes --- Build/build-reject-ip-list.ts | 4 ++-- Build/lib/fetch-assets.ts | 21 +++------------------ Build/lib/fetch-retry.ts | 7 +++---- Build/lib/parse-filter/domainlists.ts | 4 ++-- Build/lib/parse-filter/filters.ts | 4 ++-- Build/lib/parse-filter/hosts.ts | 4 ++-- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 8 files changed, 20 insertions(+), 36 deletions(-) diff --git a/Build/build-reject-ip-list.ts b/Build/build-reject-ip-list.ts index eb059d0bf..14b8ea172 100644 --- a/Build/build-reject-ip-list.ts +++ b/Build/build-reject-ip-list.ts @@ -8,7 +8,7 @@ import { processLine } from './lib/process-line'; import { RulesetOutput } from './lib/create-file'; import { SOURCE_DIR } from './constants/dir'; import { $$fetch } from './lib/fetch-retry'; -import { fetchAssetsWithout304 } from './lib/fetch-assets'; +import { fetchAssets } from './lib/fetch-assets'; const BOGUS_NXDOMAIN_URL = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf'; const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = $$fetch(BOGUS_NXDOMAIN_URL).then(async (resp) => { @@ -37,7 +37,7 @@ const BOTNET_FILTER_MIRROR_URL = [ // https://curbengh.github.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt ]; -const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssetsWithout304(BOTNET_FILTER_URL, BOTNET_FILTER_MIRROR_URL).then(text => text.split('\n').reduce<[ipv4: string[], ipv6: string[]]>((acc, cur) => { +const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets(BOTNET_FILTER_URL, BOTNET_FILTER_MIRROR_URL).then(text => text.split('\n').reduce<[ipv4: string[], ipv6: string[]]>((acc, cur) => { const ip = processLine(cur); if (ip) { if (isProbablyIpv4(ip)) { diff --git a/Build/lib/fetch-assets.ts b/Build/lib/fetch-assets.ts index 558012af4..295e293d5 100644 --- a/Build/lib/fetch-assets.ts +++ b/Build/lib/fetch-assets.ts @@ -1,6 +1,6 @@ import picocolors from 'picocolors'; import { $$fetch, defaultRequestInit, ResponseError } from './fetch-retry'; -import { wait } from 'foxts/wait'; +import { waitWithAbort } from 'foxts/wait'; // eslint-disable-next-line sukka/unicorn/custom-error-definition -- typescript is better export class CustomAbortError extends Error { @@ -26,29 +26,14 @@ export class CustomNoETagFallbackError extends Error { } } -export function sleepWithAbort(ms: number, signal: AbortSignal) { - return new Promise((resolve, reject) => { - if (signal.aborted) { - reject(signal.reason as Error); - return; - } - - signal.addEventListener('abort', stop, { once: true }); - - wait(ms).then(resolve).catch(reject).finally(() => signal.removeEventListener('abort', stop)); - - function stop(this: AbortSignal) { reject(this.reason as Error); } - }); -} - -export async function fetchAssetsWithout304(url: string, fallbackUrls: null | undefined | string[] | readonly string[]) { +export async function fetchAssets(url: string, fallbackUrls: null | undefined | string[] | readonly string[]) { const controller = new AbortController(); const createFetchFallbackPromise = async (url: string, index: number) => { if (index >= 0) { // Most assets can be downloaded within 250ms. To avoid wasting bandwidth, we will wait for 500ms before downloading from the fallback URL. try { - await sleepWithAbort(50 + (index + 1) * 100, controller.signal); + await waitWithAbort(50 + (index + 1) * 100, controller.signal); } catch { console.log(picocolors.gray('[fetch cancelled early]'), picocolors.gray(url)); throw new CustomAbortError(); diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index 7842f8210..64d2c2e3e 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -95,10 +95,9 @@ setGlobalDispatcher(agent.compose( : retryAfter * 1e3; // Retry-After is in seconds } - const retryTimeout - = retryAfter > 0 - ? Math.min(retryAfter, maxTimeout) - : Math.min(minTimeout * (timeoutFactor ** (counter - 1)), maxTimeout); + const retryTimeout = retryAfter > 0 + ? Math.min(retryAfter, maxTimeout) + : Math.min(minTimeout * (timeoutFactor ** (counter - 1)), maxTimeout); console.log('[fetch retry]', 'schedule retry', { statusCode, retryTimeout, errorCode, url: opts.origin }); // eslint-disable-next-line sukka/prefer-timer-id -- won't leak diff --git a/Build/lib/parse-filter/domainlists.ts b/Build/lib/parse-filter/domainlists.ts index 52b776d89..a02840791 100644 --- a/Build/lib/parse-filter/domainlists.ts +++ b/Build/lib/parse-filter/domainlists.ts @@ -2,7 +2,7 @@ import picocolors from 'picocolors'; import { fastNormalizeDomain } from '../normalize-domain'; import { processLine } from '../process-line'; import { onBlackFound } from './shared'; -import { fetchAssetsWithout304 } from '../fetch-assets'; +import { fetchAssets } from '../fetch-assets'; import type { Span } from '../../trace'; function domainListLineCb(l: string, set: string[], includeAllSubDomain: boolean, meta: string) { @@ -32,7 +32,7 @@ export function processDomainLists( domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false ) { return span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (span) => { - const text = await span.traceChildAsync('download', () => fetchAssetsWithout304( + const text = await span.traceChildAsync('download', () => fetchAssets( domainListsUrl, mirrors )); diff --git a/Build/lib/parse-filter/filters.ts b/Build/lib/parse-filter/filters.ts index 18b14fe94..132ab06df 100644 --- a/Build/lib/parse-filter/filters.ts +++ b/Build/lib/parse-filter/filters.ts @@ -1,6 +1,6 @@ import picocolors from 'picocolors'; import type { Span } from '../../trace'; -import { fetchAssetsWithout304 } from '../fetch-assets'; +import { fetchAssets } from '../fetch-assets'; import { onBlackFound, onWhiteFound } from './shared'; import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; import { fastNormalizeDomain } from '../normalize-domain'; @@ -27,7 +27,7 @@ export async function processFilterRules( allowThirdParty = false ): Promise<{ white: string[], black: string[] }> { const [white, black, warningMessages] = await parentSpan.traceChild(`process filter rules: ${filterRulesUrl}`).traceAsyncFn(async (span) => { - const text = await span.traceChildAsync('download', () => fetchAssetsWithout304(filterRulesUrl, fallbackUrls)); + const text = await span.traceChildAsync('download', () => fetchAssets(filterRulesUrl, fallbackUrls)); const whitelistDomainSets = new Set(); const blacklistDomainSets = new Set(); diff --git a/Build/lib/parse-filter/hosts.ts b/Build/lib/parse-filter/hosts.ts index 4d8fcec6d..6df00c40e 100644 --- a/Build/lib/parse-filter/hosts.ts +++ b/Build/lib/parse-filter/hosts.ts @@ -1,5 +1,5 @@ import type { Span } from '../../trace'; -import { fetchAssetsWithout304 } from '../fetch-assets'; +import { fetchAssets } from '../fetch-assets'; import { fastNormalizeDomain } from '../normalize-domain'; import { processLine } from '../process-line'; import { onBlackFound } from './shared'; @@ -29,7 +29,7 @@ export function processHosts( hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false ) { return span.traceChildAsync(`process hosts: ${hostsUrl}`, async (span) => { - const text = await span.traceChild('download').traceAsyncFn(() => fetchAssetsWithout304(hostsUrl, mirrors)); + const text = await span.traceChild('download').traceAsyncFn(() => fetchAssets(hostsUrl, mirrors)); const domainSets: string[] = []; diff --git a/package.json b/package.json index 0d128db14..438165a30 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "fast-cidr-tools": "^0.3.1", "fast-fifo": "^1.3.2", "fdir": "^6.4.2", - "foxts": "^1.1.5", + "foxts": "^1.1.6", "hash-wasm": "^4.12.0", "json-stringify-pretty-compact": "^3.0.0", "picocolors": "^1.1.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 40b008757..0788a03b5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,8 +53,8 @@ importers: specifier: ^6.4.2 version: 6.4.2(picomatch@4.0.2) foxts: - specifier: ^1.1.5 - version: 1.1.5 + specifier: ^1.1.6 + version: 1.1.6 hash-wasm: specifier: ^4.12.0 version: 4.12.0 @@ -1134,8 +1134,8 @@ packages: resolution: {integrity: sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==} engines: {node: '>= 6'} - foxts@1.1.5: - resolution: {integrity: sha512-s5SRvB7PTTOl5ZWYE4PrBhHXi6sw/0fLrYDDZThFHcHJCmEMaWmZwM0A5Op9APs6y0nQGrUgroV6WFKNdTbgSQ==} + foxts@1.1.6: + resolution: {integrity: sha512-O2UR/MDLo0w4igcFHwLn2KyXUD84P6bE3U4OpVsxvcYrWLFvvDO8zKLBS/o++tFJTCq7p/3USR48E8/dF2vAAQ==} fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} @@ -2932,7 +2932,7 @@ snapshots: combined-stream: 1.0.8 mime-types: 2.1.35 - foxts@1.1.5: {} + foxts@1.1.6: {} fs-constants@1.0.0: {} From 273b941723ba2507d28c4095e44ca5b9469b70b4 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 00:07:06 +0800 Subject: [PATCH 13/35] Perf(trie): merge two boolean into one mask bit --- Build/lib/trie.ts | 122 +++++++++++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 55 deletions(-) diff --git a/Build/lib/trie.ts b/Build/lib/trie.ts index 31fb9504d..0d93d27b9 100644 --- a/Build/lib/trie.ts +++ b/Build/lib/trie.ts @@ -8,9 +8,13 @@ import { noop } from 'foxts/noop'; import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; import FIFO from 'fast-fifo'; +import { deleteBit, getBit, missingBit, setBit } from 'foxts/bitwise'; + +const START = 1 << 1; +const INCLUDE_ALL_SUBDOMAIN = 1 << 2; + type TrieNode = [ - boolean, /** end */ - boolean, /** includeAllSubdomain (.example.org, ||example.com) */ + flag: number, /** end, includeAllSubdomain (.example.org, ||example.com) */ TrieNode | null, /** parent */ Map, /** children */ Meta /** meta */ @@ -19,24 +23,23 @@ type TrieNode = [ function deepTrieNodeToJSON(node: TrieNode, unpackMeta: ((meta?: any) => string) | undefined) { const obj: Record = {}; - if (node[0]) { - obj['[start]'] = node[0]; - } - obj['[subdomain]'] = node[1]; - if (node[4] != null) { + + obj['[start]'] = getBit(node[0], START); + obj['[subdomain]'] = getBit(node[0], INCLUDE_ALL_SUBDOMAIN); + if (node[3] != null) { if (unpackMeta) { obj['[meta]'] = unpackMeta(node[3]); } else { obj['[meta]'] = node[3]; } } - node[3].forEach((value, key) => { + node[2].forEach((value, key) => { obj[key] = deepTrieNodeToJSON(value, unpackMeta); }); return obj; } -const createNode = (allSubdomain = false, parent: TrieNode | null = null): TrieNode => [false, allSubdomain, parent, new Map(), null] as TrieNode; +const createNode = (parent: TrieNode | null = null): TrieNode => [1, parent, new Map(), null] as TrieNode; export function hostnameToTokens(hostname: string, hostnameFromIndex: number): string[] { const tokens = hostname.split('.'); @@ -124,8 +127,8 @@ abstract class Triebase { parent = node; - if (node[3].has(token)) { - node = node[3].get(token)!; + if (node[2].has(token)) { + node = node[2].get(token)!; } else { return null; } @@ -151,8 +154,8 @@ abstract class Triebase { parent = node; - if (node[3].has(token)) { - node = node[3].get(token)!; + if (node[2].has(token)) { + node = node[2].get(token)!; } else { return null; } @@ -174,7 +177,7 @@ abstract class Triebase { const res = this.walkIntoLeafWithSuffix(suffix, hostnameFromIndex); if (!res) return false; - if (includeAllSubdomain) return res.node[1]; + if (includeAllSubdomain) return getBit(res.node[0], INCLUDE_ALL_SUBDOMAIN); return true; }; @@ -184,7 +187,7 @@ abstract class Triebase { const node = nodeStack.shift()!; const suffix = suffixStack.shift()!; - node[3].forEach((childNode, k) => { + node[2].forEach((childNode, k) => { // Pushing the child node to the stack for next iteration of DFS nodeStack.push(childNode); @@ -201,12 +204,12 @@ abstract class Triebase { const node = nodeStack.shift()!; const suffix = suffixStack.shift()!; - if (node[3].size) { - const keys = Array.from(node[3].keys()).sort(Triebase.compare); + if (node[2].size) { + const keys = Array.from(node[2].keys()).sort(Triebase.compare); for (let i = 0, l = keys.length; i < l; i++) { const key = keys[i]; - const childNode = node[3].get(key)!; + const childNode = node[2].get(key)!; // Pushing the child node to the stack for next iteration of DFS nodeStack.push(childNode); @@ -244,8 +247,8 @@ abstract class Triebase { const suffix = r[1]; // If the node is a sentinel, we push the suffix to the results - if (node[0]) { - onMatches(suffix, node[1], node[4]); + if (getBit(node[0], START)) { + onMatches(suffix, getBit(node[0], INCLUDE_ALL_SUBDOMAIN), node[3]); } } while (nodeStack.length); }; @@ -273,12 +276,12 @@ abstract class Triebase { node = nodeStack.shift()!; const suffix = suffixStack.shift()!; - if (node[3].size) { - const keys = Array.from(node[3].keys()).sort(Triebase.compare); + if (node[2].size) { + const keys = Array.from(node[2].keys()).sort(Triebase.compare); for (let i = 0, l = keys.length; i < l; i++) { const key = keys[i]; - const childNode = node[3].get(key)!; + const childNode = node[2].get(key)!; // Pushing the child node to the stack for next iteration of DFS nodeStack.push(childNode); @@ -287,8 +290,8 @@ abstract class Triebase { } // If the node is a sentinel, we push the suffix to the results - if (node[0]) { - onMatches(suffix, node[1], node[4]); + if (getBit(node[0], START)) { + onMatches(suffix, getBit(node[0], INCLUDE_ALL_SUBDOMAIN), node[3]); } } while (nodeStack.length); }; @@ -302,7 +305,7 @@ abstract class Triebase { // Even if the node size is 1, but the single child is ".", we should retain the branch // Since the "." could be special if it is the leaf-est node - const onlyChild = node[3].size === 0 && !node[2]; + const onlyChild = node[2].size === 0 && !node[1]; if (toPrune != null) { // the top-est branch that could potentially being pruned if (!onlyChild) { @@ -368,15 +371,15 @@ abstract class Triebase { const res = this.getSingleChildLeaf(hostnameToTokens(suffix, 0)); if (res === null) return false; - if (!res.node[0]) return false; + if (missingBit(res.node[0], START)) return false; this.$size--; const { node, toPrune, tokenToPrune } = res; if (tokenToPrune && toPrune) { - toPrune[3].delete(tokenToPrune); + toPrune[2].delete(tokenToPrune); } else { - node[0] = false; + node[0] = deleteBit(node[0], START); } return true; @@ -394,8 +397,8 @@ abstract class Triebase { const res = this.walkIntoLeafWithSuffix(suffix, hostnameFromIndex); if (res === null) return false; - if (!res.node[0]) return false; - if (includeAllSubdomain) return res.node[1]; + if (missingBit(res.node[0], START)) return false; + if (includeAllSubdomain) return getBit(res.node[0], INCLUDE_ALL_SUBDOMAIN); return true; }; @@ -494,19 +497,19 @@ export class HostnameSmolTrie extends Triebase { add(suffix: string, includeAllSubdomain = suffix[0] === '.', meta?: Meta, hostnameFromIndex = suffix[0] === '.' ? 1 : 0): void { let node: TrieNode = this.$root; - let curNodeChildren: Map> = node[3]; + let curNodeChildren: Map> = node[2]; const onToken = (token: string) => { - curNodeChildren = node[3]; + curNodeChildren = node[2]; if (curNodeChildren.has(token)) { node = curNodeChildren.get(token)!; // During the adding of `[start]blog|.skk.moe` and find out that there is a `[start].skk.moe` in the trie, skip adding the rest of the node - if (node[1]) { + if (getBit(node[0], INCLUDE_ALL_SUBDOMAIN)) { return true; } } else { - const newNode = createNode(false, node); + const newNode = createNode(node); curNodeChildren.set(token, newNode); node = newNode; } @@ -527,19 +530,23 @@ export class HostnameSmolTrie extends Triebase { // (/** parent */ node[2]!)[0] = false; // Removing the rest of the parent's child nodes - node[3].clear(); + node[2].clear(); // The SENTINEL of this node will be set to true at the end of the function, so we don't need to set it here // we can use else-if here, because the children is now empty, we don't need to check the leading "." - } else if (node[1]) { + } else if (getBit(node[0], INCLUDE_ALL_SUBDOMAIN)) { // Trying to add `example.com` when there is already a `.example.com` in the trie // No need to increment size and set SENTINEL to true (skip this "new" item) return; } - node[0] = true; - node[1] = includeAllSubdomain; - node[4] = meta!; + node[0] = setBit(node[0], START); + if (includeAllSubdomain) { + node[0] = setBit(node[0], INCLUDE_ALL_SUBDOMAIN); + } else { + node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN); + } + node[3] = meta!; } public whitelist(suffix: string, includeAllSubdomain = suffix[0] === '.', hostnameFromIndex = suffix[0] === '.' ? 1 : 0) { @@ -553,22 +560,22 @@ export class HostnameSmolTrie extends Triebase { // Trying to whitelist `[start].sub.example.com` where there might already be a `[start]blog.sub.example.com` in the trie if (includeAllSubdomain) { // If there is a `[start]sub.example.com` here, remove it - node[0] = false; - node[1] = false; + node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN); + node[0] = deleteBit(node[0], START); // Removing all the child nodes by empty the children - node[3].clear(); + node[2].clear(); } else { // Trying to whitelist `example.com` when there is already a `.example.com` in the trie - node[1] = false; + node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN); } // return early if not found - if (!node[0]) return; + if (missingBit(node[0], START)) return; if (tokenToPrune && toPrune) { - toPrune[3].delete(tokenToPrune); + toPrune[2].delete(tokenToPrune); } else { - node[0] = false; + node[0] = deleteBit(node[0], START); } }; } @@ -582,11 +589,11 @@ export class HostnameTrie extends Triebase { let node: TrieNode = this.$root; const onToken = (token: string) => { - if (node[3].has(token)) { - node = node[3].get(token)!; + if (node[2].has(token)) { + node = node[2].get(token)!; } else { - const newNode = createNode(false, node); - node[3].set(token, newNode); + const newNode = createNode(node); + node[2].set(token, newNode); node = newNode; } @@ -599,14 +606,19 @@ export class HostnameTrie extends Triebase { } // if same entry has been added before, skip - if (node[0]) { + if (getBit(node[0], START)) { return; } this.$size++; - node[0] = true; - node[1] = includeAllSubdomain; - node[4] = meta!; + + node[0] = setBit(node[0], START); + if (includeAllSubdomain) { + node[0] = setBit(node[0], INCLUDE_ALL_SUBDOMAIN); + } else { + node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN); + } + node[3] = meta!; } } From 7683775dee6f973f6f45fc4a9888057b7e353f37 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 00:20:23 +0800 Subject: [PATCH 14/35] CI: run in tmpfs --- .github/workflows/main.yml | 8 +++-- Build/constants/dir.ts | 8 ++++- Build/download-previous-build.ts | 3 +- Build/index.ts | 8 ++++- Build/lib/misc.ts | 11 +++++++ package.json | 1 + pnpm-lock.yaml | 51 ++++++++++++++++++++++++++++++++ 7 files changed, 84 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 136339b16..6a45341ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,15 +56,16 @@ jobs: ${{ runner.os }}-v3- - run: pnpm install - run: pnpm run build + id: build - name: Pre-deploy check # If the public directory doesn't exist, the build should fail. # If the public directory is empty, the build should fail. run: | - if [ ! -d public ]; then + if [ ! -d ${{ steps.build.outputs.public_dir }} ]; then echo "public directory not found" exit 1 fi - if [ ! "$(ls -A public)" ]; then + if [ ! "$(ls -A ${{ steps.build.outputs.public_dir }})" ]; then echo "public directory is empty" exit 1 fi @@ -72,10 +73,11 @@ jobs: echo ".BUILD_FINISHED not found" exit 1 fi + echo "public directory is ready: ${{ steps.build.outputs.public_dir }}" - uses: actions/upload-artifact@v4 with: name: build-artifact-${{ github. ref_name }} - path: public + path: ${{ steps.build.outputs.public_dir }} if-no-files-found: error retention-days: 1 compression-level: 4 diff --git a/Build/constants/dir.ts b/Build/constants/dir.ts index 956154267..335a66792 100644 --- a/Build/constants/dir.ts +++ b/Build/constants/dir.ts @@ -1,4 +1,7 @@ import path from 'node:path'; +import os from 'node:os'; +import fs from 'node:fs'; +import { isCI } from 'ci-info'; export const ROOT_DIR = path.resolve(__dirname, '../..'); @@ -6,7 +9,10 @@ export const CACHE_DIR = path.resolve(ROOT_DIR, '.cache'); export const SOURCE_DIR = path.join(ROOT_DIR, 'Source'); -export const PUBLIC_DIR = path.resolve(ROOT_DIR, 'public'); +export const PUBLIC_DIR = isCI + ? fs.mkdtempSync(path.join(os.tmpdir(), 'sukkaw-surge-public-')) + : path.resolve(ROOT_DIR, 'public'); + export const OUTPUT_SURGE_DIR = path.join(PUBLIC_DIR, 'List'); export const OUTPUT_CLASH_DIR = path.resolve(PUBLIC_DIR, 'Clash'); export const OUTPUT_SINGBOX_DIR = path.resolve(PUBLIC_DIR, 'sing-box'); diff --git a/Build/download-previous-build.ts b/Build/download-previous-build.ts index 9484d0091..0a5ae7be3 100644 --- a/Build/download-previous-build.ts +++ b/Build/download-previous-build.ts @@ -9,12 +9,13 @@ import undici from 'undici'; import picocolors from 'picocolors'; import { PUBLIC_DIR } from './constants/dir'; import { requestWithLog } from './lib/fetch-retry'; +import { isDirectoryEmptySync } from './lib/misc'; const GITHUB_CODELOAD_URL = 'https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master'; const GITLAB_CODELOAD_URL = 'https://gitlab.com/SukkaW/ruleset.skk.moe/-/archive/master/ruleset.skk.moe-master.tar.gz'; export const downloadPreviousBuild = task(require.main === module, __filename)(async (span) => { - if (fs.existsSync(PUBLIC_DIR)) { + if (fs.existsSync(PUBLIC_DIR) && !isDirectoryEmptySync(PUBLIC_DIR)) { console.log(picocolors.blue('Public directory exists, skip downloading previous build')); return; } diff --git a/Build/index.ts b/Build/index.ts index 40c95bdb5..f0dc14d12 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -29,7 +29,9 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules'; import { createSpan, printTraceResult, whyIsNodeRunning } from './trace'; import { buildDeprecateFiles } from './build-deprecate-files'; import path from 'node:path'; -import { ROOT_DIR } from './constants/dir'; +import { PUBLIC_DIR, ROOT_DIR } from './constants/dir'; + +import { setOutput } from '@actions/core'; process.on('uncaughtException', (error) => { console.error('Uncaught exception:', error); @@ -106,6 +108,10 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); // write a file to demonstrate that the build is finished fs.writeFileSync(buildFinishedLock, 'BUILD_FINISHED\n'); + if (process.env.GITHUB_OUTPUT) { + setOutput('public_dir', PUBLIC_DIR); + } + // Finish the build to avoid leaking timer/fetch ref await whyIsNodeRunning(); process.exit(0); diff --git a/Build/lib/misc.ts b/Build/lib/misc.ts index bf44fdabb..2648de336 100644 --- a/Build/lib/misc.ts +++ b/Build/lib/misc.ts @@ -1,5 +1,6 @@ import { dirname } from 'node:path'; import fs from 'node:fs'; +import type { PathLike } from 'node:fs'; import fsp from 'node:fs/promises'; export function fastStringCompare(a: string, b: string) { @@ -79,3 +80,13 @@ export function withBannerArray(title: string, description: string[] | readonly '################## EOF ##################' ]; }; + +export function isDirectoryEmptySync(path: PathLike) { + const directoryHandle = fs.opendirSync(path); + + try { + return directoryHandle.readSync() === null; + } finally { + directoryHandle.closeSync(); + } +} diff --git a/package.json b/package.json index 438165a30..48981f221 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "author": "", "license": "ISC", "dependencies": { + "@actions/core": "^1.11.1", "@ghostery/adblocker": "^2.3.1", "@henrygd/queue": "^1.0.7", "async-retry": "^1.3.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0788a03b5..80fe7279e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -19,6 +19,9 @@ importers: .: dependencies: + '@actions/core': + specifier: ^1.11.1 + version: 1.11.1 '@ghostery/adblocker': specifier: ^2.3.1 version: 2.3.1 @@ -161,6 +164,18 @@ importers: packages: + '@actions/core@1.11.1': + resolution: {integrity: sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==} + + '@actions/exec@1.1.1': + resolution: {integrity: sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==} + + '@actions/http-client@2.2.3': + resolution: {integrity: sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==} + + '@actions/io@1.1.3': + resolution: {integrity: sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==} + '@antfu/utils@0.7.10': resolution: {integrity: sha512-+562v9k4aI80m1+VuMHehNJWLOFjBnXn3tdOitzD0il5b7smkSBal4+a3oKiQTbrwMmN/TBUMDvbdoWDehgOww==} @@ -234,6 +249,10 @@ packages: resolution: {integrity: sha512-lB05FkqEdUg2AA0xEbUz0SnkXT1LcCTa438W4IWTUh4hdOnVbQyOJ81OrDXsJk/LSiJHubgGEFoR5EHq1NsH1A==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@fastify/busboy@2.1.1': + resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} + engines: {node: '>=14'} + '@ghostery/adblocker-content@2.3.1': resolution: {integrity: sha512-8ZTY1sEE218b0EMk3Q9fv/cWwUxunSCyBOaKuviEmJY5EyvPa1VbGPuTq/Qdvo1kduD8nLEzCwgWhcT3F3TT0Q==} @@ -1726,6 +1745,10 @@ packages: tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + tunnel@0.0.6: + resolution: {integrity: sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==} + engines: {node: '>=0.6.11 <=0.7.0 || >=0.7.3'} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -1750,6 +1773,10 @@ packages: undici-types@6.20.0: resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} + undici@5.28.5: + resolution: {integrity: sha512-zICwjrDrcrUE0pyyJc1I2QzBkLM8FINsgOrt6WjA+BgajVq9Nxu2PbFFXUrAggLfDXlZGZBVZYw7WNV5KiBiBA==} + engines: {node: '>=14.0'} + undici@7.2.1: resolution: {integrity: sha512-U2k0XHLJfaciARRxDcqTk2AZQsGXerHzdvfCZcy1hNhSf5KCAF4jIQQxL+apQviOekhRFPqED6Of5/+LcUSLzQ==} engines: {node: '>=20.18.1'} @@ -1843,6 +1870,22 @@ packages: snapshots: + '@actions/core@1.11.1': + dependencies: + '@actions/exec': 1.1.1 + '@actions/http-client': 2.2.3 + + '@actions/exec@1.1.1': + dependencies: + '@actions/io': 1.1.3 + + '@actions/http-client@2.2.3': + dependencies: + tunnel: 0.0.6 + undici: 5.28.5(patch_hash=eyidnukwfhrd7exzoydz2h5cfq) + + '@actions/io@1.1.3': {} + '@antfu/utils@0.7.10': {} '@babel/code-frame@7.26.2': @@ -1942,6 +1985,8 @@ snapshots: '@eslint/core': 0.10.0 levn: 0.4.1 + '@fastify/busboy@2.1.1': {} + '@ghostery/adblocker-content@2.3.1': dependencies: '@ghostery/adblocker-extended-selectors': 2.3.1 @@ -3536,6 +3581,8 @@ snapshots: dependencies: safe-buffer: 5.2.1 + tunnel@0.0.6: {} + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 @@ -3559,6 +3606,10 @@ snapshots: undici-types@6.20.0: {} + undici@5.28.5(patch_hash=eyidnukwfhrd7exzoydz2h5cfq): + dependencies: + '@fastify/busboy': 2.1.1 + undici@7.2.1(patch_hash=eyidnukwfhrd7exzoydz2h5cfq): {} unique-filename@4.0.0: From 264e30726a4cd9d95cf9b3caf603ed40fc50b1c2 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 00:59:37 +0800 Subject: [PATCH 15/35] Perf: make `normalizeDomain` faster --- Build/constants/loose-tldts-opt.ts | 11 +++++++---- Build/lib/normalize-domain.ts | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Build/constants/loose-tldts-opt.ts b/Build/constants/loose-tldts-opt.ts index 19c4959f2..e51021c51 100644 --- a/Build/constants/loose-tldts-opt.ts +++ b/Build/constants/loose-tldts-opt.ts @@ -1,6 +1,6 @@ import type * as tldts from 'tldts'; -export const looseTldtsOpt: Parameters[1] = { +export const looseTldtsOpt: NonNullable[1]> = { allowPrivateDomains: false, extractHostname: false, validateHostname: false, @@ -8,12 +8,15 @@ export const looseTldtsOpt: Parameters[1] = { mixedInputs: false }; -export const loosTldOptWithPrivateDomains: Parameters[1] = { +export const loosTldOptWithPrivateDomains: NonNullable[1]> = { ...looseTldtsOpt, allowPrivateDomains: true }; -export const normalizeTldtsOpt: Parameters[1] = { +export const normalizeTldtsOpt: NonNullable[1]> = { allowPrivateDomains: true, - detectIp: true + // in normalizeDomain, we only care if it contains IP, we don't care if we need to extract it + // by setting detectIp to false and manually check ip outside tldts.parse, we can skip the tldts + // inner "extractHostname" call + detectIp: false }; diff --git a/Build/lib/normalize-domain.ts b/Build/lib/normalize-domain.ts index 093d415a3..8837ae459 100644 --- a/Build/lib/normalize-domain.ts +++ b/Build/lib/normalize-domain.ts @@ -3,14 +3,21 @@ // import tldts from 'tldts-experimental'; import tldts from 'tldts'; import { normalizeTldtsOpt } from '../constants/loose-tldts-opt'; +import { isProbablyIpv4, isProbablyIpv6 } from 'foxts/is-probably-ip'; type TldTsParsed = ReturnType; /** * Skipped the input non-empty check, the `domain` should not be empty. */ -export function fastNormalizeDomain(domain: string, parsed: TldTsParsed = tldts.parse(domain, normalizeTldtsOpt)) { - if (parsed.isIp) return null; +export function fastNormalizeDomain(domain: string, parsed: TldTsParsed | null = null) { + // We don't want tldts to call its own "extractHostname" on ip, bail out ip first. + // Now ip has been bailed out, we can safely set normalizeTldtsOpt.detectIp to false. + if (isProbablyIpv4(domain) || isProbablyIpv6(domain)) { + return null; + } + + parsed ??= tldts.parse(domain, normalizeTldtsOpt); // Private invalid domain (things like .tor, .dn42, etc) if (!parsed.isIcann && !parsed.isPrivate) return null; @@ -20,9 +27,11 @@ export function fastNormalizeDomain(domain: string, parsed: TldTsParsed = tldts. export function normalizeDomain(domain: string, parsed: TldTsParsed | null = null) { if (domain.length === 0) return null; - parsed ??= tldts.parse(domain, normalizeTldtsOpt); + if (isProbablyIpv4(domain) || isProbablyIpv6(domain)) { + return null; + } - if (parsed.isIp) return null; + parsed ??= tldts.parse(domain, normalizeTldtsOpt); // Private invalid domain (things like .tor, .dn42, etc) if (!parsed.isIcann && !parsed.isPrivate) return null; From af034b1abbf9b3a63049b8731f41d71d4480a1aa Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 01:22:52 +0800 Subject: [PATCH 16/35] Update Reject Hosts --- Source/non_ip/domestic.conf | 1 - Source/non_ip/reject.conf | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Source/non_ip/domestic.conf b/Source/non_ip/domestic.conf index 257d3c199..452fdcafe 100644 --- a/Source/non_ip/domestic.conf +++ b/Source/non_ip/domestic.conf @@ -530,7 +530,6 @@ DOMAIN-SUFFIX,jiemian.com DOMAIN-SUFFIX,jiguang.cn DOMAIN-SUFFIX,jb51.net DOMAIN-SUFFIX,jpush.cn -DOMAIN-SUFFIX,juejin.cn DOMAIN-SUFFIX,kaiyanapp.com DOMAIN-SUFFIX,kdslife.com DOMAIN-SUFFIX,keepcdn.com diff --git a/Source/non_ip/reject.conf b/Source/non_ip/reject.conf index 879fceb38..fa02734a8 100644 --- a/Source/non_ip/reject.conf +++ b/Source/non_ip/reject.conf @@ -120,6 +120,13 @@ DOMAIN-SUFFIX,notify6.com DOMAIN-KEYWORD,wxsnsdy DOMAIN-WILDCARD,adv0*.msa.cdn.mediaset.net +# >> Juejin +# 掘金就是一个垃圾平台，一堆三流程序员在上面自我意淫，掘金平台也在鼓励洗稿他人文章的行为 +# https://juejin.cn/post/7247028435339591740 洗稿我的 https://blog.skk.moe/post/why-you-should-not-fetch-data-directly-in-use-effect/ +# 强制拉黑掘金，永不解封 +.juejin.im +.juejin.cn + # >> Google # DOMAIN-KEYWORD,adsense # unblocks adsense.google.com DOMAIN-KEYWORD,adwords From c150b3ed8458c91d034aa87c6156614b7a474cf8 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 01:29:25 +0800 Subject: [PATCH 17/35] Revert(#53): no dedicate stream_biliintl --- Build/build-stream-service.ts | 5 ++--- Source/stream.ts | 8 ++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Build/build-stream-service.ts b/Build/build-stream-service.ts index b7b2217bc..f88256bea 100644 --- a/Build/build-stream-service.ts +++ b/Build/build-stream-service.ts @@ -2,7 +2,7 @@ import type { Span } from './trace'; import { task } from './trace'; -import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR, BILI_INTL } from '../Source/stream'; +import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream'; import { SHARED_DESCRIPTION } from './constants/description'; import { RulesetOutput } from './lib/create-file'; @@ -42,7 +42,6 @@ export const buildStreamService = task(require.main === module, __filename)(asyn createRulesetForStreamService(span, 'stream_tw', 'Taiwan', TW), createRulesetForStreamService(span, 'stream_jp', 'Japan', JP), // createRulesetForStreamService('stream_au', 'Oceania', AU), - createRulesetForStreamService(span, 'stream_kr', 'Korean', KR), + createRulesetForStreamService(span, 'stream_kr', 'Korean', KR) // createRulesetForStreamService('stream_south_east_asia', 'South East Asia', SOUTH_EAST_ASIA) - createRulesetForStreamService(span, 'stream_biliintl', 'Bilibili International', BILI_INTL) ])); diff --git a/Source/stream.ts b/Source/stream.ts index 19e365ec2..534c0cf36 100644 --- a/Source/stream.ts +++ b/Source/stream.ts @@ -879,8 +879,8 @@ export const HK: StreamService[] = [ NOW_E, VIUTV, MYTV_SUPER, - HBO_ASIA - // BILIBILI_INTL + HBO_ASIA, + BILIBILI_INTL ]; export const TW: StreamService[] = [ @@ -956,7 +956,3 @@ export const SOUTH_EAST_ASIA = [ // TV360 VN // B-Global VN ]; - -export const BILI_INTL = [ - BILIBILI_INTL -]; From ffa4e03d0868efdb1bc741563f194e7517834fa4 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 01:35:25 +0800 Subject: [PATCH 18/35] Perf: make `fileEqual` faster --- Build/lib/rules/base.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index d2e660507..1f74712ab 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -401,6 +401,10 @@ export async function fileEqual(linesA: string[], source: AsyncIterable const firstCharA = lineA.charCodeAt(0); const firstCharB = lineB.charCodeAt(0); + if (firstCharA !== firstCharB) { + return false; + } + if (firstCharA === 35 /* # */ && firstCharB === 35 /* # */) { continue; } @@ -410,13 +414,9 @@ export async function fileEqual(linesA: string[], source: AsyncIterable } if ( - firstCharA === 47 /* / */ - && firstCharB === 47 /* / */ - - && lineA[1] === '/' - && lineB[1] === '/' - && lineA[3] === '#' - && lineB[3] === '#' + firstCharA === 47 /* / */ && firstCharB === 47 /* / */ + && lineA[1] === '/' && lineB[1] === '/' + && lineA[3] === '#' && lineB[3] === '#' ) { continue; } From c098849b0a877ba6767923b362637f0766c9648d Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 02:30:42 +0800 Subject: [PATCH 19/35] Housekeeping --- package.json | 18 ++++----- pnpm-lock.yaml | 106 ++++++++++++++++++++++++------------------------- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/package.json b/package.json index 48981f221..e0414330f 100644 --- a/package.json +++ b/package.json @@ -24,23 +24,23 @@ "@ghostery/adblocker": "^2.3.1", "@henrygd/queue": "^1.0.7", "async-retry": "^1.3.3", - "better-sqlite3": "^11.7.2", + "better-sqlite3": "^11.8.0", "cacache": "^19.0.1", "ci-info": "^4.1.0", "csv-parse": "^5.6.0", "dns2": "^2.1.0", "fast-cidr-tools": "^0.3.1", "fast-fifo": "^1.3.2", - "fdir": "^6.4.2", + "fdir": "^6.4.3", "foxts": "^1.1.6", "hash-wasm": "^4.12.0", "json-stringify-pretty-compact": "^3.0.0", "picocolors": "^1.1.1", - "tar-fs": "^3.0.6", + "tar-fs": "3.0.6", "tinyexec": "^0.3.2", - "tldts": "^6.1.71", - "tldts-experimental": "^6.1.71", - "undici": "^7.2.1", + "tldts": "^6.1.72", + "tldts-experimental": "^6.1.72", + "undici": "^7.2.3", "undici-cache-store-better-sqlite3": "^0.1.1", "whoiser": "^1.18.0", "why-is-node-running": "^3.2.2", @@ -58,7 +58,7 @@ "@types/dns2": "^2.0.9", "@types/fast-fifo": "^1.3.0", "@types/mocha": "^10.0.10", - "@types/node": "^22.10.5", + "@types/node": "^22.10.7", "@types/node-fetch": "^2.6.12", "@types/tar-fs": "^2.0.4", "@types/tar-stream": "^3.1.3", @@ -66,11 +66,11 @@ "eslint-config-sukka": "^6.13.0", "eslint-formatter-sukka": "^6.13.0", "expect": "^29.7.0", - "mitata": "^1.0.28", + "mitata": "^1.0.30", "mocha": "^11.0.1", "typescript": "^5.7.3" }, - "packageManager": "pnpm@9.15.3", + "packageManager": "pnpm@9.15.4", "pnpm": { "patchedDependencies": { "undici": "patches/undici.patch", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 80fe7279e..35e01d874 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,8 +32,8 @@ importers: specifier: ^1.3.3 version: 1.3.3 better-sqlite3: - specifier: ^11.7.2 - version: 11.7.2 + specifier: ^11.8.0 + version: 11.8.0 cacache: specifier: ^19.0.1 version: 19.0.1 @@ -53,8 +53,8 @@ importers: specifier: ^1.3.2 version: 1.3.2 fdir: - specifier: ^6.4.2 - version: 6.4.2(picomatch@4.0.2) + specifier: ^6.4.3 + version: 6.4.3(picomatch@4.0.2) foxts: specifier: ^1.1.6 version: 1.1.6 @@ -74,17 +74,17 @@ importers: specifier: ^0.3.2 version: 0.3.2 tldts: - specifier: ^6.1.71 - version: 6.1.71 + specifier: ^6.1.72 + version: 6.1.72 tldts-experimental: - specifier: ^6.1.71 - version: 6.1.71 + specifier: ^6.1.72 + version: 6.1.72 undici: - specifier: ^7.2.1 - version: 7.2.1(patch_hash=eyidnukwfhrd7exzoydz2h5cfq) + specifier: ^7.2.3 + version: 7.2.3(patch_hash=eyidnukwfhrd7exzoydz2h5cfq) undici-cache-store-better-sqlite3: specifier: ^0.1.1 - version: 0.1.1(undici@7.2.1(patch_hash=eyidnukwfhrd7exzoydz2h5cfq)) + version: 0.1.1(undici@7.2.3(patch_hash=eyidnukwfhrd7exzoydz2h5cfq)) whoiser: specifier: ^1.18.0 version: 1.18.0(patch_hash=jvdx7w7gioupgqcow7nnybv6ye) @@ -129,8 +129,8 @@ importers: specifier: ^10.0.10 version: 10.0.10 '@types/node': - specifier: ^22.10.5 - version: 22.10.5 + specifier: ^22.10.7 + version: 22.10.7 '@types/node-fetch': specifier: ^2.6.12 version: 2.6.12 @@ -153,8 +153,8 @@ importers: specifier: ^29.7.0 version: 29.7.0 mitata: - specifier: ^1.0.28 - version: 1.0.28 + specifier: ^1.0.30 + version: 1.0.30 mocha: specifier: ^11.0.1 version: 11.0.1 @@ -564,8 +564,8 @@ packages: '@types/node-fetch@2.6.12': resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==} - '@types/node@22.10.5': - resolution: {integrity: sha512-F8Q+SeGimwOo86fiovQh8qiXfFEh2/ocYv7tU5pJ3EXMSSxk1Joj5wefpFK2fHTf/N6HKGSxIDBT9f3gCxXPkQ==} + '@types/node@22.10.7': + resolution: {integrity: sha512-V09KvXxFiutGp6B7XkpaDXlNadZxrzajcY50EuoLIpQ6WWYCSvf19lVIazzfIzQvhUN2HjX12spLojTnhuKlGg==} '@types/retry@0.12.5': resolution: {integrity: sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==} @@ -737,8 +737,8 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - better-sqlite3@11.7.2: - resolution: {integrity: sha512-10a57cHVDmfNQS4jrZ9AH2t+2ekzYh5Rhbcnb4ytpmYweoLdogDmyTt5D+hLiY9b44Mx9foowb/4iXBTO2yP3Q==} + better-sqlite3@11.8.0: + resolution: {integrity: sha512-aKv9s2dir7bsEX5RIjL9HHWB9uQ+f6Vch5B4qmeAOop4Y9OYHX+PNKLr+mpv6+d8L/ZYh4l7H8zPuVMbWkVMLw==} binary-extensions@2.3.0: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} @@ -1111,8 +1111,8 @@ packages: fastq@1.17.1: resolution: {integrity: sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==} - fdir@6.4.2: - resolution: {integrity: sha512-KnhMXsKSPZlAhp7+IjUkRZKPb4fUyccpDrdFXbi4QL1qkmFh9kVY09Yox+n4MaOb3lHZ1Tv829C3oaaXoMYPDQ==} + fdir@6.4.3: + resolution: {integrity: sha512-PMXmW2y1hDDfTSRc9gaXIuCCRpuoz3Kaz8cUelp3smouvfT632ozg2vrT6lJsHKKOF59YLbOGfAWGUcKEfRMQw==} peerDependencies: picomatch: ^3 || ^4 peerDependenciesMeta: @@ -1410,8 +1410,8 @@ packages: resolution: {integrity: sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==} engines: {node: '>= 18'} - mitata@1.0.28: - resolution: {integrity: sha512-5zKXgSxfzEeMVEwvTPeduMy39RB0UuAx/fgbok9knwABM/zKOr5E7f9zU5CvK30LDXtTjRs0EfrNRhaIWMWT3Q==} + mitata@1.0.30: + resolution: {integrity: sha512-zefhVHdXznlD92aojkn7sZ+ZUzvbG8vmFvLllIHZZoUwGlO+YMqMbnA3tcutwe/yAVdkL5/3z65isEfvdomoJQ==} mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} @@ -1719,14 +1719,14 @@ packages: tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} - tldts-core@6.1.71: - resolution: {integrity: sha512-LRbChn2YRpic1KxY+ldL1pGXN/oVvKfCVufwfVzEQdFYNo39uF7AJa/WXdo+gYO7PTvdfkCPCed6Hkvz/kR7jg==} + tldts-core@6.1.72: + resolution: {integrity: sha512-FW3H9aCaGTJ8l8RVCR3EX8GxsxDbQXuwetwwgXA2chYdsX+NY1ytCBl61narjjehWmCw92tc1AxlcY3668CU8g==} - tldts-experimental@6.1.71: - resolution: {integrity: sha512-78lfP/3fRJ3HoCT5JSLOLj5ElHiWCAyglYNzjkFqBO7ykLZYst2u2jM1igSHWV0J2GFfOplApeDsfTF+XACrlA==} + tldts-experimental@6.1.72: + resolution: {integrity: sha512-mfPL+Pzn3nJ0JeI9AHuO4l0NbxldZhpWUYokb8HdK8gbZ2k0/qEqs5E/FqcOjVr4vzTZpbRtiwMPXv77VwXpyQ==} - tldts@6.1.71: - resolution: {integrity: sha512-LQIHmHnuzfZgZWAf2HzL83TIIrD8NhhI0DVxqo9/FdOd4ilec+NTNZOlDZf7EwrTNoutccbsHjvWHYXLAtvxjw==} + tldts@6.1.72: + resolution: {integrity: sha512-QNtgIqSUb9o2CoUjX9T5TwaIvUUJFU1+12PJkgt42DFV2yf9J6549yTF2uGloQsJ/JOC8X+gIB81ind97hRiIQ==} hasBin: true to-regex-range@5.0.1: @@ -1777,8 +1777,8 @@ packages: resolution: {integrity: sha512-zICwjrDrcrUE0pyyJc1I2QzBkLM8FINsgOrt6WjA+BgajVq9Nxu2PbFFXUrAggLfDXlZGZBVZYw7WNV5KiBiBA==} engines: {node: '>=14.0'} - undici@7.2.1: - resolution: {integrity: sha512-U2k0XHLJfaciARRxDcqTk2AZQsGXerHzdvfCZcy1hNhSf5KCAF4jIQQxL+apQviOekhRFPqED6Of5/+LcUSLzQ==} + undici@7.2.3: + resolution: {integrity: sha512-2oSLHaDalSt2/O/wHA9M+/ZPAOcU2yrSP/cdBYJ+YxZskiPYDSqHbysLSlD7gq3JMqOoJI5O31RVU3BxX/MnAA==} engines: {node: '>=20.18.1'} unique-filename@4.0.0: @@ -2000,7 +2000,7 @@ snapshots: '@remusao/guess-url-type': 1.3.0 '@remusao/small': 1.3.0 '@remusao/smaz': 1.10.0 - tldts-experimental: 6.1.71 + tldts-experimental: 6.1.72 '@henrygd/queue@1.0.7': {} @@ -2043,7 +2043,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 22.10.5 + '@types/node': 22.10.7 '@types/yargs': 17.0.33 chalk: 4.1.2 @@ -2239,15 +2239,15 @@ snapshots: '@types/better-sqlite3@7.6.12': dependencies: - '@types/node': 22.10.5 + '@types/node': 22.10.7 '@types/cacache@17.0.2': dependencies: - '@types/node': 22.10.5 + '@types/node': 22.10.7 '@types/dns2@2.0.9': dependencies: - '@types/node': 22.10.5 + '@types/node': 22.10.7 '@types/doctrine@0.0.9': {} @@ -2276,10 +2276,10 @@ snapshots: '@types/node-fetch@2.6.12': dependencies: - '@types/node': 22.10.5 + '@types/node': 22.10.7 form-data: 4.0.1 - '@types/node@22.10.5': + '@types/node@22.10.7': dependencies: undici-types: 6.20.0 @@ -2289,12 +2289,12 @@ snapshots: '@types/tar-fs@2.0.4': dependencies: - '@types/node': 22.10.5 + '@types/node': 22.10.7 '@types/tar-stream': 3.1.3 '@types/tar-stream@3.1.3': dependencies: - '@types/node': 22.10.5 + '@types/node': 22.10.7 '@types/yargs-parser@21.0.3': {} @@ -2487,7 +2487,7 @@ snapshots: base64-js@1.5.1: {} - better-sqlite3@11.7.2: + better-sqlite3@11.8.0: dependencies: bindings: 1.5.0 prebuild-install: 7.1.2 @@ -2938,7 +2938,7 @@ snapshots: dependencies: reusify: 1.0.4 - fdir@6.4.2(picomatch@4.0.2): + fdir@6.4.3(picomatch@4.0.2): optionalDependencies: picomatch: 4.0.2 @@ -3113,7 +3113,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 22.10.5 + '@types/node': 22.10.7 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -3216,7 +3216,7 @@ snapshots: minipass: 7.1.2 rimraf: 5.0.10 - mitata@1.0.28: {} + mitata@1.0.30: {} mkdirp-classic@0.5.3: {} @@ -3557,15 +3557,15 @@ snapshots: tinyexec@0.3.2: {} - tldts-core@6.1.71: {} + tldts-core@6.1.72: {} - tldts-experimental@6.1.71: + tldts-experimental@6.1.72: dependencies: - tldts-core: 6.1.71 + tldts-core: 6.1.72 - tldts@6.1.71: + tldts@6.1.72: dependencies: - tldts-core: 6.1.71 + tldts-core: 6.1.72 to-regex-range@5.0.1: dependencies: @@ -3599,10 +3599,10 @@ snapshots: typescript@5.7.3: {} - undici-cache-store-better-sqlite3@0.1.1(undici@7.2.1(patch_hash=eyidnukwfhrd7exzoydz2h5cfq)): + undici-cache-store-better-sqlite3@0.1.1(undici@7.2.3(patch_hash=eyidnukwfhrd7exzoydz2h5cfq)): dependencies: - better-sqlite3: 11.7.2 - undici: 7.2.1(patch_hash=eyidnukwfhrd7exzoydz2h5cfq) + better-sqlite3: 11.8.0 + undici: 7.2.3(patch_hash=eyidnukwfhrd7exzoydz2h5cfq) undici-types@6.20.0: {} @@ -3610,7 +3610,7 @@ snapshots: dependencies: '@fastify/busboy': 2.1.1 - undici@7.2.1(patch_hash=eyidnukwfhrd7exzoydz2h5cfq): {} + undici@7.2.3(patch_hash=eyidnukwfhrd7exzoydz2h5cfq): {} unique-filename@4.0.0: dependencies: From 408328ee489e5048dd2003502618744ba3373984 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 02:30:45 +0800 Subject: [PATCH 20/35] Fix typo --- Source/non_ip/reject.conf | 4 ++-- pnpm-lock.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/non_ip/reject.conf b/Source/non_ip/reject.conf index fa02734a8..2b83900c4 100644 --- a/Source/non_ip/reject.conf +++ b/Source/non_ip/reject.conf @@ -124,8 +124,8 @@ DOMAIN-WILDCARD,adv0*.msa.cdn.mediaset.net # 掘金就是一个垃圾平台，一堆三流程序员在上面自我意淫，掘金平台也在鼓励洗稿他人文章的行为 # https://juejin.cn/post/7247028435339591740 洗稿我的 https://blog.skk.moe/post/why-you-should-not-fetch-data-directly-in-use-effect/ # 强制拉黑掘金，永不解封 -.juejin.im -.juejin.cn +DOMAIN-SUFFIX,juejin.im +DOMAIN-SUFFIX,juejin.cn # >> Google # DOMAIN-KEYWORD,adsense # unblocks adsense.google.com diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 35e01d874..e0cce3f13 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,7 +68,7 @@ importers: specifier: ^1.1.1 version: 1.1.1 tar-fs: - specifier: ^3.0.6 + specifier: 3.0.6 version: 3.0.6 tinyexec: specifier: ^0.3.2 From 42428e14de2ce39a4777f3edd897961dd2d14830 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 02:46:44 +0800 Subject: [PATCH 21/35] CI: run on ramdisk --- .github/workflows/main.yml | 1 + Build/constants/dir.ts | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6a45341ca..2c13b0efb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,6 +16,7 @@ jobs: runs-on: ubuntu-24.04-arm steps: + - run: df -h # - name: Tune GitHub-hosted runner network # # https://github.com/actions/runner-images/issues/1187 # uses: smorimoto/tune-github-hosted-runner-network@v1 diff --git a/Build/constants/dir.ts b/Build/constants/dir.ts index 335a66792..a22ed7225 100644 --- a/Build/constants/dir.ts +++ b/Build/constants/dir.ts @@ -1,5 +1,4 @@ import path from 'node:path'; -import os from 'node:os'; import fs from 'node:fs'; import { isCI } from 'ci-info'; @@ -10,7 +9,7 @@ export const CACHE_DIR = path.resolve(ROOT_DIR, '.cache'); export const SOURCE_DIR = path.join(ROOT_DIR, 'Source'); export const PUBLIC_DIR = isCI - ? fs.mkdtempSync(path.join(os.tmpdir(), 'sukkaw-surge-public-')) + ? fs.mkdtempSync('/dev/shm/sukkaw-surge-public-') : path.resolve(ROOT_DIR, 'public'); export const OUTPUT_SURGE_DIR = path.join(PUBLIC_DIR, 'List'); From 96cf0fc1f9995a07f177f40ff303fbdc1edc759d Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 02:59:32 +0800 Subject: [PATCH 22/35] Perf: preload phishing data source --- Build/lib/get-phishing-domains.ts | 15 +++++++++------ Build/lib/parse-filter/domainlists.ts | 18 ++++++++++++++++++ Build/lib/parse-filter/hosts.ts | 20 ++++++++++++++++++++ 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/Build/lib/get-phishing-domains.ts b/Build/lib/get-phishing-domains.ts index f25b37643..99a073a93 100644 --- a/Build/lib/get-phishing-domains.ts +++ b/Build/lib/get-phishing-domains.ts @@ -1,5 +1,5 @@ -import { processHosts } from './parse-filter/hosts'; -import { processDomainLists } from './parse-filter/domainlists'; +import { processHostsWithPreload } from './parse-filter/hosts'; +import { processDomainListsWithPreload } from './parse-filter/domainlists'; import * as tldts from 'tldts-experimental'; @@ -207,15 +207,18 @@ const processPhihsingDomains = cache(function processPhihsingDomains(domainArr: temporaryBypass: !isCI || DEBUG_DOMAIN_TO_FIND !== null }); +const downloads = [ + ...PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry)), + ...PHISHING_HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry)) +]; + export function getPhishingDomains(parentSpan: Span) { return parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span) => { const domainArr = await span.traceChildAsync('download/parse/merge phishing domains', async (curSpan) => { const domainArr: string[] = []; - await Promise.all([ - ...PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainLists(curSpan, ...entry)), - ...PHISHING_HOSTS_EXTRA.map(entry => processHosts(curSpan, ...entry)) - ]).then(domainGroups => domainGroups.forEach(appendArrayInPlaceCurried(domainArr))); + const domainGroups = await Promise.all(downloads.map(task => task(curSpan))); + domainGroups.forEach(appendArrayInPlaceCurried(domainArr)); return domainArr; }); diff --git a/Build/lib/parse-filter/domainlists.ts b/Build/lib/parse-filter/domainlists.ts index a02840791..428c73a06 100644 --- a/Build/lib/parse-filter/domainlists.ts +++ b/Build/lib/parse-filter/domainlists.ts @@ -48,3 +48,21 @@ export function processDomainLists( return domainSets; }); } + +export function processDomainListsWithPreload(domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false) { + const downloadPromise = fetchAssets(domainListsUrl, mirrors); + + return (span: Span) => span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (span) => { + const text = await span.traceChildPromise('download', downloadPromise); + const domainSets: string[] = []; + const filterRules = text.split('\n'); + + span.traceChildSync('parse domain list', () => { + for (let i = 0, len = filterRules.length; i < len; i++) { + domainListLineCb(filterRules[i], domainSets, includeAllSubDomain, domainListsUrl); + } + }); + + return domainSets; + }); +} diff --git a/Build/lib/parse-filter/hosts.ts b/Build/lib/parse-filter/hosts.ts index 6df00c40e..db4dd2ae0 100644 --- a/Build/lib/parse-filter/hosts.ts +++ b/Build/lib/parse-filter/hosts.ts @@ -44,3 +44,23 @@ export function processHosts( return domainSets; }); } + +export function processHostsWithPreload(hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false) { + const downloadPromise = fetchAssets(hostsUrl, mirrors); + + return (span: Span) => span.traceChildAsync(`process hosts: ${hostsUrl}`, async (span) => { + const text = await span.traceChild('download').tracePromise(downloadPromise); + + const domainSets: string[] = []; + + const filterRules = text.split('\n'); + + span.traceChild('parse hosts').traceSyncFn(() => { + for (let i = 0, len = filterRules.length; i < len; i++) { + hostsLineCb(filterRules[i], domainSets, includeAllSubDomain, hostsUrl); + } + }); + + return domainSets; + }); +} From 5dd476cc480bb094cf54dfb39bc1bb4c3dd9ef26 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 03:05:17 +0800 Subject: [PATCH 23/35] Fix: fileEqual handling w/ empty line --- Build/lib/rules/base.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index 1f74712ab..c1c7fef12 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -398,6 +398,15 @@ export async function fileEqual(linesA: string[], source: AsyncIterable const lineA = linesA[index]; + if (lineA.length === 0 && lineB.length === 0) { + continue; + } + + // not both line are empty + if (lineA.length === 0 || lineB.length === 0) { + return false; + } + const firstCharA = lineA.charCodeAt(0); const firstCharB = lineB.charCodeAt(0); From 4cedc81b6c19b4f6a8365e4e6c88b40db9110f1a Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 03:09:54 +0800 Subject: [PATCH 24/35] Perf: preload more reject data source --- Build/build-reject-domainset.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index bd713292d..cf35fad46 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -2,8 +2,8 @@ import path from 'node:path'; import process from 'node:process'; -import { processHosts } from './lib/parse-filter/hosts'; -import { processDomainLists } from './lib/parse-filter/domainlists'; +import { processHostsWithPreload } from './lib/parse-filter/hosts'; +import { processDomainListsWithPreload } from './lib/parse-filter/domainlists'; import { processFilterRules } from './lib/parse-filter/filters'; import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_WHITELIST } from './constants/reject-data-source'; @@ -29,6 +29,11 @@ const readLocalRejectDropRulesetPromise = readFileIntoProcessedArray(path.join(S const readLocalRejectNoDropRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/reject-no-drop.conf')); const readLocalMyRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/my_reject.conf')); +const hostsDownloads = HOSTS.map(entry => processHostsWithPreload(...entry)); +const hostsExtraDownloads = HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry)); +const domainListsDownloads = DOMAIN_LISTS.map(entry => processDomainListsWithPreload(...entry)); +const domainListsExtraDownloads = DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry)); + export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => { const rejectBaseDescription = [ ...SHARED_DESCRIPTION, @@ -70,11 +75,11 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as .traceChild('download and process hosts / adblock filter rules') .traceAsyncFn((childSpan) => Promise.all([ // Parse from remote hosts & domain lists - HOSTS.map(entry => processHosts(childSpan, ...entry).then(appendArrayToRejectOutput)), - HOSTS_EXTRA.map(entry => processHosts(childSpan, ...entry).then(appendArrayToRejectExtraOutput)), + hostsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)), + hostsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)), - DOMAIN_LISTS.map(entry => processDomainLists(childSpan, ...entry).then(appendArrayToRejectOutput)), - DOMAIN_LISTS_EXTRA.map(entry => processDomainLists(childSpan, ...entry).then(appendArrayToRejectExtraOutput)), + domainListsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)), + domainListsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)), ADGUARD_FILTERS.map( entry => processFilterRules(childSpan, ...entry) From 5e0780af35367079cac79c19ba9d66f4193ae8d2 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 03:17:38 +0800 Subject: [PATCH 25/35] Perf: preload all reject data source --- Build/build-reject-domainset.ts | 37 ++++++------ Build/lib/parse-filter.test.ts | 16 +----- Build/lib/parse-filter/filters.ts | 96 +++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 32 deletions(-) diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index cf35fad46..55603c333 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -4,7 +4,7 @@ import process from 'node:process'; import { processHostsWithPreload } from './lib/parse-filter/hosts'; import { processDomainListsWithPreload } from './lib/parse-filter/domainlists'; -import { processFilterRules } from './lib/parse-filter/filters'; +import { processFilterRulesWithPreload } from './lib/parse-filter/filters'; import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_WHITELIST } from './constants/reject-data-source'; import { compareAndWriteFile } from './lib/create-file'; @@ -33,6 +33,9 @@ const hostsDownloads = HOSTS.map(entry => processHostsWithPreload(...entry)); const hostsExtraDownloads = HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry)); const domainListsDownloads = DOMAIN_LISTS.map(entry => processDomainListsWithPreload(...entry)); const domainListsExtraDownloads = DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry)); +const adguardFiltersDownloads = ADGUARD_FILTERS.map(entry => processFilterRulesWithPreload(...entry)); +const adguardFiltersExtraDownloads = ADGUARD_FILTERS_EXTRA.map(entry => processFilterRulesWithPreload(...entry)); +const adguardFiltersWhitelistsDownloads = ADGUARD_FILTERS_WHITELIST.map(entry => processFilterRulesWithPreload(...entry)); export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => { const rejectBaseDescription = [ @@ -81,24 +84,24 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as domainListsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)), domainListsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)), - ADGUARD_FILTERS.map( - entry => processFilterRules(childSpan, ...entry) - .then(({ white, black }) => { - addArrayElementsToSet(filterRuleWhitelistDomainSets, white); - appendArrayToRejectOutput(black); - }) + adguardFiltersDownloads.map( + task => task(childSpan).then(({ white, black }) => { + addArrayElementsToSet(filterRuleWhitelistDomainSets, white); + appendArrayToRejectOutput(black); + }) ), - ADGUARD_FILTERS_EXTRA.map( - entry => processFilterRules(childSpan, ...entry) - .then(({ white, black }) => { - addArrayElementsToSet(filterRuleWhitelistDomainSets, white); - appendArrayToRejectExtraOutput(black); - }) + adguardFiltersExtraDownloads.map( + task => task(childSpan).then(({ white, black }) => { + addArrayElementsToSet(filterRuleWhitelistDomainSets, white); + appendArrayToRejectExtraOutput(black); + }) + ), + adguardFiltersWhitelistsDownloads.map( + task => task(childSpan).then(({ white, black }) => { + addArrayElementsToSet(filterRuleWhitelistDomainSets, white); + addArrayElementsToSet(filterRuleWhitelistDomainSets, black); + }) ), - ADGUARD_FILTERS_WHITELIST.map(entry => processFilterRules(childSpan, ...entry).then(({ white, black }) => { - addArrayElementsToSet(filterRuleWhitelistDomainSets, white); - addArrayElementsToSet(filterRuleWhitelistDomainSets, black); - })), getPhishingDomains(childSpan).then(appendArrayToRejectExtraOutput), readLocalRejectDomainsetPromise.then(appendArrayToRejectOutput), readLocalRejectDomainsetPromise.then(appendArrayToRejectExtraOutput), diff --git a/Build/lib/parse-filter.test.ts b/Build/lib/parse-filter.test.ts index 043571bb8..e112bd0b0 100644 --- a/Build/lib/parse-filter.test.ts +++ b/Build/lib/parse-filter.test.ts @@ -1,11 +1,7 @@ import { describe, it } from 'mocha'; -import { parse, processFilterRules } from './parse-filter/filters'; +import { parse } from './parse-filter/filters'; import type { ParseType } from './parse-filter/filters'; -import { createCacheKey } from './cache-filesystem'; -import { createSpan } from '../trace'; - -const cacheKey = createCacheKey(__filename); describe('parse', () => { const MUTABLE_PARSE_LINE_RESULT: [string, ParseType] = ['', 1000]; @@ -14,13 +10,3 @@ describe('parse', () => { console.log(parse('||top.mail.ru^$badfilter', MUTABLE_PARSE_LINE_RESULT, false)); }); }); - -describe.skip('processFilterRules', () => { - it('https://filters.adtidy.org/extension/ublock/filters/18_optimized.txt', () => { - console.log(processFilterRules( - createSpan('noop'), - cacheKey('https://filters.adtidy.org/extension/ublock/filters/18_optimized.txt'), - [] - )); - }); -}); diff --git a/Build/lib/parse-filter/filters.ts b/Build/lib/parse-filter/filters.ts index 132ab06df..b2db3acc9 100644 --- a/Build/lib/parse-filter/filters.ts +++ b/Build/lib/parse-filter/filters.ts @@ -20,6 +20,102 @@ const enum ParseType { export { type ParseType }; +export function processFilterRulesWithPreload( + filterRulesUrl: string, + fallbackUrls?: string[] | null, + allowThirdParty = false +) { + const downloadPromise = fetchAssets(filterRulesUrl, fallbackUrls); + + return (span: Span) => span.traceChildAsync<{ white: string[], black: string[] }>(`process filter rules: ${filterRulesUrl}`, async (span) => { + const text = await span.traceChildPromise('download', downloadPromise); + + const whitelistDomainSets = new Set(); + const blacklistDomainSets = new Set(); + + const warningMessages: string[] = []; + + const MUTABLE_PARSE_LINE_RESULT: [string, ParseType] = ['', ParseType.NotParsed]; + /** + * @param {string} line + */ + const lineCb = (line: string) => { + const result = parse(line, MUTABLE_PARSE_LINE_RESULT, allowThirdParty); + const flag = result[1]; + + if (flag === ParseType.NotParsed) { + throw new Error(`Didn't parse line: ${line}`); + } + if (flag === ParseType.Null) { + return; + } + + const hostname = result[0]; + + if (flag === ParseType.WhiteIncludeSubdomain || flag === ParseType.WhiteAbsolute) { + onWhiteFound(hostname, filterRulesUrl); + } else { + onBlackFound(hostname, filterRulesUrl); + } + + switch (flag) { + case ParseType.WhiteIncludeSubdomain: + if (hostname[0] === '.') { + whitelistDomainSets.add(hostname); + } else { + whitelistDomainSets.add(`.${hostname}`); + } + break; + case ParseType.WhiteAbsolute: + whitelistDomainSets.add(hostname); + break; + case ParseType.BlackIncludeSubdomain: + if (hostname[0] === '.') { + blacklistDomainSets.add(hostname); + } else { + blacklistDomainSets.add(`.${hostname}`); + } + break; + case ParseType.BlackAbsolute: + blacklistDomainSets.add(hostname); + break; + case ParseType.ErrorMessage: + warningMessages.push(hostname); + break; + default: + break; + } + }; + + const filterRules = text.split('\n'); + + span.traceChild('parse adguard filter').traceSyncFn(() => { + for (let i = 0, len = filterRules.length; i < len; i++) { + lineCb(filterRules[i]); + } + }); + + for (let i = 0, len = warningMessages.length; i < len; i++) { + console.warn( + picocolors.yellow(warningMessages[i]), + picocolors.gray(picocolors.underline(filterRulesUrl)) + ); + } + + console.log( + picocolors.gray('[process filter]'), + picocolors.gray(filterRulesUrl), + picocolors.gray(`white: ${whitelistDomainSets.size}`), + picocolors.gray(`black: ${blacklistDomainSets.size}`) + ); + + return { + white: Array.from(whitelistDomainSets), + black: Array.from(blacklistDomainSets) + }; + }); +} + export async function processFilterRules( parentSpan: Span, filterRulesUrl: string, From a812c40384b74411ded9d8bf3d88782dd52185b6 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 22:57:41 +0800 Subject: [PATCH 26/35] CI: download previous build using GitHub Actions --- .github/workflows/main.yml | 38 ++++++++++++++++++++---- Build/constants/dir.ts | 7 ++--- Build/download-previous-build.ts | 6 ++++ Build/index.ts | 8 +---- package.json | 1 - pnpm-lock.yaml | 51 -------------------------------- 6 files changed, 42 insertions(+), 69 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2c13b0efb..c67f30286 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,6 +30,33 @@ jobs: with: node-version-file: ".node-version" cache: "pnpm" + - name: Create RAM Disk for building + id: ramdisk + run: | + BUILD_DIR=$(mktemp -d -p /dev/shm/ -t sukka-surge-public.XXXXXXXXXX) + echo "Build dir created at $BUILD_DIR" + echo "build_dir=$BUILD_DIR" >> $GITHUB_OUTPUT + - name: Download Previous Build + uses: actions/checkout@v4 + with: + repository: SukkaLab/ruleset.skk.moe + persist-credentials: false + path: previous-build-${{ github.run_id }}-${{ github.run_number }} + - run: mv previous-build-${{ github.run_id }}-${{ github.run_number }}/{.,}* ${{ steps.ramdisk.outputs.build_dir }}/ + - name: build folder check + # If the public directory doesn't exist, the build should fail. + # If the public directory is empty, the build should fail. + run: | + if [ ! -d ${{ steps.ramdisk.outputs.build_dir }}/.git ]; then + echo ".git not found" + exit 1 + fi + if [ ! -d ${{ steps.ramdisk.outputs.build_dir }}/List ]; then + echo "List not found" + exit 1 + fi + echo "public directory is ready: ${{ steps.ramdisk.outputs.build_dir }}" + - run: rm -rf "${{ steps.ramdisk.outputs.build_dir }}/.git" - name: Get current date id: date run: | @@ -57,16 +84,17 @@ jobs: ${{ runner.os }}-v3- - run: pnpm install - run: pnpm run build - id: build + env: + PUBLIC_DIR: ${{ steps.ramdisk.outputs.build_dir }} - name: Pre-deploy check # If the public directory doesn't exist, the build should fail. # If the public directory is empty, the build should fail. run: | - if [ ! -d ${{ steps.build.outputs.public_dir }} ]; then + if [ ! -d ${{ steps.ramdisk.outputs.build_dir }} ]; then echo "public directory not found" exit 1 fi - if [ ! "$(ls -A ${{ steps.build.outputs.public_dir }})" ]; then + if [ ! "$(ls -A ${{ steps.ramdisk.outputs.build_dir }})" ]; then echo "public directory is empty" exit 1 fi @@ -74,11 +102,11 @@ jobs: echo ".BUILD_FINISHED not found" exit 1 fi - echo "public directory is ready: ${{ steps.build.outputs.public_dir }}" + echo "public directory is ready: ${{ steps.ramdisk.outputs.build_dir }}" - uses: actions/upload-artifact@v4 with: name: build-artifact-${{ github. ref_name }} - path: ${{ steps.build.outputs.public_dir }} + path: ${{ steps.ramdisk.outputs.build_dir }} if-no-files-found: error retention-days: 1 compression-level: 4 diff --git a/Build/constants/dir.ts b/Build/constants/dir.ts index a22ed7225..5064a79f1 100644 --- a/Build/constants/dir.ts +++ b/Build/constants/dir.ts @@ -1,6 +1,5 @@ import path from 'node:path'; -import fs from 'node:fs'; -import { isCI } from 'ci-info'; +import process from 'node:process'; export const ROOT_DIR = path.resolve(__dirname, '../..'); @@ -8,9 +7,7 @@ export const CACHE_DIR = path.resolve(ROOT_DIR, '.cache'); export const SOURCE_DIR = path.join(ROOT_DIR, 'Source'); -export const PUBLIC_DIR = isCI - ? fs.mkdtempSync('/dev/shm/sukkaw-surge-public-') - : path.resolve(ROOT_DIR, 'public'); +export const PUBLIC_DIR = process.env.PUBLIC_DIR || path.resolve(ROOT_DIR, 'public'); export const OUTPUT_SURGE_DIR = path.join(PUBLIC_DIR, 'List'); export const OUTPUT_CLASH_DIR = path.resolve(PUBLIC_DIR, 'Clash'); diff --git a/Build/download-previous-build.ts b/Build/download-previous-build.ts index 0a5ae7be3..8080c5bb4 100644 --- a/Build/download-previous-build.ts +++ b/Build/download-previous-build.ts @@ -10,6 +10,7 @@ import picocolors from 'picocolors'; import { PUBLIC_DIR } from './constants/dir'; import { requestWithLog } from './lib/fetch-retry'; import { isDirectoryEmptySync } from './lib/misc'; +import { isCI } from 'ci-info'; const GITHUB_CODELOAD_URL = 'https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master'; const GITLAB_CODELOAD_URL = 'https://gitlab.com/SukkaW/ruleset.skk.moe/-/archive/master/ruleset.skk.moe-master.tar.gz'; @@ -20,6 +21,11 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a return; } + // we uses actions/checkout to download the previous build now, so we should throw if the directory is empty + if (isCI) { + throw new Error('CI environment detected, but public directory is empty'); + } + const tarGzUrl = await span.traceChildAsync('get tar.gz url', async () => { const resp = await requestWithLog(GITHUB_CODELOAD_URL, { method: 'HEAD' }); if (resp.statusCode !== 200) { diff --git a/Build/index.ts b/Build/index.ts index f0dc14d12..40c95bdb5 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -29,9 +29,7 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules'; import { createSpan, printTraceResult, whyIsNodeRunning } from './trace'; import { buildDeprecateFiles } from './build-deprecate-files'; import path from 'node:path'; -import { PUBLIC_DIR, ROOT_DIR } from './constants/dir'; - -import { setOutput } from '@actions/core'; +import { ROOT_DIR } from './constants/dir'; process.on('uncaughtException', (error) => { console.error('Uncaught exception:', error); @@ -108,10 +106,6 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); // write a file to demonstrate that the build is finished fs.writeFileSync(buildFinishedLock, 'BUILD_FINISHED\n'); - if (process.env.GITHUB_OUTPUT) { - setOutput('public_dir', PUBLIC_DIR); - } - // Finish the build to avoid leaking timer/fetch ref await whyIsNodeRunning(); process.exit(0); diff --git a/package.json b/package.json index e0414330f..3876add1b 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,6 @@ "author": "", "license": "ISC", "dependencies": { - "@actions/core": "^1.11.1", "@ghostery/adblocker": "^2.3.1", "@henrygd/queue": "^1.0.7", "async-retry": "^1.3.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e0cce3f13..1a8d049eb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -19,9 +19,6 @@ importers: .: dependencies: - '@actions/core': - specifier: ^1.11.1 - version: 1.11.1 '@ghostery/adblocker': specifier: ^2.3.1 version: 2.3.1 @@ -164,18 +161,6 @@ importers: packages: - '@actions/core@1.11.1': - resolution: {integrity: sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==} - - '@actions/exec@1.1.1': - resolution: {integrity: sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==} - - '@actions/http-client@2.2.3': - resolution: {integrity: sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==} - - '@actions/io@1.1.3': - resolution: {integrity: sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==} - '@antfu/utils@0.7.10': resolution: {integrity: sha512-+562v9k4aI80m1+VuMHehNJWLOFjBnXn3tdOitzD0il5b7smkSBal4+a3oKiQTbrwMmN/TBUMDvbdoWDehgOww==} @@ -249,10 +234,6 @@ packages: resolution: {integrity: sha512-lB05FkqEdUg2AA0xEbUz0SnkXT1LcCTa438W4IWTUh4hdOnVbQyOJ81OrDXsJk/LSiJHubgGEFoR5EHq1NsH1A==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@fastify/busboy@2.1.1': - resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} - engines: {node: '>=14'} - '@ghostery/adblocker-content@2.3.1': resolution: {integrity: sha512-8ZTY1sEE218b0EMk3Q9fv/cWwUxunSCyBOaKuviEmJY5EyvPa1VbGPuTq/Qdvo1kduD8nLEzCwgWhcT3F3TT0Q==} @@ -1745,10 +1726,6 @@ packages: tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - tunnel@0.0.6: - resolution: {integrity: sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==} - engines: {node: '>=0.6.11 <=0.7.0 || >=0.7.3'} - type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -1773,10 +1750,6 @@ packages: undici-types@6.20.0: resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} - undici@5.28.5: - resolution: {integrity: sha512-zICwjrDrcrUE0pyyJc1I2QzBkLM8FINsgOrt6WjA+BgajVq9Nxu2PbFFXUrAggLfDXlZGZBVZYw7WNV5KiBiBA==} - engines: {node: '>=14.0'} - undici@7.2.3: resolution: {integrity: sha512-2oSLHaDalSt2/O/wHA9M+/ZPAOcU2yrSP/cdBYJ+YxZskiPYDSqHbysLSlD7gq3JMqOoJI5O31RVU3BxX/MnAA==} engines: {node: '>=20.18.1'} @@ -1870,22 +1843,6 @@ packages: snapshots: - '@actions/core@1.11.1': - dependencies: - '@actions/exec': 1.1.1 - '@actions/http-client': 2.2.3 - - '@actions/exec@1.1.1': - dependencies: - '@actions/io': 1.1.3 - - '@actions/http-client@2.2.3': - dependencies: - tunnel: 0.0.6 - undici: 5.28.5(patch_hash=eyidnukwfhrd7exzoydz2h5cfq) - - '@actions/io@1.1.3': {} - '@antfu/utils@0.7.10': {} '@babel/code-frame@7.26.2': @@ -1985,8 +1942,6 @@ snapshots: '@eslint/core': 0.10.0 levn: 0.4.1 - '@fastify/busboy@2.1.1': {} - '@ghostery/adblocker-content@2.3.1': dependencies: '@ghostery/adblocker-extended-selectors': 2.3.1 @@ -3581,8 +3536,6 @@ snapshots: dependencies: safe-buffer: 5.2.1 - tunnel@0.0.6: {} - type-check@0.4.0: dependencies: prelude-ls: 1.2.1 @@ -3606,10 +3559,6 @@ snapshots: undici-types@6.20.0: {} - undici@5.28.5(patch_hash=eyidnukwfhrd7exzoydz2h5cfq): - dependencies: - '@fastify/busboy': 2.1.1 - undici@7.2.3(patch_hash=eyidnukwfhrd7exzoydz2h5cfq): {} unique-filename@4.0.0: From edfdbc8cdc327448a9e6594a6e5554fab06901d2 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 23:45:38 +0800 Subject: [PATCH 27/35] Fix: parse adguard filter ignore domain with `/` --- Build/lib/parse-filter/filters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Build/lib/parse-filter/filters.ts b/Build/lib/parse-filter/filters.ts index b2db3acc9..2406f0583 100644 --- a/Build/lib/parse-filter/filters.ts +++ b/Build/lib/parse-filter/filters.ts @@ -517,7 +517,7 @@ export function parse($line: string, result: [string, ParseType], allowThirdPart } const sliced = (sliceStart > 0 || sliceEnd < 0) ? line.slice(sliceStart, sliceEnd === 0 ? undefined : sliceEnd) : line; - if (sliced.length === 0) { + if (sliced.length === 0 || sliced.includes('/')) { result[1] = ParseType.Null; return result; } From a46f24dd9aefd6b72bf6dcfbb87e5de6cb0e43c4 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 18 Jan 2025 23:49:00 +0800 Subject: [PATCH 28/35] Chore: housekeeping [skip ci] --- package.json | 10 +++--- pnpm-lock.yaml | 92 ++++++++++++++++++++++++++------------------------ 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/package.json b/package.json index 3876add1b..881162e20 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "@ghostery/adblocker": "^2.3.1", "@henrygd/queue": "^1.0.7", "async-retry": "^1.3.3", - "better-sqlite3": "^11.8.0", + "better-sqlite3": "^11.8.1", "cacache": "^19.0.1", "ci-info": "^4.1.0", "csv-parse": "^5.6.0", @@ -35,10 +35,10 @@ "hash-wasm": "^4.12.0", "json-stringify-pretty-compact": "^3.0.0", "picocolors": "^1.1.1", - "tar-fs": "3.0.6", + "tar-fs": "^3.0.8", "tinyexec": "^0.3.2", - "tldts": "^6.1.72", - "tldts-experimental": "^6.1.72", + "tldts": "^6.1.73", + "tldts-experimental": "^6.1.73", "undici": "^7.2.3", "undici-cache-store-better-sqlite3": "^0.1.1", "whoiser": "^1.18.0", @@ -65,7 +65,7 @@ "eslint-config-sukka": "^6.13.0", "eslint-formatter-sukka": "^6.13.0", "expect": "^29.7.0", - "mitata": "^1.0.30", + "mitata": "^1.0.31", "mocha": "^11.0.1", "typescript": "^5.7.3" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1a8d049eb..f9a5b19ca 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,8 +29,8 @@ importers: specifier: ^1.3.3 version: 1.3.3 better-sqlite3: - specifier: ^11.8.0 - version: 11.8.0 + specifier: ^11.8.1 + version: 11.8.1 cacache: specifier: ^19.0.1 version: 19.0.1 @@ -65,17 +65,17 @@ importers: specifier: ^1.1.1 version: 1.1.1 tar-fs: - specifier: 3.0.6 - version: 3.0.6 + specifier: ^3.0.8 + version: 3.0.8 tinyexec: specifier: ^0.3.2 version: 0.3.2 tldts: - specifier: ^6.1.72 - version: 6.1.72 + specifier: ^6.1.73 + version: 6.1.73 tldts-experimental: - specifier: ^6.1.72 - version: 6.1.72 + specifier: ^6.1.73 + version: 6.1.73 undici: specifier: ^7.2.3 version: 7.2.3(patch_hash=eyidnukwfhrd7exzoydz2h5cfq) @@ -150,8 +150,8 @@ importers: specifier: ^29.7.0 version: 29.7.0 mitata: - specifier: ^1.0.30 - version: 1.0.30 + specifier: ^1.0.31 + version: 1.0.31 mocha: specifier: ^11.0.1 version: 11.0.1 @@ -703,14 +703,16 @@ packages: bare-events@2.5.0: resolution: {integrity: sha512-/E8dDe9dsbLyh2qrZ64PEPadOQ0F4gbl1sUJOrmph7xOiIxfY8vwab/4bFLh4Y88/Hk/ujKcrQKc+ps0mv873A==} - bare-fs@2.3.5: - resolution: {integrity: sha512-SlE9eTxifPDJrT6YgemQ1WGFleevzwY+XAP1Xqgl56HtcrisC2CHCZ2tq6dBpcH2TnNxwUEUGhweo+lrQtYuiw==} + bare-fs@4.0.1: + resolution: {integrity: sha512-ilQs4fm/l9eMfWY2dY0WCIUplSUp7U0CT1vrqMg1MUdeZl4fypu5UP0XcDBK5WBQPJAKP1b7XEodISmekH/CEg==} + engines: {bare: '>=1.7.0'} - bare-os@2.4.4: - resolution: {integrity: sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==} + bare-os@3.4.0: + resolution: {integrity: sha512-9Ous7UlnKbe3fMi7Y+qh0DwAup6A1JkYgPnjvMDNOlmnxNRQvQ/7Nst+OnUQKzk0iAT0m9BisbDVp9gCv8+ETA==} + engines: {bare: '>=1.6.0'} - bare-path@2.1.3: - resolution: {integrity: sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==} + bare-path@3.0.0: + resolution: {integrity: sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==} bare-stream@2.3.2: resolution: {integrity: sha512-EFZHSIBkDgSHIwj2l2QZfP4U5OcD4xFAOwhSb/vlr9PIqyGJGvB/nfClJbcnh3EY4jtPE4zsb5ztae96bVF79A==} @@ -718,8 +720,8 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - better-sqlite3@11.8.0: - resolution: {integrity: sha512-aKv9s2dir7bsEX5RIjL9HHWB9uQ+f6Vch5B4qmeAOop4Y9OYHX+PNKLr+mpv6+d8L/ZYh4l7H8zPuVMbWkVMLw==} + better-sqlite3@11.8.1: + resolution: {integrity: sha512-9BxNaBkblMjhJW8sMRZxnxVTRgbRmssZW0Oxc1MPBTfiR+WW21e2Mk4qu8CzrcZb1LwPCnFsfDEzq+SNcBU8eg==} binary-extensions@2.3.0: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} @@ -1391,8 +1393,8 @@ packages: resolution: {integrity: sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==} engines: {node: '>= 18'} - mitata@1.0.30: - resolution: {integrity: sha512-zefhVHdXznlD92aojkn7sZ+ZUzvbG8vmFvLllIHZZoUwGlO+YMqMbnA3tcutwe/yAVdkL5/3z65isEfvdomoJQ==} + mitata@1.0.31: + resolution: {integrity: sha512-WHQaVvR1zQlAhYQhsGcXdlVMqzgvgscMQeXg7pxLh7t8kAH0BI4uD3MU3gVfJT+6jrQS37YhviXq1CAnhUl6Fg==} mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} @@ -1680,8 +1682,8 @@ packages: tar-fs@2.1.1: resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==} - tar-fs@3.0.6: - resolution: {integrity: sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==} + tar-fs@3.0.8: + resolution: {integrity: sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==} tar-stream@2.2.0: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} @@ -1700,14 +1702,14 @@ packages: tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} - tldts-core@6.1.72: - resolution: {integrity: sha512-FW3H9aCaGTJ8l8RVCR3EX8GxsxDbQXuwetwwgXA2chYdsX+NY1ytCBl61narjjehWmCw92tc1AxlcY3668CU8g==} + tldts-core@6.1.73: + resolution: {integrity: sha512-k1g5eX87vxu3g//6XMn62y4qjayu4cYby/PF7Ksnh4F4uUK1Z1ze/mJ4a+y5OjdJ+cXRp+YTInZhH+FGdUWy1w==} - tldts-experimental@6.1.72: - resolution: {integrity: sha512-mfPL+Pzn3nJ0JeI9AHuO4l0NbxldZhpWUYokb8HdK8gbZ2k0/qEqs5E/FqcOjVr4vzTZpbRtiwMPXv77VwXpyQ==} + tldts-experimental@6.1.73: + resolution: {integrity: sha512-5rljzPNMgdbjGs3WGvQIUi9oV93JaMsZKSslCe6qK62RGqRv+DIuiCkKsj8n+tw9v8Q9bSgRbB4iNwa1/yfd3g==} - tldts@6.1.72: - resolution: {integrity: sha512-QNtgIqSUb9o2CoUjX9T5TwaIvUUJFU1+12PJkgt42DFV2yf9J6549yTF2uGloQsJ/JOC8X+gIB81ind97hRiIQ==} + tldts@6.1.73: + resolution: {integrity: sha512-/h4bVmuEMm57c2uCiAf1Q9mlQk7cA22m+1Bu0K92vUUtTVT9D4mOFWD9r4WQuTULcG9eeZtNKhLl0Il1LdKGog==} hasBin: true to-regex-range@5.0.1: @@ -1955,7 +1957,7 @@ snapshots: '@remusao/guess-url-type': 1.3.0 '@remusao/small': 1.3.0 '@remusao/smaz': 1.10.0 - tldts-experimental: 6.1.72 + tldts-experimental: 6.1.73 '@henrygd/queue@1.0.7': {} @@ -2420,19 +2422,19 @@ snapshots: bare-events@2.5.0: optional: true - bare-fs@2.3.5: + bare-fs@4.0.1: dependencies: bare-events: 2.5.0 - bare-path: 2.1.3 + bare-path: 3.0.0 bare-stream: 2.3.2 optional: true - bare-os@2.4.4: + bare-os@3.4.0: optional: true - bare-path@2.1.3: + bare-path@3.0.0: dependencies: - bare-os: 2.4.4 + bare-os: 3.4.0 optional: true bare-stream@2.3.2: @@ -2442,7 +2444,7 @@ snapshots: base64-js@1.5.1: {} - better-sqlite3@11.8.0: + better-sqlite3@11.8.1: dependencies: bindings: 1.5.0 prebuild-install: 7.1.2 @@ -3171,7 +3173,7 @@ snapshots: minipass: 7.1.2 rimraf: 5.0.10 - mitata@1.0.30: {} + mitata@1.0.31: {} mkdirp-classic@0.5.3: {} @@ -3477,13 +3479,13 @@ snapshots: pump: 3.0.2 tar-stream: 2.2.0 - tar-fs@3.0.6: + tar-fs@3.0.8: dependencies: pump: 3.0.2 tar-stream: 3.1.7 optionalDependencies: - bare-fs: 2.3.5 - bare-path: 2.1.3 + bare-fs: 4.0.1 + bare-path: 3.0.0 tar-stream@2.2.0: dependencies: @@ -3512,15 +3514,15 @@ snapshots: tinyexec@0.3.2: {} - tldts-core@6.1.72: {} + tldts-core@6.1.73: {} - tldts-experimental@6.1.72: + tldts-experimental@6.1.73: dependencies: - tldts-core: 6.1.72 + tldts-core: 6.1.73 - tldts@6.1.72: + tldts@6.1.73: dependencies: - tldts-core: 6.1.72 + tldts-core: 6.1.73 to-regex-range@5.0.1: dependencies: @@ -3554,7 +3556,7 @@ snapshots: undici-cache-store-better-sqlite3@0.1.1(undici@7.2.3(patch_hash=eyidnukwfhrd7exzoydz2h5cfq)): dependencies: - better-sqlite3: 11.8.0 + better-sqlite3: 11.8.1 undici: 7.2.3(patch_hash=eyidnukwfhrd7exzoydz2h5cfq) undici-types@6.20.0: {} From ca9415ecc6a77fbee6e2956442bdedb8cc9dc76b Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sun, 19 Jan 2025 12:33:27 +0800 Subject: [PATCH 29/35] Phishing domains trim `www` --- Build/lib/get-phishing-domains.ts | 2 +- Build/lib/normalize-domain.ts | 18 ++++++++++++ Build/lib/parse-filter/domainlists.ts | 40 +++++++++++++++------------ 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/Build/lib/get-phishing-domains.ts b/Build/lib/get-phishing-domains.ts index 99a073a93..2f5c0408f 100644 --- a/Build/lib/get-phishing-domains.ts +++ b/Build/lib/get-phishing-domains.ts @@ -208,7 +208,7 @@ const processPhihsingDomains = cache(function processPhihsingDomains(domainArr: }); const downloads = [ - ...PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry)), + ...PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry, true)), ...PHISHING_HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry)) ]; diff --git a/Build/lib/normalize-domain.ts b/Build/lib/normalize-domain.ts index 8837ae459..2e18f5e59 100644 --- a/Build/lib/normalize-domain.ts +++ b/Build/lib/normalize-domain.ts @@ -24,6 +24,24 @@ export function fastNormalizeDomain(domain: string, parsed: TldTsParsed | null = return parsed.hostname; } +export function fastNormalizeDomainIgnoreWww(domain: string, parsed: TldTsParsed | null = null) { + // We don't want tldts to call its own "extractHostname" on ip, bail out ip first. + // Now ip has been bailed out, we can safely set normalizeTldtsOpt.detectIp to false. + if (isProbablyIpv4(domain) || isProbablyIpv6(domain)) { + return null; + } + + parsed ??= tldts.parse(domain, normalizeTldtsOpt); + + // Private invalid domain (things like .tor, .dn42, etc) + if (!parsed.isIcann && !parsed.isPrivate) return null; + + if (parsed.subdomain === 'www') { + return parsed.domain; + } + return parsed.hostname; +} + export function normalizeDomain(domain: string, parsed: TldTsParsed | null = null) { if (domain.length === 0) return null; diff --git a/Build/lib/parse-filter/domainlists.ts b/Build/lib/parse-filter/domainlists.ts index 428c73a06..531084aa7 100644 --- a/Build/lib/parse-filter/domainlists.ts +++ b/Build/lib/parse-filter/domainlists.ts @@ -1,36 +1,35 @@ -import picocolors from 'picocolors'; -import { fastNormalizeDomain } from '../normalize-domain'; +import { fastNormalizeDomain, fastNormalizeDomainIgnoreWww } from '../normalize-domain'; import { processLine } from '../process-line'; import { onBlackFound } from './shared'; import { fetchAssets } from '../fetch-assets'; import type { Span } from '../../trace'; -function domainListLineCb(l: string, set: string[], includeAllSubDomain: boolean, meta: string) { +function domainListLineCb(l: string, set: string[], includeAllSubDomain: boolean, meta: string, normalizeDomain = fastNormalizeDomain) { const line = processLine(l); if (!line) return; - const domain = fastNormalizeDomain(line); - if (!domain) return; - if (domain !== line) { - console.log( - picocolors.red('[process domain list]'), - picocolors.gray(`line: ${line}`), - picocolors.gray(`domain: ${domain}`), - picocolors.gray(meta) - ); - + const domain = normalizeDomain(line); + if (!domain) { + // console.log( + // picocolors.red('[process domain list]'), + // picocolors.gray(`line: ${line}`), + // picocolors.gray(`domain: ${domain}`), + // picocolors.gray(meta) + // ); return; } onBlackFound(domain, meta); - set.push(includeAllSubDomain ? `.${line}` : line); + set.push(includeAllSubDomain ? `.${domain}` : domain); } export function processDomainLists( span: Span, - domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false + domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false, wwwToApex = false ) { + const domainNormalizer = wwwToApex ? fastNormalizeDomainIgnoreWww : fastNormalizeDomain; + return span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (span) => { const text = await span.traceChildAsync('download', () => fetchAssets( domainListsUrl, @@ -41,7 +40,7 @@ export function processDomainLists( span.traceChildSync('parse domain list', () => { for (let i = 0, len = filterRules.length; i < len; i++) { - domainListLineCb(filterRules[i], domainSets, includeAllSubDomain, domainListsUrl); + domainListLineCb(filterRules[i], domainSets, includeAllSubDomain, domainListsUrl, domainNormalizer); } }); @@ -49,7 +48,12 @@ export function processDomainLists( }); } -export function processDomainListsWithPreload(domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false) { +export function processDomainListsWithPreload( + domainListsUrl: string, mirrors: string[] | null, + includeAllSubDomain = false, wwwToApex = false +) { + const domainNormalizer = wwwToApex ? fastNormalizeDomainIgnoreWww : fastNormalizeDomain; + const downloadPromise = fetchAssets(domainListsUrl, mirrors); return (span: Span) => span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (span) => { @@ -59,7 +63,7 @@ export function processDomainListsWithPreload(domainListsUrl: string, mirrors: s span.traceChildSync('parse domain list', () => { for (let i = 0, len = filterRules.length; i < len; i++) { - domainListLineCb(filterRules[i], domainSets, includeAllSubDomain, domainListsUrl); + domainListLineCb(filterRules[i], domainSets, includeAllSubDomain, domainListsUrl, domainNormalizer); } }); From ced9f3fb0c25c30541ce617a864383a927c22c70 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sun, 19 Jan 2025 12:49:35 +0800 Subject: [PATCH 30/35] Perf: improve domainlists performance --- Build/lib/parse-filter/domainlists.ts | 32 ++++++++++++++++----------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/Build/lib/parse-filter/domainlists.ts b/Build/lib/parse-filter/domainlists.ts index 531084aa7..63df41073 100644 --- a/Build/lib/parse-filter/domainlists.ts +++ b/Build/lib/parse-filter/domainlists.ts @@ -4,24 +4,28 @@ import { onBlackFound } from './shared'; import { fetchAssets } from '../fetch-assets'; import type { Span } from '../../trace'; -function domainListLineCb(l: string, set: string[], includeAllSubDomain: boolean, meta: string, normalizeDomain = fastNormalizeDomain) { +function domainListLineCb(l: string, set: string[], meta: string, normalizeDomain = fastNormalizeDomain) { const line = processLine(l); if (!line) return; const domain = normalizeDomain(line); - if (!domain) { - // console.log( - // picocolors.red('[process domain list]'), - // picocolors.gray(`line: ${line}`), - // picocolors.gray(`domain: ${domain}`), - // picocolors.gray(meta) - // ); - return; - } + if (!domain) return; onBlackFound(domain, meta); - set.push(includeAllSubDomain ? `.${domain}` : domain); + set.push(domain); +} + +function domainListLineCbIncludeAllSubdomain(l: string, set: string[], meta: string, normalizeDomain = fastNormalizeDomain) { + const line = processLine(l); + if (!line) return; + + const domain = normalizeDomain(line); + if (!domain) return; + + onBlackFound(domain, meta); + + set.push('.' + domain); } export function processDomainLists( @@ -29,6 +33,7 @@ export function processDomainLists( domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false, wwwToApex = false ) { const domainNormalizer = wwwToApex ? fastNormalizeDomainIgnoreWww : fastNormalizeDomain; + const lineCb = includeAllSubDomain ? domainListLineCbIncludeAllSubdomain : domainListLineCb; return span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (span) => { const text = await span.traceChildAsync('download', () => fetchAssets( @@ -40,7 +45,7 @@ export function processDomainLists( span.traceChildSync('parse domain list', () => { for (let i = 0, len = filterRules.length; i < len; i++) { - domainListLineCb(filterRules[i], domainSets, includeAllSubDomain, domainListsUrl, domainNormalizer); + lineCb(filterRules[i], domainSets, domainListsUrl, domainNormalizer); } }); @@ -55,6 +60,7 @@ export function processDomainListsWithPreload( const domainNormalizer = wwwToApex ? fastNormalizeDomainIgnoreWww : fastNormalizeDomain; const downloadPromise = fetchAssets(domainListsUrl, mirrors); + const lineCb = includeAllSubDomain ? domainListLineCbIncludeAllSubdomain : domainListLineCb; return (span: Span) => span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (span) => { const text = await span.traceChildPromise('download', downloadPromise); @@ -63,7 +69,7 @@ export function processDomainListsWithPreload( span.traceChildSync('parse domain list', () => { for (let i = 0, len = filterRules.length; i < len; i++) { - domainListLineCb(filterRules[i], domainSets, includeAllSubDomain, domainListsUrl, domainNormalizer); + lineCb(filterRules[i], domainSets, domainListsUrl, domainNormalizer); } }); From 7be1fa9b587db2dc98c439cad51052f5c0c6f4ac Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sun, 19 Jan 2025 14:18:17 +0800 Subject: [PATCH 31/35] Fix: remove `localhost` from DNS Mapping --- Source/non_ip/direct.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/non_ip/direct.ts b/Source/non_ip/direct.ts index 4b55af003..5ce21f6fd 100644 --- a/Source/non_ip/direct.ts +++ b/Source/non_ip/direct.ts @@ -108,7 +108,7 @@ export const LAN = { LAN: { dns: 'system', hosts: { - localhost: ['127.0.0.1'] + // localhost: ['127.0.0.1'] }, realip: true, ruleset: true, From 4a9147b7ab11b92e7307fee7cd743d59d8001714 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sun, 19 Jan 2025 16:54:04 +0800 Subject: [PATCH 32/35] Update README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index ccd9e2b9a..526653c70 100644 --- a/README.md +++ b/README.md @@ -798,3 +798,7 @@ The `List/ip/china_ip.conf` file is licensed under [CC BY-SA 2.0](https://creati

+ + + + From 0774cb4fa41a06097fc88c8c607393231324ece6 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sun, 19 Jan 2025 16:54:08 +0800 Subject: [PATCH 33/35] Update CDN Hosts --- Source/domainset/cdn.conf | 6 +++--- Source/non_ip/cdn.conf | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index a4306db47..e709124e9 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -1030,9 +1030,6 @@ na-library.klarnaservices.com cdn.onesignal.com cdn.privacy-mgmt.com cdn.concert.io -euc-widget.freshworks.com -widget.freshworks.com -dam.freshworks.com cdn-public.sociabble.com embed.savvycal.com js.pusher.com @@ -3183,6 +3180,8 @@ creativajs.altervista.org s.namemc.com minotar.net dam.freshworks.com +euc-widget.freshworks.com +widget.freshworks.com cdn.now.gg cms-cdn.now.gg cdn-www.bluestacks.com @@ -3897,3 +3896,4 @@ image.vuukle.com cdn.vuukle.com touchgal-image.moyu.moe libraries.hund.io +cdn.geekzone.co.nz diff --git a/Source/non_ip/cdn.conf b/Source/non_ip/cdn.conf index e41e04f22..3cdc25d3f 100644 --- a/Source/non_ip/cdn.conf +++ b/Source/non_ip/cdn.conf @@ -43,6 +43,8 @@ DOMAIN-WILDCARD,static-??-cdn.eporner.com # Mikufuns DOMAIN-WILDCARD,file?.mikuclub.fun DOMAIN-WILDCARD,cdn?.mikuclub.fun +# FreshDesk +DOMAIN-WILDCARD,assets*.freshdesk.com # >> Embed DOMAIN-WILDCARD,cdns.*.gigya.com # >> Braze SDK From 86bec809cbd155346e8a86c79c2721d9c551c567 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Mon, 20 Jan 2025 09:23:25 +0800 Subject: [PATCH 34/35] Update CDN & Global Hosts --- Source/domainset/cdn.conf | 1 + Source/non_ip/global.conf | 1 + 2 files changed, 2 insertions(+) diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index e709124e9..4174adb04 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -112,6 +112,7 @@ eslint.style .eggjs.org .stylexjs.com pytorch.org +nextra.site # JS.ORG .js.org diff --git a/Source/non_ip/global.conf b/Source/non_ip/global.conf index 6f9305bf8..1f2b3e43e 100644 --- a/Source/non_ip/global.conf +++ b/Source/non_ip/global.conf @@ -774,6 +774,7 @@ DOMAIN-SUFFIX,npm.io DOMAIN-SUFFIX,npmjs.com DOMAIN-SUFFIX,nssurge.com DOMAIN-SUFFIX,netify.ai +DOMAIN-SUFFIX,nextra.site DOMAIN-SUFFIX,nyaa.si DOMAIN-SUFFIX,nypost.com DOMAIN-SUFFIX,nyt.com From a0ccd43be978973fcd1b535dfc32538bce299cea Mon Sep 17 00:00:00 2001 From: SukkaW Date: Mon, 20 Jan 2025 09:55:01 +0800 Subject: [PATCH 35/35] Update Reject Hosts --- Source/domainset/reject_sukka.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/domainset/reject_sukka.conf b/Source/domainset/reject_sukka.conf index 4d5b2ffa8..45b54f09b 100644 --- a/Source/domainset/reject_sukka.conf +++ b/Source/domainset/reject_sukka.conf @@ -714,6 +714,8 @@ pub-7a9aae2813a742e1b02d588e632e401b.r2.dev .sc.origins.en25.com .ignite.technology .zayac2volk11.com +.xad.js.org +.affiliates.purevpn.com adimg.daumcdn.net .ad.daum.net