From 99eb1446580f439d49b37af375b382d57d7d212c Mon Sep 17 00:00:00 2001 From: SukkaW Date: Tue, 17 Dec 2024 20:57:46 +0800 Subject: [PATCH 1/9] Update CDN & Global Hosts --- Source/domainset/cdn.conf | 3 +++ Source/non_ip/global.conf | 1 + 2 files changed, 4 insertions(+) diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index 999f13d87..50d7509e0 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -2421,6 +2421,7 @@ ubistatic-a.ubisoft.com ubistatic2-a.akamaihd.net # >> RedHat +cdn.redhat.com static.redhat.com assets.openshift.com assets.openshift.net @@ -3405,6 +3406,8 @@ assets.vogue.com .sixyik.com .fivetiu.com static.oomol.com +package-assets.oomol.com +registry.oomol.com uploads.sitepoint.com static.gnome.org assets.exercism.org diff --git a/Source/non_ip/global.conf b/Source/non_ip/global.conf index 57eebc154..e2b601067 100644 --- a/Source/non_ip/global.conf +++ b/Source/non_ip/global.conf @@ -796,6 +796,7 @@ DOMAIN-SUFFIX,producthunt.com DOMAIN-SUFFIX,proton.me DOMAIN-SUFFIX,phncdn.com DOMAIN-SUFFIX,pusher.com +DOMAIN-SUFFIX,quad9.net DOMAIN-SUFFIX,quora.com DOMAIN-SUFFIX,quoracdn.net DOMAIN-SUFFIX,raspberrypi.com From 571c8064b5a9b390831260d09e076ab7ac332514 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Tue, 17 Dec 2024 20:58:42 +0800 Subject: [PATCH 2/9] Chore: housekeeping --- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index cebbb0865..58bf35d67 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "tinyexec": "^0.3.1", "tldts": "^6.1.68", "tldts-experimental": "^6.1.68", - "undici": "^7.1.0", + "undici": "^7.1.1", "whoiser": "^1.18.0", "why-is-node-running": "^3.2.1", "xbits": "^0.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bfa5c1d5f..c0c8b2b58 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -75,8 +75,8 @@ importers: specifier: ^6.1.68 version: 6.1.68 undici: - specifier: ^7.1.0 - version: 7.1.0 + specifier: ^7.1.1 + version: 7.1.1 whoiser: specifier: ^1.18.0 version: 1.18.0 @@ -1791,8 +1791,8 @@ packages: undici-types@6.20.0: resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} - undici@7.1.0: - resolution: {integrity: sha512-3+mdX2R31khuLCm2mKExSlMdJsfol7bJkIMH80tdXA74W34rT1jKemUTlYR7WY3TqsV4wfOgpatWmmB2Jl1+5g==} + undici@7.1.1: + resolution: {integrity: sha512-WZkQ6eH9f5ZT93gaIffsbUaDpBwjbpvmMbfaEhOnbdUneurTESeRxwPGwjI28mRFESH3W3e8Togijh37ptOQqA==} engines: {node: '>=20.18.1'} unique-filename@4.0.0: @@ -3646,7 +3646,7 @@ snapshots: undici-types@6.20.0: {} - undici@7.1.0: {} + undici@7.1.1: {} unique-filename@4.0.0: dependencies: From 18d55e5b39156c05e6d092adbe9f711b97d4d779 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Wed, 18 Dec 2024 00:25:22 +0800 Subject: [PATCH 3/9] Chore: expose cache-control headers --- Build/build-public.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Build/build-public.ts b/Build/build-public.ts index e71d596ee..ac750a513 100644 --- a/Build/build-public.ts +++ b/Build/build-public.ts @@ -51,10 +51,12 @@ export const buildPublic = task(require.main === module, __filename)(async (span span, [ '/*', - ' cloudflare-cdn-cache-control: public, max-age=180, stale-while-revalidate=60, stale-if-error=30', + ' cache-control: public, max-age=180, stale-while-revalidate=60, stale-if-error=30', 'https://:project.pages.dev/*', ' X-Robots-Tag: noindex', '/Modules/*', + ' content-type: text/plain; charset=utf-8', + '/List/*', ' content-type: text/plain; charset=utf-8' ], path.join(PUBLIC_DIR, '_headers') From 5ed7a62a2c40172d260d0e1ba7069ead5cb2c503 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Wed, 18 Dec 2024 00:47:43 +0800 Subject: [PATCH 4/9] Chore: bring up Undici Cache Store --- Build/lib/fetch-retry.ts | 65 +++++++++++++++++++++++----------------- package.json | 1 + pnpm-lock.yaml | 13 ++++++++ 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index 6863c81af..bcd233b03 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -7,12 +7,15 @@ import undici, { import type { Dispatcher, - Response + Response, + RequestInit } from 'undici'; +import { BetterSqlite3CacheStore } from 'undici-cache-store-better-sqlite3'; export type UndiciResponseData = Dispatcher.ResponseData; import { inspect } from 'node:util'; +import path from 'node:path'; const agent = new Agent({}); @@ -99,6 +102,11 @@ setGlobalDispatcher(agent.compose( }), interceptors.redirect({ maxRedirections: 5 + }), + interceptors.cache({ + store: new BetterSqlite3CacheStore({ + location: path.resolve(__dirname, '../../.cache/undici-better-sqlite3-cache-store.db') + }) }) )); @@ -133,34 +141,35 @@ export const defaultRequestInit = { } }; -// export async function fetchWithLog(url: string, init?: RequestInit) { -// try { -// const res = await undici.fetch(url, init); -// if (res.status >= 400) { -// throw new ResponseError(res, url); -// } - -// if (!(res.status >= 200 && res.status <= 299) && res.status !== 304) { -// throw new ResponseError(res, url); -// } - -// return res; -// } catch (err: unknown) { -// if (typeof err === 'object' && err !== null && 'name' in err) { -// if (( -// err.name === 'AbortError' -// || ('digest' in err && err.digest === 'AbortError') -// )) { -// console.log(picocolors.gray('[fetch abort]'), url); -// } -// } else { -// console.log(picocolors.gray('[fetch fail]'), url, { name: (err as any).name }, err); -// } - -// throw err; -// } -// } +export async function $$fetch(url: string, init?: RequestInit) { + try { + const res = await undici.fetch(url, init); + if (res.status >= 400) { + throw new ResponseError(res, url); + } + + if (!(res.status >= 200 && res.status <= 299) && res.status !== 304) { + throw new ResponseError(res, url); + } + + return res; + } catch (err: unknown) { + if (typeof err === 'object' && err !== null && 'name' in err) { + if (( + err.name === 'AbortError' + || ('digest' in err && err.digest === 'AbortError') + )) { + console.log(picocolors.gray('[fetch abort]'), url); + } + } else { + console.log(picocolors.gray('[fetch fail]'), url, { name: (err as any).name }, err); + } + + throw err; + } +} +/** @deprecated -- undici.requests doesn't support gzip/br/deflate, and has difficulty w/ undidi cache */ export async function requestWithLog(url: string, opt?: Parameters[1]) { try { const res = await undici.request(url, opt); diff --git a/package.json b/package.json index 58bf35d67..0cba06c8e 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "tldts": "^6.1.68", "tldts-experimental": "^6.1.68", "undici": "^7.1.1", + "undici-cache-store-better-sqlite3": "^0.1.1", "whoiser": "^1.18.0", "why-is-node-running": "^3.2.1", "xbits": "^0.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c0c8b2b58..ab540b2af 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -77,6 +77,9 @@ importers: undici: specifier: ^7.1.1 version: 7.1.1 + undici-cache-store-better-sqlite3: + specifier: ^0.1.1 + version: 0.1.1(undici@7.1.1) whoiser: specifier: ^1.18.0 version: 1.18.0 @@ -1788,6 +1791,11 @@ packages: engines: {node: '>=14.17'} hasBin: true + undici-cache-store-better-sqlite3@0.1.1: + resolution: {integrity: sha512-F/PxGx+QY3oScnFQ/YGbN7AYtqKLTzps0izFi8LddDaobso3SYXyQORuPcnUd2JbqpdxmLLkvS2zSvG6VPPMpw==} + peerDependencies: + undici: '>=7.0.0' + undici-types@6.20.0: resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} @@ -3644,6 +3652,11 @@ snapshots: typescript@5.7.2: {} + undici-cache-store-better-sqlite3@0.1.1(undici@7.1.1): + dependencies: + better-sqlite3: 11.7.0 + undici: 7.1.1 + undici-types@6.20.0: {} undici@7.1.1: {} From 8a62743c2a02cc70cfb1d4db8f9ea973a38f6166 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Wed, 18 Dec 2024 16:22:12 +0800 Subject: [PATCH 5/9] Update CDN Hosts --- Source/domainset/cdn.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index 50d7509e0..21d12ba7b 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -1695,6 +1695,7 @@ checkout.stripe.com .stripeassets.com # response header has a x-amz-bucket-region: us-west-2, which means AWS S3 js.stripe.com +connect-js.stripe.com # >> SimilarWeb .similarcdn.com @@ -3769,3 +3770,4 @@ cdn.nba.com sp.nba.com tags.nba.com cdn.royaleapi.com +cdn.lemonsqueezy.com From 1d3015a1212a58caf9b04b953d762d21f1a3bd74 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 19 Dec 2024 10:42:52 +0800 Subject: [PATCH 6/9] Update CDN & Global Hosts --- Source/domainset/cdn.conf | 2 ++ Source/non_ip/global.conf | 1 + 2 files changed, 3 insertions(+) diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index 21d12ba7b..62ea5621c 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -906,6 +906,8 @@ telemetry.business.githubcopilot.com # but they can definitely go through secondary policy .ogads-pa.googleapis.com .ogads-pa.clients6.google.com +# Surge Diagnostic Report Upload, powered by LeanCloud Global +lc.nssurge.com # >> Telegram .cdn-telegram.org diff --git a/Source/non_ip/global.conf b/Source/non_ip/global.conf index e2b601067..3532ce086 100644 --- a/Source/non_ip/global.conf +++ b/Source/non_ip/global.conf @@ -808,6 +808,7 @@ DOMAIN-SUFFIX,redditmail.com DOMAIN-SUFFIX,redditmedia.com DOMAIN-SUFFIX,redditstatic.com DOMAIN-SUFFIX,redditspace.com +DOMAIN-SUFFIX,redhat.org DOMAIN-SUFFIX,redtube.com DOMAIN-SUFFIX,redtubepremium.com DOMAIN-SUFFIX,render.com From 99497d3d1c6acd15356048a138b0b001503e9492 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 19 Dec 2024 20:59:06 +0800 Subject: [PATCH 7/9] CI: change task name --- .github/workflows/check-source-domain.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-source-domain.yml b/.github/workflows/check-source-domain.yml index d7a124f6a..4bc0fa07b 100644 --- a/.github/workflows/check-source-domain.yml +++ b/.github/workflows/check-source-domain.yml @@ -4,8 +4,8 @@ on: workflow_dispatch: jobs: - build: - name: Build + check: + name: Check runs-on: ubuntu-latest steps: From 412ac53ef2e45a1491c02f339237b3e353cbf71a Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 19 Dec 2024 21:03:17 +0800 Subject: [PATCH 8/9] Remove Outdated & Dead Domains && Update CDN Hosts --- Source/domainset/cdn.conf | 23 +++++++++++++++++++++-- Source/domainset/reject_sukka.conf | 3 ++- Source/non_ip/my_plus.conf | 22 +--------------------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index 62ea5621c..f9f497fac 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -596,6 +596,26 @@ embed-ssl.wistia.com cdn.vidible.tv .video-cdn.net demo.unified-streaming.com +.mapi.4kvm.net +.v82u1l.com +.whweitao.com +.fsvod1.com +.szjal.cn +.hdzyk-cdn.com +.wolongzywcdn3.com +.ceres9350.com +.wolongcdnm3u8.com +.img.haiwaikan.com +.cdn.iz8qkg.com +.tokyvideo.com +.hey10.cjkypo.com +.yle888.vip +static-cdn.bytegiftia.top +.ffzy888.com +.image8899.net +.rrcdnbf3.com +.myvideos.info +.vplay.life # vimeo player.vimeo.com .vimeocdn.com @@ -1059,7 +1079,6 @@ app-static.turtl.co .intuitcdn.net adkit.9pub.io .dwcdn.net -tags.bkrtx.com public.flourish.studio players.brightcove.net sync.getpublica.com @@ -3747,7 +3766,7 @@ cdn2.edgedatg.com cdn3.edgedatg.com cdn4.edgedatg.com cdn5.edgedatg.com -cdn6.edgedatg. +cdn6.edgedatg.com .dtci-pe-vp-image-resizer.platform.edgedatg.com img.3movs.com image.ap-mail.org diff --git a/Source/domainset/reject_sukka.conf b/Source/domainset/reject_sukka.conf index ea9864457..06653239f 100644 --- a/Source/domainset/reject_sukka.conf +++ b/Source/domainset/reject_sukka.conf @@ -959,6 +959,7 @@ a.tool.lu eas.etherscan.com bitanalytics.casperverswijvelt.be .unifyintent.com +.bkrtx.com # CNAME: dualstack.beaconserver-ce-vpc0-1537565064.eu-west-1.elb.amazonaws.com # note "beaconserver" @@ -2358,7 +2359,7 @@ tongji.tom.com .svd-cdn.com .hola-shopping.com .hola.hk -.hola.ph +# .hola.ph # expired .hola.sk .hola.org .holacdn.com diff --git a/Source/non_ip/my_plus.conf b/Source/non_ip/my_plus.conf index 6eedc09ee..15d8ee0b0 100644 --- a/Source/non_ip/my_plus.conf +++ b/Source/non_ip/my_plus.conf @@ -1,27 +1,7 @@ # $ meta_title Sukka's Ruleset - Internal Special # Video CDN -DOMAIN-SUFFIX,mapi.4kvm.net -DOMAIN-SUFFIX,v82u1l.com -DOMAIN-SUFFIX,whweitao.com -DOMAIN-SUFFIX,fsvod1.com -DOMAIN-SUFFIX,szjal.cn -DOMAIN-SUFFIX,hdzyk-cdn.com -DOMAIN-SUFFIX,wolongzywcdn3.com -DOMAIN-SUFFIX,ceres9350.com -DOMAIN-SUFFIX,wolongcdnm3u8.com -DOMAIN-SUFFIX,img.haiwaikan.com -DOMAIN-SUFFIX,cdn.iz8qkg.com -DOMAIN-SUFFIX,tokyvideo.com -DOMAIN-SUFFIX,hey10.cjkypo.com -DOMAIN-SUFFIX,yle888.vip -DOMAIN,static-cdn.bytegiftia.top -DOMAIN,cdn-ms.cosplayeringoodfunk.me -DOMAIN-SUFFIX,ffzy888.com -DOMAIN-SUFFIX,image8899.net -DOMAIN-SUFFIX,rrcdnbf3.com -DOMAIN-SUFFIX,myvideos.info -DOMAIN-SUFFIX,vplay.life + # SKK.MOE DOMAIN,hv-monster-submit.skk.moe # Wakatime From cba19c462e7a6d47f0443b62f55ce97ddc2ef45b Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 19 Dec 2024 22:30:14 +0800 Subject: [PATCH 9/9] Chore: minor changes AdGuardHome Mapping --- ...c-direct-lan-ruleset-dns-mapping-module.ts | 34 ++++++++++++------- Source/non_ip/domestic.ts | 9 +++++ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts index 8fd86cab6..e501aded1 100644 --- a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts +++ b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts @@ -1,6 +1,6 @@ // @ts-check import path from 'node:path'; -import { DOMESTICS, DOH_BOOTSTRAP } from '../Source/non_ip/domestic'; +import { DOMESTICS, DOH_BOOTSTRAP, AdGuardHomeDNSMapping } from '../Source/non_ip/domestic'; import { DIRECTS, LAN } from '../Source/non_ip/direct'; import type { DNSMapping } from '../Source/non_ip/direct'; import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; @@ -186,20 +186,28 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as span, [ '# Local DNS Mapping for AdGuard Home', - '', + 'tls://1.12.12.12', + 'tls://120.53.53.53', + 'https://1.12.12.12/dns-query', + 'https://120.53.53.53/dns-query', '[//]udp://10.10.1.1:53', - ...dataset.flatMap(({ domains, dns: _dns }) => domains.flatMap((domain) => { - const dns = _dns === 'system' - ? 'udp://10.10.1.1:53' - : _dns; - if ( - // AdGuard Home has built-in AS112 / private PTR handling - domain.endsWith('.arpa') - // Ignore simple hostname - || !domain.includes('.') - ) { - return []; + ...(([DOMESTICS, DIRECTS, LAN] as const).flatMap(Object.values) as DNSMapping[]).flatMap(({ domains, dns: _dns }) => domains.flatMap((domain) => { + let dns; + if (_dns in AdGuardHomeDNSMapping) { + dns = AdGuardHomeDNSMapping[_dns as keyof typeof AdGuardHomeDNSMapping].join(' '); + } else { + console.warn(`Unknown DNS "${_dns}" not in AdGuardHomeDNSMapping`); + dns = _dns; } + + // if ( + // // AdGuard Home has built-in AS112 / private PTR handling + // domain.endsWith('.arpa') + // // Ignore simple hostname + // || !domain.includes('.') + // ) { + // return []; + // } if (domain[0] === '$') { return [ `[/${domain.slice(1)}/]${dns}` diff --git a/Source/non_ip/domestic.ts b/Source/non_ip/domestic.ts index b22070036..fb00d5dee 100644 --- a/Source/non_ip/domestic.ts +++ b/Source/non_ip/domestic.ts @@ -392,3 +392,12 @@ export const DOH_BOOTSTRAP: Record = { ] } }; + +export const AdGuardHomeDNSMapping = { + system: ['udp://10.10.1.1:53'], + 'https://doh.pub/dns-query': ['tls://1.12.12.12', 'tls://120.53.53.53', 'https://1.12.12.12/dns-query', 'https://120.53.53.53/dns-query'], + 'quic://dns.alidns.com:853': ['quic://223.5.5.5', 'quic://223.6.6.6', 'h3://223.5.5.5/dns-query', 'h3://223.6.6.6/dns-query'], + 'https://doh.360.cn/dns-query': ['https://doh.360.cn/dns-query', 'tls://dot.360.cn'], + '180.76.76.76': ['udp://180.76.76.76'], + '180.184.2.2': ['udp://180.184.2.2', 'udp://180.184.1.1'] +};