Merge branch 'SukkaW:master' into master

FYLSen · Feb 3, 2024 · c0b27e9 · c0b27e9
2 parents 10445aa + 124d8f1
commit c0b27e9
Show file tree

Hide file tree

Showing 8 changed files with 56 additions and 21 deletions.
diff --git a/Build/lib/parse-filter.ts b/Build/lib/parse-filter.ts
@@ -10,6 +10,7 @@ import { normalizeDomain } from './normalize-domain';
 import { fetchAssets } from './fetch-assets';
 import { deserializeSet, fsFetchCache, serializeSet } from './cache-filesystem';
 import type { Span } from '../trace';
+import createKeywordFilter from './aho-corasick';
 
 const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null
 let foundDebugDomain = false;
@@ -78,9 +79,6 @@ export function processHosts(span: Span, hostsUrl: string, mirrors: string[] | n
           lineCb(l);
         }
       } else {
-        // Avoid event loop starvation, so we wait for a macrotask before we start fetching.
-        await Promise.resolve();
-
         const filterRules = await childSpan
           .traceChild('download hosts')
           .traceAsyncFn(() => fetchAssets(hostsUrl, mirrors).then(text => text.split('\n')));
@@ -198,9 +196,6 @@ export async function processFilterRules(
           lineCb(line);
         }
       } else {
-        // Avoid event loop starvation, so we wait for a macrotask before we start fetching.
-        await Promise.resolve();
-
         const filterRules = await span.traceChild('download adguard filter').traceAsyncFn(() => {
           return fetchAssets(filterRulesUrl, fallbackUrls).then(text => text.split('\n'));
         });
@@ -247,25 +242,37 @@ export async function processFilterRules(
   };
 }
 
-const R_KNOWN_NOT_NETWORK_FILTER_PATTERN = /[#%&=~]/;
-const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder|\$cname)/;
+// const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder|\$cname)/;
 // cname exceptional filter can not be parsed by NetworkFilter
 // Surge / Clash can't handle CNAME either, so we just ignore them
 
+const kwfilter = createKeywordFilter([
+  '!',
+  '?',
+  '*',
+  '[',
+  '(',
+  ']',
+  ')',
+  ',',
+  '#',
+  '%',
+  '&',
+  '=',
+  '~',
+  // special modifier
+  '$popup',
+  '$removeparam',
+  '$popunder',
+  '$cname'
+]);
+
 function parse($line: string, gorhill: PublicSuffixList): null | [hostname: string, flag: ParseType] {
   if (
     // doesn't include
     !$line.includes('.') // rule with out dot can not be a domain
     // includes
-    || $line.includes('!')
-    || $line.includes('?')
-    || $line.includes('*')
-    || $line.includes('[')
-    || $line.includes('(')
-    || $line.includes(']')
-    || $line.includes(')')
-    || $line.includes(',')
-    || R_KNOWN_NOT_NETWORK_FILTER_PATTERN.test($line)
+    || kwfilter($line)
   ) {
     return null;
   }
@@ -287,8 +294,6 @@ function parse($line: string, gorhill: PublicSuffixList): null | [hostname: stri
     || lastCharCode === 46 // 46 `.`, line.endsWith('.')
     || lastCharCode === 45 // 45 `-`, line.endsWith('-')
     || lastCharCode === 95 // 95 `_`, line.endsWith('_')
-    // special modifier
-    || R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2.test(line)
     // || line.includes('$popup')
     // || line.includes('$removeparam')
     // || line.includes('$popunder')

diff --git a/Build/validate-gfwlist.ts b/Build/validate-gfwlist.ts
@@ -89,6 +89,7 @@ export const parseGfwList = async () => {
   };
 
   await Promise.all([
+    runAgainstRuleset(path.resolve(import.meta.dir, '../Source/non_ip/global.conf')),
     runAgainstRuleset(path.resolve(import.meta.dir, '../Source/non_ip/global_plus.conf')),
     runAgainstRuleset(path.resolve(import.meta.dir, '../List/non_ip/stream.conf'))
   ]);

diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf
@@ -787,6 +787,7 @@ cdn.wishpond.net
 app.viral-loops.com
 widget.cluster.groovehq.com
 bot.ivy.ai
+cdn.dotaoutpost.com
 
 staticw2.yotpo.com
 cdn-yotpo-images-production.yotpo.com
@@ -1120,6 +1121,8 @@ info.cloudflare.com
 cf-assets.www.cloudflare.com
 .nel.cloudflare.com
 favicons.teams.cloudflare.com
+# ByteDance
+sf-flow-web-cdn.ciciai.com
 # Tumblr
 assets.tumblr.com
 static.tumblr.com
@@ -1425,6 +1428,8 @@ scripts.dailymail.co.uk
 # washington post
 gfx-data.news-engineering.aws.wapo.pub
 talk.washingtonpost.com
+# booking.com
+.bstatic.com
 # The Guardian
 i.guim.co.uk
 interactive.guim.co.uk
@@ -2365,3 +2370,7 @@ cdn.ywxi.net
 .ezocdn.com
 static.reo.dev
 assets.buttondown.email
+.uvcdn.com
+.xda-cdn.com
+static.underhentai.net
+redircdn.com
diff --git a/Source/domainset/reject_sukka.conf b/Source/domainset/reject_sukka.conf
@@ -796,7 +796,6 @@ vortex-win-sandbox.data.microsoft.com
 onecollector.cloudapp.aria.akadns.net
 prod.nexusrules.live.com.akadns.net
 vortex-sandbox.data.microsoft.com
-.telemetry.microsoft.com
 .telemetry.urs.microsoft.com
 choice.microsoft.com
 redir.metaservices.microsoft.com
@@ -813,6 +812,7 @@ cache.datamart.windows.com
 spynet2.microsoft.com
 spynetalt.microsoft.com
 .msads.net
+bingads.microsoft.com
 
 # >> Google
 .2mdn.net
@@ -1277,6 +1277,8 @@ adstats.tencentmusic.com
 adcdn.tencentmusic.com
 adexpo.tencentmusic.com
 adclick.tencentmusic.com
+# qcloud P2P Video CDN
+.qvb.qcloud.com
 
 # >> Sina
 ad4.sina.com.cn
@@ -1496,6 +1498,8 @@ static.app.m.letv.com
 .mistat.india.xiaomi.com
 .mistat.rus.xiaomi.com
 
+.api.miwifi.com
+
 adv.sec.miui.com
 adv.sec.intl.miui.com
 misc.in.duokanbox.com

diff --git a/Source/non_ip/domestic.conf b/Source/non_ip/domestic.conf
@@ -146,6 +146,7 @@ DOMAIN-SUFFIX,qy.net
 DOMAIN-SUFFIX,qiyipic.com
 
 # >> JD
+DOMAIN-SUFFIX,3.cn
 DOMAIN-SUFFIX,360buyimg.com
 DOMAIN-SUFFIX,jd.com
 DOMAIN-SUFFIX,jd.com.cn
@@ -395,6 +396,8 @@ DOMAIN-SUFFIX,huxiucdn.com
 DOMAIN-SUFFIX,huya.com
 DOMAIN-SUFFIX,ibreader.com
 DOMAIN-SUFFIX,icourse163.org
+DOMAIN-SUFFIX,ifeng.com
+DOMAIN-SUFFIX,ifengimg.com
 DOMAIN-SUFFIX,infinitynewtab.com
 DOMAIN-SUFFIX,ithome.com
 DOMAIN-SUFFIX,jianshu.com
@@ -416,6 +419,8 @@ DOMAIN-SUFFIX,mobike.com
 DOMAIN-SUFFIX,msstatic.com
 DOMAIN-SUFFIX,mubu.com
 DOMAIN-SUFFIX,myzaker.com
+DOMAIN-SUFFIX,oray.com
+DOMAIN-SUFFIX,oray-img.com
 DOMAIN-SUFFIX,peiluyou.com
 DOMAIN-SUFFIX,pingan.com
 DOMAIN-SUFFIX,pplive.com

diff --git a/Source/non_ip/domestic.ts b/Source/non_ip/domestic.ts
@@ -12,6 +12,7 @@ export const DOMESTICS = {
       'aliyun.com',
       'aliyuncs.com',
       'alikunlun.com',
+      'alikunlun.net',
       'cdngslb.com',
       'alipay.com',
       'alipay.cn',
@@ -46,6 +47,7 @@ export const DOMESTICS = {
       'xiami.net',
       'ykimg.com',
       'youku.com',
+      'tudou.com',
       'cibntv.net',
       'ele.me',
       'elemecdn.com',
@@ -87,6 +89,7 @@ export const DOMESTICS = {
       'qq.com',
       'qq.com.cn',
       'qq.wang',
+      'qqmail.cn',
       'qqmail.com',
       'qzone.com',
       'tencent-cloud.net',
@@ -127,6 +130,7 @@ export const DOMESTICS = {
       'biliapi.net',
       'biligame.cn',
       'biligame.com',
+      'biligame.net',
       'bilicomics.com',
       'bilibilipay.cn',
       'bilibilipay.com'
@@ -206,6 +210,7 @@ export const DOMESTICS = {
       'iqiyi.com',
       'iqiyipic.com',
       'baidubce.com',
+      'bcelive.com',
       'baiducontent.com',
       'baidustatic.com',
       'bdstatic.com',

diff --git a/Source/non_ip/global.conf b/Source/non_ip/global.conf
@@ -67,6 +67,7 @@ DOMAIN-SUFFIX,github.blog
 DOMAIN-SUFFIX,github.community
 DOMAIN-SUFFIX,githubstatus.com
 DOMAIN-SUFFIX,git.io
+DOMAIN-SUFFIX,githubapp.com
 
 # >> News
 DOMAIN-SUFFIX,globo.com
@@ -116,7 +117,7 @@ DOMAIN-SUFFIX,codepen.io
 
 DOMAIN-SUFFIX,istatmenus.app
 
-DOMAIN-SUFFIX,xing.com
+DOMAIN-SUFFIX,themoviedb.org
 
 USER-AGENT,Roam*
 

diff --git a/Source/non_ip/global_plus.conf b/Source/non_ip/global_plus.conf
@@ -11,8 +11,10 @@ DOMAIN-SUFFIX,appspot.com
 DOMAIN-SUFFIX,blogger.com
 DOMAIN-SUFFIX,blogblog.com
 DOMAIN-SUFFIX,cloudfunctions.net
+DOMAIN-SUFFIX,feedburner.com
 DOMAIN-SUFFIX,firebaseio.com
 DOMAIN-SUFFIX,getoutline.org
+DOMAIN-SUFFIX,gcr.io
 DOMAIN-SUFFIX,gvt0.com
 DOMAIN-SUFFIX,gvt1.com
 DOMAIN-SUFFIX,gvt2.com
@@ -166,6 +168,7 @@ DOMAIN-SUFFIX,dailymotion.com
 DOMAIN-SUFFIX,dailymail.co.uk
 DOMAIN-SUFFIX,deezer.com
 DOMAIN-SUFFIX,deviantart.com
+DOMAIN-SUFFIX,deviantart.net
 DOMAIN-SUFFIX,disqus.com
 DOMAIN-SUFFIX,dropbox.com
 DOMAIN-SUFFIX,dropboxapi.com
@@ -234,6 +237,7 @@ DOMAIN-SUFFIX,saucenao.com
 DOMAIN-SUFFIX,scribd.com
 DOMAIN-SUFFIX,shadowsocks.org
 DOMAIN-SUFFIX,shopee.tw
+DOMAIN-SUFFIX,shutterstock.com
 DOMAIN-SUFFIX,slideshare.net
 DOMAIN-SUFFIX,soundcloud.com
 DOMAIN-SUFFIX,spankbang.com
@@ -280,6 +284,7 @@ DOMAIN-SUFFIX,wsj.com
 DOMAIN-SUFFIX,wsj.net
 DOMAIN-SUFFIX,xfinity.com
 DOMAIN-SUFFIX,xhamster.com
+DOMAIN-SUFFIX,xing.com
 DOMAIN-SUFFIX,xnxx.com
 DOMAIN-SUFFIX,xvideos.com
 DOMAIN-SUFFIX,xvideos-cdn.com