diff --git a/Build/lib/parse-filter.ts b/Build/lib/parse-filter.ts index ddbfdab6f..22fb44ced 100644 --- a/Build/lib/parse-filter.ts +++ b/Build/lib/parse-filter.ts @@ -10,6 +10,7 @@ import { normalizeDomain } from './normalize-domain'; import { fetchAssets } from './fetch-assets'; import { deserializeSet, fsFetchCache, serializeSet } from './cache-filesystem'; import type { Span } from '../trace'; +import createKeywordFilter from './aho-corasick'; const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null let foundDebugDomain = false; @@ -78,9 +79,6 @@ export function processHosts(span: Span, hostsUrl: string, mirrors: string[] | n lineCb(l); } } else { - // Avoid event loop starvation, so we wait for a macrotask before we start fetching. - await Promise.resolve(); - const filterRules = await childSpan .traceChild('download hosts') .traceAsyncFn(() => fetchAssets(hostsUrl, mirrors).then(text => text.split('\n'))); @@ -198,9 +196,6 @@ export async function processFilterRules( lineCb(line); } } else { - // Avoid event loop starvation, so we wait for a macrotask before we start fetching. - await Promise.resolve(); - const filterRules = await span.traceChild('download adguard filter').traceAsyncFn(() => { return fetchAssets(filterRulesUrl, fallbackUrls).then(text => text.split('\n')); }); @@ -247,25 +242,37 @@ export async function processFilterRules( }; } -const R_KNOWN_NOT_NETWORK_FILTER_PATTERN = /[#%&=~]/; -const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder|\$cname)/; +// const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder|\$cname)/; // cname exceptional filter can not be parsed by NetworkFilter // Surge / Clash can't handle CNAME either, so we just ignore them +const kwfilter = createKeywordFilter([ + '!', + '?', + '*', + '[', + '(', + ']', + ')', + ',', + '#', + '%', + '&', + '=', + '~', + // special modifier + '$popup', + '$removeparam', + '$popunder', + '$cname' +]); + function parse($line: string, gorhill: PublicSuffixList): null | [hostname: string, flag: ParseType] { if ( // doesn't include !$line.includes('.') // rule with out dot can not be a domain // includes - || $line.includes('!') - || $line.includes('?') - || $line.includes('*') - || $line.includes('[') - || $line.includes('(') - || $line.includes(']') - || $line.includes(')') - || $line.includes(',') - || R_KNOWN_NOT_NETWORK_FILTER_PATTERN.test($line) + || kwfilter($line) ) { return null; } @@ -287,8 +294,6 @@ function parse($line: string, gorhill: PublicSuffixList): null | [hostname: stri || lastCharCode === 46 // 46 `.`, line.endsWith('.') || lastCharCode === 45 // 45 `-`, line.endsWith('-') || lastCharCode === 95 // 95 `_`, line.endsWith('_') - // special modifier - || R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2.test(line) // || line.includes('$popup') // || line.includes('$removeparam') // || line.includes('$popunder') diff --git a/Build/validate-gfwlist.ts b/Build/validate-gfwlist.ts index 72c25b4a5..6eb8f87b1 100644 --- a/Build/validate-gfwlist.ts +++ b/Build/validate-gfwlist.ts @@ -89,6 +89,7 @@ export const parseGfwList = async () => { }; await Promise.all([ + runAgainstRuleset(path.resolve(import.meta.dir, '../Source/non_ip/global.conf')), runAgainstRuleset(path.resolve(import.meta.dir, '../Source/non_ip/global_plus.conf')), runAgainstRuleset(path.resolve(import.meta.dir, '../List/non_ip/stream.conf')) ]); diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index 3ae823024..84e0ba050 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -787,6 +787,7 @@ cdn.wishpond.net app.viral-loops.com widget.cluster.groovehq.com bot.ivy.ai +cdn.dotaoutpost.com staticw2.yotpo.com cdn-yotpo-images-production.yotpo.com @@ -1120,6 +1121,8 @@ info.cloudflare.com cf-assets.www.cloudflare.com .nel.cloudflare.com favicons.teams.cloudflare.com +# ByteDance +sf-flow-web-cdn.ciciai.com # Tumblr assets.tumblr.com static.tumblr.com @@ -1425,6 +1428,8 @@ scripts.dailymail.co.uk # washington post gfx-data.news-engineering.aws.wapo.pub talk.washingtonpost.com +# booking.com +.bstatic.com # The Guardian i.guim.co.uk interactive.guim.co.uk @@ -2365,3 +2370,7 @@ cdn.ywxi.net .ezocdn.com static.reo.dev assets.buttondown.email +.uvcdn.com +.xda-cdn.com +static.underhentai.net +redircdn.com diff --git a/Source/domainset/reject_sukka.conf b/Source/domainset/reject_sukka.conf index efb9d7d0e..372a64010 100644 --- a/Source/domainset/reject_sukka.conf +++ b/Source/domainset/reject_sukka.conf @@ -796,7 +796,6 @@ vortex-win-sandbox.data.microsoft.com onecollector.cloudapp.aria.akadns.net prod.nexusrules.live.com.akadns.net vortex-sandbox.data.microsoft.com -.telemetry.microsoft.com .telemetry.urs.microsoft.com choice.microsoft.com redir.metaservices.microsoft.com @@ -813,6 +812,7 @@ cache.datamart.windows.com spynet2.microsoft.com spynetalt.microsoft.com .msads.net +bingads.microsoft.com # >> Google .2mdn.net @@ -1277,6 +1277,8 @@ adstats.tencentmusic.com adcdn.tencentmusic.com adexpo.tencentmusic.com adclick.tencentmusic.com +# qcloud P2P Video CDN +.qvb.qcloud.com # >> Sina ad4.sina.com.cn @@ -1496,6 +1498,8 @@ static.app.m.letv.com .mistat.india.xiaomi.com .mistat.rus.xiaomi.com +.api.miwifi.com + adv.sec.miui.com adv.sec.intl.miui.com misc.in.duokanbox.com diff --git a/Source/non_ip/domestic.conf b/Source/non_ip/domestic.conf index 06daea103..1d7d201ed 100644 --- a/Source/non_ip/domestic.conf +++ b/Source/non_ip/domestic.conf @@ -146,6 +146,7 @@ DOMAIN-SUFFIX,qy.net DOMAIN-SUFFIX,qiyipic.com # >> JD +DOMAIN-SUFFIX,3.cn DOMAIN-SUFFIX,360buyimg.com DOMAIN-SUFFIX,jd.com DOMAIN-SUFFIX,jd.com.cn @@ -395,6 +396,8 @@ DOMAIN-SUFFIX,huxiucdn.com DOMAIN-SUFFIX,huya.com DOMAIN-SUFFIX,ibreader.com DOMAIN-SUFFIX,icourse163.org +DOMAIN-SUFFIX,ifeng.com +DOMAIN-SUFFIX,ifengimg.com DOMAIN-SUFFIX,infinitynewtab.com DOMAIN-SUFFIX,ithome.com DOMAIN-SUFFIX,jianshu.com @@ -416,6 +419,8 @@ DOMAIN-SUFFIX,mobike.com DOMAIN-SUFFIX,msstatic.com DOMAIN-SUFFIX,mubu.com DOMAIN-SUFFIX,myzaker.com +DOMAIN-SUFFIX,oray.com +DOMAIN-SUFFIX,oray-img.com DOMAIN-SUFFIX,peiluyou.com DOMAIN-SUFFIX,pingan.com DOMAIN-SUFFIX,pplive.com diff --git a/Source/non_ip/domestic.ts b/Source/non_ip/domestic.ts index d7b799b73..f29060d4a 100644 --- a/Source/non_ip/domestic.ts +++ b/Source/non_ip/domestic.ts @@ -12,6 +12,7 @@ export const DOMESTICS = { 'aliyun.com', 'aliyuncs.com', 'alikunlun.com', + 'alikunlun.net', 'cdngslb.com', 'alipay.com', 'alipay.cn', @@ -46,6 +47,7 @@ export const DOMESTICS = { 'xiami.net', 'ykimg.com', 'youku.com', + 'tudou.com', 'cibntv.net', 'ele.me', 'elemecdn.com', @@ -87,6 +89,7 @@ export const DOMESTICS = { 'qq.com', 'qq.com.cn', 'qq.wang', + 'qqmail.cn', 'qqmail.com', 'qzone.com', 'tencent-cloud.net', @@ -127,6 +130,7 @@ export const DOMESTICS = { 'biliapi.net', 'biligame.cn', 'biligame.com', + 'biligame.net', 'bilicomics.com', 'bilibilipay.cn', 'bilibilipay.com' @@ -206,6 +210,7 @@ export const DOMESTICS = { 'iqiyi.com', 'iqiyipic.com', 'baidubce.com', + 'bcelive.com', 'baiducontent.com', 'baidustatic.com', 'bdstatic.com', diff --git a/Source/non_ip/global.conf b/Source/non_ip/global.conf index 74031f9ec..8ac82486e 100644 --- a/Source/non_ip/global.conf +++ b/Source/non_ip/global.conf @@ -67,6 +67,7 @@ DOMAIN-SUFFIX,github.blog DOMAIN-SUFFIX,github.community DOMAIN-SUFFIX,githubstatus.com DOMAIN-SUFFIX,git.io +DOMAIN-SUFFIX,githubapp.com # >> News DOMAIN-SUFFIX,globo.com @@ -116,7 +117,7 @@ DOMAIN-SUFFIX,codepen.io DOMAIN-SUFFIX,istatmenus.app -DOMAIN-SUFFIX,xing.com +DOMAIN-SUFFIX,themoviedb.org USER-AGENT,Roam* diff --git a/Source/non_ip/global_plus.conf b/Source/non_ip/global_plus.conf index a6d7d6858..1742a88bd 100644 --- a/Source/non_ip/global_plus.conf +++ b/Source/non_ip/global_plus.conf @@ -11,8 +11,10 @@ DOMAIN-SUFFIX,appspot.com DOMAIN-SUFFIX,blogger.com DOMAIN-SUFFIX,blogblog.com DOMAIN-SUFFIX,cloudfunctions.net +DOMAIN-SUFFIX,feedburner.com DOMAIN-SUFFIX,firebaseio.com DOMAIN-SUFFIX,getoutline.org +DOMAIN-SUFFIX,gcr.io DOMAIN-SUFFIX,gvt0.com DOMAIN-SUFFIX,gvt1.com DOMAIN-SUFFIX,gvt2.com @@ -166,6 +168,7 @@ DOMAIN-SUFFIX,dailymotion.com DOMAIN-SUFFIX,dailymail.co.uk DOMAIN-SUFFIX,deezer.com DOMAIN-SUFFIX,deviantart.com +DOMAIN-SUFFIX,deviantart.net DOMAIN-SUFFIX,disqus.com DOMAIN-SUFFIX,dropbox.com DOMAIN-SUFFIX,dropboxapi.com @@ -234,6 +237,7 @@ DOMAIN-SUFFIX,saucenao.com DOMAIN-SUFFIX,scribd.com DOMAIN-SUFFIX,shadowsocks.org DOMAIN-SUFFIX,shopee.tw +DOMAIN-SUFFIX,shutterstock.com DOMAIN-SUFFIX,slideshare.net DOMAIN-SUFFIX,soundcloud.com DOMAIN-SUFFIX,spankbang.com @@ -280,6 +284,7 @@ DOMAIN-SUFFIX,wsj.com DOMAIN-SUFFIX,wsj.net DOMAIN-SUFFIX,xfinity.com DOMAIN-SUFFIX,xhamster.com +DOMAIN-SUFFIX,xing.com DOMAIN-SUFFIX,xnxx.com DOMAIN-SUFFIX,xvideos.com DOMAIN-SUFFIX,xvideos-cdn.com