[feat] 新增同傳字幕AI總結功能

.github/workflows/partial-test.yml

@@ -80,7 +80,8 @@ jobs:
                            --grep=@scoped \
                            --pass-with-no-tests \
                            --global-timeout=3600000 \
-                           --max-failures=2
+                           --max-failures=2 \
+                           --retries=3
             env:
               DEBUG: true
     fast-e2e-test:
@@ -122,7 +123,8 @@ jobs:
                              --pass-with-no-tests \
                              --global-timeout=3600000 \
                              --timeout=60000 \
-                             --max-failures=5
+                             --max-failures=5 \
+                             --retries=3
               env:
                 DEBUG: true
             - name: Upload Test Results

package.json

@@ -27,6 +27,7 @@
     "@ffmpeg/ffmpeg": "^0.12.10",
     "@ffmpeg/util": "^0.12.1",
     "@material-tailwind/react": "^2.1.9",
+    "@mlc-ai/web-llm": "^0.2.73",
     "@plasmohq/messaging": "^0.6.2",
     "@plasmohq/storage": "^1.9.3",
     "@react-hooks-library/core": "^0.5.2",
@@ -36,6 +37,7 @@
     "dexie-react-hooks": "^1.1.7",
     "hash-wasm": "^4.11.0",
     "hls.js": "^1.5.8",
+    "markdown-to-jsx": "^7.5.0",
     "media-chrome": "^2.2.5",
     "mpegts.js": "^1.7.3",
     "n-danmaku": "^2.2.1",
@@ -63,6 +65,7 @@
     "@types/react": "18.2.37",
     "@types/react-dom": "18.2.15",
     "@types/semver": "^7.5.8",
+    "@webgpu/types": "^0.1.49",
     "dotenv": "^16.4.5",
     "esbuild": "^0.20.2",
     "gify-parse": "^1.0.7",
@@ -82,7 +85,8 @@
       "*://api.live.bilibili.com/*",
       "*://live.bilibili.com/*",
       "*://*.bilivideo.com/*",
-      "*://*.ericlamm.xyz/*"
+      "*://*.ericlamm.xyz/*",
+      "*://*.cloudflare.com/*"
     ],
     "permissions": [
       "notifications",

src/api/cloudflare.ts

@@ -0,0 +1,52 @@
+import type { AIResponse, Result } from "~types/cloudflare";
+import { parseSSEResponses } from "~utils/binary";
+
+const BASE_URL = 'https://api.cloudflare.com/client/v4'
+
+export async function runAI(data: any, { token, account, model }: { token: string, account: string, model: string }): Promise<Result<AIResponse>> {
+    const res = await fetch(`${BASE_URL}/accounts/${account}/ai/run/${model}`, {
+        method: 'POST',
+        headers: {
+            Authorization: `Bearer ${token}`
+        },
+        body: JSON.stringify({ ...data, stream: false })
+    })
+    const json = await res.json() as Result<AIResponse>
+    if (!res.ok) throw new Error(json.errors.join('\n'))
+    return json
+}
+
+export async function* runAIStream(data: any, { token, account, model }: { token: string, account: string, model: string }): AsyncGenerator<string> {
+    const res = await fetch(`${BASE_URL}/accounts/${account}/ai/run/${model}`, {
+        method: 'POST',
+        headers: {
+            Authorization: `Bearer ${token}`
+        },
+        body: JSON.stringify({ ...data, stream: true })
+    })
+    if (!res.ok) {
+        const json = await res.json() as Result<AIResponse>
+        throw new Error(json.errors.join('\n'))
+    }
+    if (!res.body) throw new Error('Cloudflare AI response body is not readable')
+    const reader = res.body.getReader()
+    for await (const response of parseSSEResponses(reader, '[DONE]')) {
+        yield response
+    }
+}
+
+export async function validateAIToken(accountId: string, token: string, model: string): Promise<string | boolean> {
+    const res = await fetch(`${BASE_URL}/accounts/${accountId}/ai/models/search?search=${model}&per_page=1`, {
+        headers: {
+            Authorization: `Bearer ${token}`
+        }
+    })
+    const data = await res.json() as Result<any>
+    if (!data.success) {
+        return false
+    } else if (data.result.length === 0) {
+        return '找不到指定 AI 模型'
+    } else {
+        return true
+    }
+}

src/background/forwards.ts

@@ -4,6 +4,7 @@ import * as danmaku from './forwards/danmaku'
 import * as jimaku from './forwards/jimaku'
 import * as redirect from './forwards/redirect'
 import * as streamContent from './forwards/stream-content'
+import * as jimakuSummarize from './forwards/summerize'
 
 export type ForwardData = typeof forwards
 
@@ -89,6 +90,9 @@ export function isForwardMessage<T extends object>(message: any): message is For
  * forwarder.addHandler((data) => {
  *   console.log('Received message:', data)
  * })
+ * forwarder.addHandlerOnce((data) => {
+ *  console.log('Received message:', data)
+ * })
  * forwarder.sendForward('background', { message: 'Hello' })
  */
 export function getForwarder<K extends keyof ForwardData>(command: K, target: ChannelType): Forwarder<K> {
@@ -131,6 +135,14 @@ export function getForwarder<K extends keyof ForwardData>(command: K, target: Ch
             chrome.runtime.onMessage.addListener(fn)
             return () => chrome.runtime.onMessage.removeListener(fn)
         },
+        addHandlerOnce: (handler: (data: R) => void): VoidCallback => {
+            const fn = listener((data: R) => {
+                handler(data)
+                chrome.runtime.onMessage.removeListener(fn)
+            })
+            chrome.runtime.onMessage.addListener(fn)
+            return () => chrome.runtime.onMessage.removeListener(fn)
+        },
         sendForward: <C extends ChannelType>(toTarget: C, body: T, queryInfo?: ChannelQueryInfo[C]): void => {
             sendForward<K, T, C>(toTarget, command, body, queryInfo)
         }
@@ -144,6 +156,7 @@ export function useDefaultHandler<T extends object>(): ForwardHandler<T> {
 
 export type Forwarder<K extends keyof ForwardData> = {
     addHandler: (handler: (data: ForwardResponse<ForwardData[K]>) => void) => VoidCallback
+    addHandlerOnce: (handler: (data: ForwardResponse<ForwardData[K]>) => void) => VoidCallback
     sendForward: <C extends ChannelType>(toTarget: C, body: ForwardBody<ForwardData[K]>, queryInfo?: ChannelQueryInfo[C]) => void
 }
 
@@ -153,5 +166,6 @@ const forwards = {
     'redirect': redirect,
     'danmaku': danmaku,
     'blive-data': bliveData,
-    'stream-content': streamContent
+    'stream-content': streamContent,
+    'jimaku-summarize': jimakuSummarize
 }

src/background/forwards/summerize.ts

@@ -0,0 +1,8 @@
+import { useDefaultHandler } from "~background/forwards"
+
+export type ForwardBody = {
+    roomId: string
+    jimakus: string[]
+}
+
+export default useDefaultHandler<ForwardBody>()

src/background/messages/open-tab.ts

@@ -5,18 +5,26 @@ export type RequestBody = {
     tab?: string
     active?: boolean
     params?: Record<string, string>
+    singleton?: boolean | string[]
 }
 
 const handler: PlasmoMessaging.MessageHandler<RequestBody, chrome.tabs.Tab> = async (req, res) => {
     const { url, tab, active } = req.body
     const queryString = req.body.params ? `?${new URLSearchParams(req.body.params).toString()}` : ''
-    const result = await chrome.tabs.create({
-        url: tab ?
-            chrome.runtime.getURL(`/tabs/${tab}.html${queryString}`) :
-            url + queryString,
-        active
-    })
-    res.send(result)
+    const fullUrl = tab ? chrome.runtime.getURL(`/tabs/${tab}.html${queryString}`) : url + queryString
+    const pathUrl = (tab ? chrome.runtime.getURL(`/tabs/${tab}.html`) : url) + '*'
+    if (req.body.singleton) {
+        const tabs = await chrome.tabs.query({ url: typeof req.body.singleton === 'boolean' ? fullUrl : pathUrl })
+        const tab = tabs.find(tab =>
+            typeof req.body.singleton === 'boolean' ||
+            req.body.singleton.some(param => new URL(tab.url).searchParams.get(param) === req.body.params[param])
+        )
+        if (tab) {
+            res.send(await chrome.tabs.update(tab.id, { active: true }))
+            return
+        }
+    }
+    res.send(await chrome.tabs.create({ url: fullUrl, active }))
 }