Codecs breakout 2

apps/chat/app/new/editor.tsx

@@ -45,7 +45,8 @@ import { env } from '@/lib/env'
 import { makeAgentSourceCode } from 'react-agents/util/agent-source-code-formatter.mjs';
 import { currencies, intervals } from 'react-agents/constants.mjs';
 import type { FetchableWorker } from 'react-agents-client/types';
-import { buildAgentSrc, ReactAgentsWorker } from 'react-agents-client';
+import { buildAgentSrc } from 'react-agents-client/builder';
+import { ReactAgentsWorker } from 'react-agents-client/runtime';
 import { IconButton } from 'ucom';
 
 //

apps/chat/components/account/voices.tsx

@@ -9,12 +9,11 @@ import { makeAnonymousClient } from '@/utils/supabase/supabase-client';
 import { getJWT } from '@/lib/jwt';
 import { env } from '@/lib/env'
 import { AutoVoiceEndpoint, VoiceEndpointVoicer } from 'react-agents/lib/voice-output/voice-endpoint-voicer.mjs';
-import { AudioDecodeStream } from '@upstreet/multiplayer/public/audio/audio-decode.mjs';
 import { AudioContextOutputStream } from '@/lib/audio/audio-context-output';
 import { aiProxyHost } from '../../utils/const/endpoints';
-import * as codecs from '@upstreet/multiplayer/public/audio/ws-codec-runtime-worker.mjs';
+import { AudioDecodeStream } from 'codecs/audio-decode.mjs';
+import * as codecs from 'codecs/ws-codec-runtime-worker.mjs';
 
-// const voicesEndpointProxyUrl = `https://${aiProxyHost}/api/ai/voices`;
 const voicesEndpointApiUrl = `https://${aiProxyHost}/api/ai-voice/voices`;
 
 export interface AgentsProps {

apps/chat/components/prompt-form.tsx

@@ -18,10 +18,9 @@ import type {
 } from 'react-agents/types';
 import { shuffle } from 'react-agents/util/util.mjs';
 import { Icon } from 'ucom';
-import { createPcmF32MicrophoneSource } from '@upstreet/multiplayer/public/audio/audio-client.mjs';
+import { createPcmF32MicrophoneSource } from 'codecs/audio-client.mjs';
 import { createVideoSource } from '@upstreet/multiplayer/public/video/video-client.mjs';
 import { ensureAudioContext } from '@/lib/audio/audio-context-output';
-// import { consoleImageWidth } from 'react-agents/constants.mjs'
 
 export function PromptForm({
   input,
@@ -272,8 +271,6 @@ export function PromptForm({
               className={cn("flex justify-start relative rounded bg-background mx-2 p-2 overflow-hidden")}
               onClick={async () => {
                 if (!screenSource) {
-                  // list the available mics
-                  const devices = await navigator.mediaDevices.enumerateDevices();
                   // console.log('got devices', devices);
                   // const videoInputDevices = devices.filter((device) => device.kind === 'videoinput');
                   const mediaStream = await navigator.mediaDevices.getDisplayMedia({

apps/chat/components/ui/multiplayer-actions.tsx

@@ -8,7 +8,6 @@ import { multiplayerEndpointUrl } from '@/utils/const/endpoints';
 import { getAgentEndpointUrl, getAgentHost } from '@/lib/utils'
 import { r2EndpointUrl } from '@/utils/const/endpoints';
 import { getJWT } from '@/lib/jwt';
-import { AudioDecodeStream } from '@upstreet/multiplayer/public/audio/audio-decode.mjs';
 import { AudioContextOutputStream } from '@/lib/audio/audio-context-output';
 import type {
   ActionMessage,
@@ -17,7 +16,8 @@ import type {
   PlayableVideoStream,
 } from 'react-agents/types';
 import { useLoading } from '@/lib/client/hooks/use-loading';
-import * as codecs from '@upstreet/multiplayer/public/audio/ws-codec-runtime-worker.mjs';
+import { AudioDecodeStream } from 'codecs/audio-decode.mjs';
+import * as codecs from 'codecs/ws-codec-runtime-worker.mjs';
 
 //
 

apps/chat/lib/audio/audio-context-output.ts

@@ -1,6 +1,3 @@
-// import { createMp3DecodeTransformStream } from '../lib/multiplayer/public/audio/audio-client.mjs';
-// import { QueueManager } from '../../utils/queue-manager.mjs';
-
 type LoadableAudioContext = AudioContext & {
   waitForLoad: () => Promise<void>;
 };

apps/chat/package.json

@@ -60,6 +60,7 @@
     "class-variance-authority": "0.7.0",
     "classnames": "2.5.1",
     "clsx": "2.1.0",
+    "codecs": "file:../../packages/usdk/packages/upstreet-agent/packages/react-agents/lib/codecs",
     "cookie": "^0.6.0",
     "d3-scale": "4.0.2",
     "date-fns": "3.3.1",

packages/usdk/cli.js

@@ -6,7 +6,7 @@ import stream from 'stream';
 import repl from 'repl';
 
 import { program } from 'commander';
-import WebSocket, { WebSocketServer } from 'ws';
+import WebSocket from 'ws';
 import EventSource from 'eventsource';
 import toml from '@iarna/toml';
 import open from 'open';
@@ -68,11 +68,9 @@ import {
 import { NetworkRealms } from './packages/upstreet-agent/packages/react-agents/lib/multiplayer/public/network-realms.mjs'; // XXX should be a deduplicated import, in a separate npm module
 
 import { AutoVoiceEndpoint, VoiceEndpointVoicer } from './packages/upstreet-agent/packages/react-agents/lib/voice-output/voice-endpoint-voicer.mjs';
-import { AudioDecodeStream } from './packages/upstreet-agent/packages/react-agents/lib/multiplayer/public/audio/audio-decode.mjs';
+import { AudioDecodeStream } from './packages/upstreet-agent/packages/react-agents/lib/codecs/audio-decode.mjs';
 
-// import * as codecs from './packages/upstreet-agent/packages/react-agents/lib/multiplayer/public/audio/ws-codec-runtime-worker.mjs';
-// import * as codecs from './packages/upstreet-agent/packages/react-agents/lib/multiplayer/public/audio/ws-codec-runtime-edge.mjs';
-import * as codecs from './packages/upstreet-agent/packages/react-agents/lib/multiplayer/public/audio/ws-codec-runtime-fs.mjs';
+import * as codecs from './packages/upstreet-agent/packages/react-agents/lib/codecs/ws-codec-runtime-fs.mjs';
 
 import { webbrowserActionsToText } from './packages/upstreet-agent/packages/react-agents/util/browser-action-utils.mjs';
 

packages/usdk/package.json

@@ -37,6 +37,7 @@
     "ansi-escapes": "^7.0.0",
     "chalk": "^5.3.0",
     "cli-table3": "^0.6.4",
+    "codecs": "file:./packages/upstreet-agent/packages/react-agents/lib/codecs",
     "commander": "^12.0.0",
     "cookie": "^0.6.0",
     "dedent": "^1.5.3",
@@ -49,7 +50,6 @@
     "jimp": "^0.22.12",
     "jszip": "^3.10.1",
     "memoize-one": "^6.0.0",
-    "multiplayer": "file:./packages/upstreet-agent/packages/react-agents/lib/multiplayer",
     "mime": "^4.0.1",
     "mkdirp": "^3.0.1",
     "open": "^10.1.0",

packages/usdk/packages/upstreet-agent/durable-object.tsx

@@ -1,7 +1,7 @@
 import { DurableObjectImpl } from './durable-object-impl.tsx';
 
 import userRender from '../../agent.tsx'; // note: this will be overwritten by the build process
-import * as codecs from 'react-agents/lib/multiplayer/public/audio/ws-codec-runtime-edge.mjs';
+import * as codecs from 'codecs/ws-codec-runtime-edge.mjs';
 
 Error.stackTraceLimit = 300;
 

packages/usdk/packages/upstreet-agent/package.json

@@ -7,6 +7,7 @@
     "@tsndr/cloudflare-worker-jwt": "2.5.3",
     "@types/react": "^18.3.3",
     "browser-util-inspect": "^0.2.0",
+    "codecs": "file:./packages/react-agents/lib/codecs",
     "dedent": "^1.5.3",
     "ethers": "^6.12.0",
     "format-util": "^1.0.5",
@@ -15,7 +16,6 @@
     "jimp": "^0.22.12",
     "memoize-one": "^6.0.0",
     "minimatch": "^9.0.4",
-    "multiplayer": "file:./packages/react-agents/lib/multiplayer",
     "openai": "^4.56.0",
     "react": "^19.0.0-rc-df5f2736-20240712",
     "react-agents": "file:./packages/react-agents",