Plugins externalized (WIP)

.github/workflows/usdk-test.yml

@@ -10,6 +10,9 @@ on:
 jobs:
   test-usdk:
     runs-on: ubuntu-22.04
+    env:
+      FAL_API_KEY: ${{ secrets.FAL_API_KEY }}
+
     strategy:
       matrix:
         node-version: [22]
@@ -28,3 +31,5 @@ jobs:
         run: cd packages/usdk && pnpm install --no-optional --no-frozen-lockfile
       - name: Run USDK command
         run: cd packages/usdk && ./usdk.js --version
+      - name: Run Jest tests
+        run: cd packages/usdk && pnpm test

apps/chat/next.config.js

@@ -2,6 +2,7 @@
 
 const path = require('path');
 const webpack = require('webpack');
+const { builtinModules } = require('module');
 
 module.exports = {
   // experimental: {
@@ -14,6 +15,7 @@ module.exports = {
     'react-agents-client',
     'ecctrl',
     'ucom',
+    '@elizaos/core',
   ],
   async redirects() {
     return [
@@ -88,28 +90,42 @@ module.exports = {
       use: [options.defaultLoaders.babel],
     }); */
 
+    config.plugins.push(
+      new webpack.NormalModuleReplacementPlugin(/^node:(.*)$/, (resource) => {
+        resource.request = resource.request.replace(/^node:/, '');
+      })
+    );
+
     // fix react resolution in sdk subpackage
     const usdkPath = path.resolve(__dirname, '../../packages/usdk');
     const upstreetAgentPath = path.resolve(__dirname, '../../packages/usdk/packages/upstreet-agent');
     const reactAgentsPath = path.resolve(__dirname, '../../packages/usdk/packages/upstreet-agent/packages/react-agents');
     const reactAgentsClientPath = path.resolve(__dirname, '../../packages/usdk/packages/upstreet-agent/packages/react-agents-client');
     const reactAgentsBrowserPath = path.resolve(__dirname, '../../packages/usdk/packages/upstreet-agent/packages/react-agents-browser');
-    const replacePlugin = (scopePath, moduleRegexp) => {
+    const replacePlugin = (scopePath, moduleRegexp, importPath) => {
       return new webpack.NormalModuleReplacementPlugin(moduleRegexp, (resource) => {
-        if (resource.context.includes(scopePath)) {
+        if (
+          (typeof scopePath == 'string' && resource.context.includes(scopePath)) ||
+          (scopePath instanceof RegExp && scopePath.test(resource.context))
+        ) {
+          if (builtinModules.includes(resource.request)) {
+            resource.request = path.join(importPath, 'node_modules', resource.request);
+          }
           const p = require.resolve(resource.request, {
-            paths: [scopePath],
+            paths: [importPath],
           });
           resource.request = p;
         }
       });
     };
+    const modulesRegex = /^react|^fs$|^child_process$/;
     config.plugins.push(
-      replacePlugin(reactAgentsPath, /^react/),
-      replacePlugin(reactAgentsClientPath, /^react/),
-      replacePlugin(reactAgentsBrowserPath, /^react/),
-      replacePlugin(upstreetAgentPath, /^react/),
-      replacePlugin(usdkPath, /^react/),
+      replacePlugin(reactAgentsPath, modulesRegex, upstreetAgentPath),
+      replacePlugin(reactAgentsClientPath, modulesRegex, upstreetAgentPath),
+      replacePlugin(reactAgentsBrowserPath, modulesRegex, upstreetAgentPath),
+      replacePlugin(upstreetAgentPath, modulesRegex, upstreetAgentPath),
+      replacePlugin(usdkPath, modulesRegex, upstreetAgentPath),
+      replacePlugin(/\/path-scurry\/|\/0g-ts-sdk\//, modulesRegex, upstreetAgentPath),
     );
 
     // config.experiments = {
@@ -123,6 +139,11 @@ module.exports = {
       type: 'asset/resource',
     });
 
+    config.module.rules.push({
+      test: /\.cdc$/,
+      type: 'asset/source'
+    });
+
     return config;
   },
   async headers() {

package.json

@@ -4,9 +4,11 @@
   "private": true,
   "type": "module",
   "scripts": {
+    "test": "npm run usdk test",
     "clean": "find . -type d -name node_modules -exec rm -rf {} +",
     "chat": "npm run -w @upstreet/chat --",
-    "docs": "npm run -w @upstreet/docs --"
+    "docs": "npm run -w @upstreet/docs --",
+    "usdk": "npm run -w usdk --"
   },
   "workspaces": [
     "apps/*",

packages/usdk/jest.config.js

@@ -0,0 +1,10 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} **/
+export default {
+  preset: "ts-jest/presets/default-esm",
+  testEnvironment: "node",
+  transformIgnorePatterns: [],
+  testMatch: [
+    '**/test/?(*.)+(spec|test).[jt]s?(x)',
+  ],
+  extensionsToTreatAsEsm: [".ts", ".tsx"],
+};

packages/usdk/package.json

@@ -18,7 +18,8 @@
   "type": "module",
   "main": "module.mjs",
   "scripts": {
-    "preinstall": "[[ $npm_config_global == 'true' ]] || exit 0; cd packages/upstreet-agent && pnpm install --ignore-workspace --no-frozen-lockfile"
+    "preinstall": "[[ $npm_config_global == 'true' ]] || exit 0; cd packages/upstreet-agent && pnpm install --ignore-workspace --no-frozen-lockfile",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testTimeout 600000"
   },
   "keywords": [
     "ai",
@@ -60,6 +61,7 @@
     "@supabase/supabase-js": "^2.47.1",
     "@tsndr/cloudflare-worker-jwt": "2.5.3",
     "@types/estree": "^1.0.5",
+    "@types/react": "19.0.0",
     "@wasm-audio-decoders/common": "file:./packages/upstreet-agent/packages/codecs/packages/mpg123-decoder/packages/wasm-audio-decoders-common",
     "alea": "^1.0.1",
     "ansi-escape-sequences": "^6.2.2",
@@ -103,6 +105,7 @@
     "png-chunks-extract": "^1.0.0",
     "pnpm": "^9.12.3",
     "pretty-bytes": "^6.1.1",
+    "react": "file:./packages/upstreet-agent/packages/react",
     "react-agents": "file:./packages/upstreet-agent/packages/react-agents",
     "queue-manager": "file:./packages/upstreet-agent/packages/queue-manager",
     "react-agents-electron": "file:./packages/upstreet-agent/packages/react-agents-electron",
@@ -130,7 +133,6 @@
     "web-worker": "1.2.0",
     "webp-wasm": "^1.0.6",
     "winston": "^3.16.0",
-    "wrangler": "^3.95.0",
     "ws": "^8.17.0",
     "yjs": "^13.6.18",
     "zjs": "file:./packages/upstreet-agent/packages/react-agents-client/packages/multiplayer/packages/zjs",
@@ -150,7 +152,9 @@
     "whatwg-url": "14.0.0"
   },
   "devDependencies": {
+    "@types/jest": "^29.5.14",
     "@types/node": "^18.0.0",
-    "jest": "^29.0.0"
+    "jest": "^29.0.0",
+    "ts-jest": "^29.2.5"
   }
 }

packages/usdk/packages/upstreet-agent/durable-object.tsx

@@ -1,3 +1,5 @@
+import './init.ts';
+import './main.jsx';
 import React from 'react';
 import dotenv from 'dotenv';
 import { createRoot, Root } from 'react-agents/root.ts';
@@ -6,8 +8,6 @@ import App from './agent.tsx';
 import agentJsonSource from './agent.json';
 import envTxt from './.env.txt';
 
-Error.stackTraceLimit = 300;
-
 const alarmTimeout = 10 * 1000;
 
 const parseAgentJson = (agentJsonSource) => {

packages/usdk/packages/upstreet-agent/init.ts

@@ -0,0 +1,7 @@
+Error.stackTraceLimit = 300;
+globalThis.__filename = 'node.js';
+globalThis.__dirname = '/';
+(globalThis.process as any).version = '20.0.0';
+if (typeof import.meta.url === 'undefined') {
+  import.meta.url = `file://localhost/${globalThis.__filename}`;
+}

packages/usdk/packages/upstreet-agent/main.jsx

@@ -1,9 +1,8 @@
+import './init.ts';
 export { DurableObject } from './durable-object.tsx';
 import { headers } from 'react-agents/constants.mjs';
 import agentJsonSource from './agent.json';
 
-Error.stackTraceLimit = 300;
-
 const parseAgentJson = (agentJsonSource) => {
   try {
     if (typeof agentJsonSource === 'string') {

packages/usdk/packages/upstreet-agent/package.json

@@ -4,21 +4,27 @@
   "dependencies": {
     "@aws-sdk/util-format-url": "^3.577.0",
     "@electric-sql/pglite": "^0.2.15",
+    "@elizaos/core": "file:./packages/elizaos-core-proxy",
     "@iarna/toml": "^2.2.5",
     "@supabase/postgrest-js": "^1.17.7",
     "@supabase/supabase-js": "^2.47.1",
     "@tsndr/cloudflare-worker-jwt": "2.5.3",
     "@types/jest": "^29.5.13",
+    "@types/react": "19.0.0",
     "browser-util-inspect": "^0.2.0",
+    "child_process": "file:./packages/fs-proxy",
     "codecs": "file:./packages/codecs",
     "debouncer": "file:./packages/debouncer",
     "dedent": "^1.5.3",
     "dotenv": "^16.4.5",
     "ethers": "^6.12.0",
     "format-util": "^1.0.5",
+    "fs": "file:./packages/fs-proxy",
     "javascript-time-ago": "^2.5.11",
     "jest": "^29.0.0",
     "jimp": "^1.6.0",
+    "json-schema-to-zod-object": "^2.6.7",
+    "memfs": "^4.15.1",
     "memoize-one": "^6.0.0",
     "minimatch": "^9.0.4",
     "openai": "^4.56.0",
@@ -31,6 +37,7 @@
     "react-agents-node": "file:./packages/react-agents-node",
     "react-reconciler": "file:./packages/react-reconciler",
     "stripe": "^16.10.0",
+    "tls": "file:./packages/tls-proxy",
     "together-ai": "^0.6.0-alpha.4",
     "ts-jest": "^29.2.5",
     "twitter-api-sdk": "^1.2.1",

packages/usdk/packages/upstreet-agent/packages/child-process-proxy/child-process.mjs

@@ -0,0 +1,7 @@
+export const spawn = () => {
+  throw new Error('not implemented');
+};
+const childProcess = {
+  spawn,
+};
+export default childProcess;

packages/usdk/packages/upstreet-agent/packages/child-process-proxy/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "child_process",
+  "main": "child-process.mjs"
+}

packages/usdk/packages/upstreet-agent/packages/elizaos-core-proxy/context.ts

@@ -0,0 +1,104 @@
+import handlebars from "handlebars";
+import { type State } from "./types.ts";
+import { names, uniqueNamesGenerator } from "unique-names-generator";
+
+/**
+ * Composes a context string by replacing placeholders in a template with corresponding values from the state.
+ *
+ * This function takes a template string with placeholders in the format `{{placeholder}}` and a state object.
+ * It replaces each placeholder with the value from the state object that matches the placeholder's name.
+ * If a matching key is not found in the state object for a given placeholder, the placeholder is replaced with an empty string.
+ *
+ * By default, this function uses a simple string replacement approach. However, when `templatingEngine` is set to `'handlebars'`, it uses Handlebars templating engine instead, compiling the template into a reusable function and evaluating it with the provided state object.
+ *
+ * @param {Object} params - The parameters for composing the context.
+ * @param {State} params.state - The state object containing values to replace the placeholders in the template.
+ * @param {string} params.template - The template string containing placeholders to be replaced with state values.
+ * @param {"handlebars" | undefined} [params.templatingEngine] - The templating engine to use for compiling and evaluating the template (optional, default: `undefined`).
+ * @returns {string} The composed context string with placeholders replaced by corresponding state values.
+ *
+ * @example
+ * // Given a state object and a template
+ * const state = { userName: "Alice", userAge: 30 };
+ * const template = "Hello, {{userName}}! You are {{userAge}} years old";
+ *
+ * // Composing the context with simple string replacement will result in:
+ * // "Hello, Alice! You are 30 years old."
+ * const contextSimple = composeContext({ state, template });
+ */
+export const composeContext = ({
+    state,
+    template,
+    templatingEngine,
+}: {
+    state: State;
+    template: string;
+    templatingEngine?: "handlebars";
+}) => {
+    if (templatingEngine === "handlebars") {
+        const templateFunction = handlebars.compile(template);
+        return templateFunction(state);
+    }
+
+    // @ts-expect-error match isn't working as expected
+    const out = template.replace(/{{\w+}}/g, (match) => {
+        const key = match.replace(/{{|}}/g, "");
+        return state[key] ?? "";
+    });
+    return out;
+};
+
+/**
+ * Adds a header to a body of text.
+ *
+ * This function takes a header string and a body string and returns a new string with the header prepended to the body.
+ * If the body string is empty, the header is returned as is.
+ *
+ * @param {string} header - The header to add to the body.
+ * @param {string} body - The body to which to add the header.
+ * @returns {string} The body with the header prepended.
+ *
+ * @example
+ * // Given a header and a body
+ * const header = "Header";
+ * const body = "Body";
+ *
+ * // Adding the header to the body will result in:
+ * // "Header\nBody"
+ * const text = addHeader(header, body);
+ */
+export const addHeader = (header: string, body: string) => {
+    return body.length > 0 ? `${header ? header + "\n" : header}${body}\n` : "";
+};
+
+/**
+ * Generates a string with random user names populated in a template.
+ *
+ * This function generates a specified number of random user names and populates placeholders
+ * in the provided template with these names. Placeholders in the template should follow the format `{{userX}}`
+ * where `X` is the position of the user (e.g., `{{user1}}`, `{{user2}}`).
+ *
+ * @param {string} params.template - The template string containing placeholders for random user names.
+ * @param {number} params.length - The number of random user names to generate.
+ * @returns {string} The template string with placeholders replaced by random user names.
+ *
+ * @example
+ * // Given a template and a length
+ * const template = "Hello, {{user1}}! Meet {{user2}} and {{user3}}.";
+ * const length = 3;
+ *
+ * // Composing the random user string will result in:
+ * // "Hello, John! Meet Alice and Bob."
+ * const result = composeRandomUser({ template, length });
+ */
+export const composeRandomUser = (template: string, length: number) => {
+    const exampleNames = Array.from({ length }, () =>
+        uniqueNamesGenerator({ dictionaries: [names] })
+    );
+    let result = template;
+    for (let i = 0; i < exampleNames.length; i++) {
+        result = result.replaceAll(`{{user${i + 1}}}`, exampleNames[i]);
+    }
+
+    return result;
+};