diff --git a/src/compute/compute/dev/README.md b/src/compute/compute/dev/README.md index 560147a228..96dc439476 100644 --- a/src/compute/compute/dev/README.md +++ b/src/compute/compute/dev/README.md @@ -47,3 +47,16 @@ For debugging set the DEBUG env variable to different things according to the de DEBUG_CONSOLE=yes DEBUG=* ./2-syncfs.sh ``` +### Console + +Once you have everything setup you can get an interactive console by doing this instead of [3\-compute.sh](http://3-compute.sh): + +```sh +~/cocalc/src/compute/compute/dev$ . env.sh +~/cocalc/src/compute/compute/dev$ node +Welcome to Node.js v18.17.1. +Type ".help" for more information. +> manager = require('./start-compute.js').manager + +``` + diff --git a/src/compute/compute/dev/start-compute.js b/src/compute/compute/dev/start-compute.js index b58f1c051a..5f35d93199 100644 --- a/src/compute/compute/dev/start-compute.js +++ b/src/compute/compute/dev/start-compute.js @@ -1,11 +1,23 @@ #!/usr/bin/env node +/* +To get an interactive console with access to the manager: + +~/cocalc/src/compute/compute/dev$ node +Welcome to Node.js v18.17.1. +Type ".help" for more information. +> manager = require('./start-compute.js').manager + +*/ + process.env.API_SERVER = process.env.API_SERVER ?? "https://cocalc.com"; console.log("API_SERVER=", process.env.API_SERVER); const { manager } = require("../dist/lib"); const PROJECT_HOME = process.env.PROJECT_HOME ?? "/tmp/home"; +const PORT = process.env.PORT ?? 5004; +const HOST = process.env.HOST ?? "0.0.0.0"; async function main() { const exitHandler = async () => { @@ -28,6 +40,8 @@ async function main() { process.env.UNIONFS_UPPER && process.env.UNIONFS_LOWER ? "fuse.unionfs-fuse" : "fuse", + host: HOST, + port: PORT, }); exports.manager = M; await M.init(); diff --git a/src/compute/compute/lib/http-server/http-next-api.ts b/src/compute/compute/lib/http-server/http-next-api.ts new file mode 100644 index 0000000000..05590347fd --- /dev/null +++ b/src/compute/compute/lib/http-server/http-next-api.ts @@ -0,0 +1,52 @@ +/* +HTTP Next API + +- this is whatever we need from cocalc/src/packages/next/pages/api/v2 specifically + for working with one project. + +*/ + +import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data"; +import { Router } from "express"; +import { getLogger } from "../logger"; +import type { Manager } from "../manager"; + +const logger = getLogger("http-next-api"); + +export default function initHttpNextApi({ manager }: { manager }): Router { + logger.info("initHttpNextApi"); + const router = Router(); + + for (const path in HANDLERS) { + router.post("/" + path, handler(manager, HANDLERS[path])); + router.get("/" + path, handler(manager, HANDLERS[path])); + } + + router.post("*", (req, res) => { + res.json({ error: `api endpoint '${req.path}' is not implemented` }); + }); + router.get("*", (req, res) => { + res.json({ error: `api endpoint '${req.path}' is not implemented` }); + }); + + return router; +} + +function handler( + manager, + f: (x: { req; res; manager: Manager }) => Promise, +) { + return async (req, res) => { + try { + res.json({ success: true, ...(await f({ req, res, manager })) }); + } catch (err) { + res.json({ error: `${err}` }); + } + }; +} + +const HANDLERS = { + "jupyter/kernels": async () => { + return { kernels: await get_kernel_data() }; + }, +}; diff --git a/src/compute/compute/lib/http-server/hub-websocket.ts b/src/compute/compute/lib/http-server/hub-websocket.ts new file mode 100644 index 0000000000..0f9ef4f28b --- /dev/null +++ b/src/compute/compute/lib/http-server/hub-websocket.ts @@ -0,0 +1,98 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +/* +Create websocket similar to connection to normal hub. +Handle stuff that doesn't directly involve the project or compute server, +e.g., user identity. +*/ + +import { Router } from "express"; +import { Server } from "http"; +import Primus from "primus"; +import { getLogger } from "../logger"; +import { from_json_socket, to_json_socket } from "@cocalc/util/misc"; +import * as message from "@cocalc/util/message"; + +const logger = getLogger("hub-websocket"); + +export default function initHubWebsocket({ + server, + manager, +}: { + server: Server; + manager; +}): Router { + const opts = { + pathname: "/hub", + transformer: "websockets", + } as const; + logger.debug(`Initializing primus websocket server at "${opts.pathname}"...`); + const primus = new Primus(server, opts); + initApi({ primus, manager }); + + const router = Router(); + const library: string = primus.library(); + + // it isn't actually minified, but this is what the static code expects. + router.get("/primus.min.js", (_, res) => { + logger.debug("serving up /primus.min.js to a specific client"); + res.send(library); + }); + logger.debug( + `waiting for browser client to request primus.min.js (length=${library.length})...`, + ); + + return router; +} + +function initApi({ primus, manager }): void { + primus.on("connection", (spark) => { + logger.debug(`HUB: new connection from ${spark.address.ip} -- ${spark.id}`); + + const sendResponse = (mesg) => { + const data = to_json_socket(mesg); + spark.write(data); + }; + + spark.on("data", async (data) => { + const mesg = from_json_socket(data); + logger.debug("HUB:", "request", mesg, "REQUEST"); + const t0 = Date.now(); + try { + const resp0 = await handleApiCall(mesg, spark, manager, primus); + const resp = { + ...resp0, + id: mesg.id, + }; + sendResponse(resp); + logger.debug( + "HUB", + "response", + resp, + `FINISHED: time=${Date.now() - t0}ms`, + ); + } catch (err) { + // console.trace(); logger.debug("primus-api error stacktrack", err.stack, err); + logger.debug("HUB:", "failed response to", mesg, "FAILED", `${err}`); + sendResponse({ id: mesg.id, error: err.toString() }); + } + }); + }); + + primus.on("disconnection", (spark) => { + logger.debug(`HUB: end connection from ${spark.address.ip} -- ${spark.id}`); + }); +} + +async function handleApiCall(mesg, spark, manager, primus): Promise { + // @ts-ignore + const _foo = { mesg, spark, manager, primus }; + switch (mesg.event) { + case "ping": + return message.pong({ now: new Date() }); + } + throw Error("not implemented"); +} diff --git a/src/compute/compute/lib/http-server/index.ts b/src/compute/compute/lib/http-server/index.ts new file mode 100644 index 0000000000..058ec94aa6 --- /dev/null +++ b/src/compute/compute/lib/http-server/index.ts @@ -0,0 +1,88 @@ +/* + * This file is part of CoCalc: Copyright © 2024 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +import compression from "compression"; +import express from "express"; +import { createServer } from "http"; +import { getLogger } from "../logger"; +import type { Manager } from "../manager"; +import { path as STATIC_PATH } from "@cocalc/static"; +import { join } from "path"; +import { cacheShortTerm, cacheLongTerm } from "@cocalc/util/http-caching"; +import initWebsocket from "./websocket"; +import initHubWebsocket from "./hub-websocket"; +import initHttpNextApi from "./http-next-api"; +import initRaw from "./raw-server"; + +const logger = getLogger("http-server"); + +const ENTRY_POINT = "compute.html"; + +export function initHttpServer({ + port = 5004, + host = "localhost", + manager, +}: { + port?: number; + host?: string; + manager: Manager; +}) { + logger.info("starting http-server..."); + + const app = express(); + const server = createServer(app); + + // this is expected by the frontend code for where to find the project. + const projectBase = `/${manager.project_id}/raw/`; + logger.info({ projectBase }); + + app.use(projectBase, initWebsocket({ server, projectBase, manager })); + + app.use("/", initHubWebsocket({ server, manager })); + + // CRITICAL: compression must be after websocket above! + app.use(compression()); + + app.get("/", (_req, res) => { + const files = manager.getOpenFiles(); + res.send( + `

Compute Server

CoCalc App

Open Files: ${files.join(", ")}`, + ); + }); + + app.use( + `/static/${ENTRY_POINT}`, + express.static(`/${STATIC_PATH}/${ENTRY_POINT}`, { + setHeaders: cacheShortTerm, + }), + ); + app.use( + "/static", + express.static(STATIC_PATH, { setHeaders: cacheLongTerm }), + ); + + app.get("/customize", (_req, res) => { + res.json({ + configuration: { + compute_server: { project_id: manager.project_id }, + }, + registration: false, + }); + }); + + app.use("/api/v2", initHttpNextApi({ manager })); + + const rawUrl = `/${manager.project_id}/raw`; + logger.debug("raw server at ", { rawUrl }); + app.use(rawUrl, initRaw({ home: manager.home })); + + app.get("*", (_req, res) => { + res.redirect(join("/static", ENTRY_POINT)); + }); + + server.listen(port, host, () => { + logger.info(`Server listening http://${host}:${port}`); + }); +} diff --git a/src/compute/compute/lib/http-server/raw-server.ts b/src/compute/compute/lib/http-server/raw-server.ts new file mode 100644 index 0000000000..c4e0b810c6 --- /dev/null +++ b/src/compute/compute/lib/http-server/raw-server.ts @@ -0,0 +1,30 @@ +/* +Serve static files from the home directory. + +NOTE: There is a very similar server in /src/packages/project/servers/browser/static.ts +See comments there. +*/ + +import { static as staticServer } from "express"; +import index from "serve-index"; +import { getLogger } from "../logger"; +import { Router } from "express"; + +const log = getLogger("http-server:static"); + +export default function initStatic({ home }: { home: string }): Router { + const router = Router(); + router.use("/", (req, res, next) => { + if (req.query.download != null) { + res.setHeader("Content-Type", "application/octet-stream"); + } + res.setHeader("Cache-Control", "private, must-revalidate"); + next(); + }); + + log.info(`serving up HOME="${home}"`); + + router.use("/", index(home, { hidden: true, icons: true })); + router.use("/", staticServer(home, { dotfiles: "allow" })); + return router; +} diff --git a/src/compute/compute/lib/http-server/synctable-channel.ts b/src/compute/compute/lib/http-server/synctable-channel.ts new file mode 100644 index 0000000000..7ac8782aea --- /dev/null +++ b/src/compute/compute/lib/http-server/synctable-channel.ts @@ -0,0 +1,19 @@ +import { getLogger } from "../logger"; +import { synctable_channel } from "@cocalc/sync-server/server/server"; + +const log = getLogger("synctable-channel"); + +export default async function synctableChannel({ + manager, + query, + options, + primus, +}: { + manager; + query; + options; + primus; +}) { + log.debug("synctableChannel ", query, options); + return await synctable_channel(manager.client, primus, log, query, options); +} diff --git a/src/compute/compute/lib/http-server/websocket-api.ts b/src/compute/compute/lib/http-server/websocket-api.ts new file mode 100644 index 0000000000..81871025d6 --- /dev/null +++ b/src/compute/compute/lib/http-server/websocket-api.ts @@ -0,0 +1,137 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +// Similar to @cocalc/project/browser-websocket/api.ts +// It might make sense to refactor this with that -- not sure yet. + +import { getLogger } from "../logger"; +import { version } from "@cocalc/util/smc-version"; +import type { Mesg } from "@cocalc/comm/websocket/types"; +import getListing from "@cocalc/backend/get-listing"; +import { executeCode } from "@cocalc/backend/execute-code"; +import { callback2 } from "@cocalc/util/async-utils"; +import { terminal } from "@cocalc/terminal"; +import realpath from "@cocalc/backend/realpath"; +import { eval_code } from "@cocalc/backend/eval-code"; +import synctableChannel from "./synctable-channel"; +import { project_info_ws } from "@cocalc/sync-server/monitor/activity"; + +const log = getLogger("websocket-api"); + +export function initWebsocketApi({ primus, manager }): void { + primus.on("connection", function (spark) { + log.debug(`new connection from ${spark.address.ip} -- ${spark.id}`); + + spark.on("request", async (data, done) => { + log.debug("primus-api", "request", data, "REQUEST"); + const t0 = Date.now(); + try { + const resp = await handleApiCall(data, spark, manager, primus); + done(resp); + } catch (err) { + // console.trace(); log.debug("primus-api error stacktrack", err.stack, err); + log.debug("primus-api", "request", data, "FAILED", err); + done({ error: err.toString(), status: "error" }); + } + log.debug( + "primus-api", + "request", + data, + `FINISHED: time=${Date.now() - t0}ms`, + ); + }); + }); + + primus.on("disconnection", function (spark) { + log.debug( + "primus-api", + `end connection from ${spark.address.ip} -- ${spark.id}`, + ); + }); +} + +async function handleApiCall( + data: Mesg, + _spark, + manager, + primus, +): Promise { + switch (data.cmd) { + case "version": + return version; + case "listing": + // see packages/sync-fs/lib/index.ts + return await getListing(data.path, data.hidden, manager.home); + + case "exec": + if (data.opts == null) { + throw Error("opts must not be null"); + } + return await executeCode({ + ...data.opts, + home: manager.home, + ccNewFile: true, + }); + + case "query": + if (data.opts?.changes) { + throw Error(`changefeeds are not supported for api queries`); + } + return await callback2( + manager.client.query.bind(manager.client), + data.opts, + ); + + case "terminal": + // this might work but be TOTALLY WRONG (?)... or require + // some thought about who "hosts" the terminal. + return await terminal(primus, data.path, data.options); + + case "eval_code": + return await eval_code(data.code); + + case "realpath": + return realpath(data.path, manager.home); + + case "synctable_channel": + return await synctableChannel({ + manager, + query: data.query, + options: data.options, + primus, + }); + + case "project_info": + return await project_info_ws(primus, log); + + // TODO + case "delete_files": + case "move_files": + case "rename_file": + case "canonical_paths": + case "configuration": + case "prettier": // deprecated + case "formatter": + case "prettier_string": // deprecated + case "formatter_string": + case "lean": + case "jupyter_strip_notebook": + case "jupyter_nbconvert": + case "jupyter_run_notebook": + case "lean_channel": + case "x11_channel": + case "syncdoc_call": + case "symmetric_channel": + case "compute_filesystem_cache": + case "sync_fs": + case "compute_server_sync_register": + case "compute_server_compute_register": + case "compute_server_sync_request": + case "copy_from_project_to_compute_server": + case "copy_from_compute_server_to_project": + default: + throw Error(`command "${(data as any).cmd}" not implemented`); + } +} diff --git a/src/compute/compute/lib/http-server/websocket.ts b/src/compute/compute/lib/http-server/websocket.ts new file mode 100644 index 0000000000..ec8c53c166 --- /dev/null +++ b/src/compute/compute/lib/http-server/websocket.ts @@ -0,0 +1,56 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +/* +Create the Primus realtime socket server. + +Similar to @cocalc/project/browser-websocket/server.ts +*/ + +import { join } from "path"; +import { Router } from "express"; +import { Server } from "http"; +import Primus from "primus"; +import type { PrimusWithChannels } from "@cocalc/terminal"; +import { initWebsocketApi } from "./websocket-api"; +import { getLogger } from "../logger"; + +const logger = getLogger("websocket"); + +export default function initWebsocket({ + server, + projectBase, + manager, +}: { + server: Server; + projectBase: string; + manager; +}): Router { + const opts = { + pathname: join(projectBase, ".smc", "ws"), + transformer: "websockets", + } as const; + logger.debug(`Initializing primus websocket server at "${opts.pathname}"...`); + const primus = new Primus(server, opts) as PrimusWithChannels; + + // add multiplex to Primus so we have channels. + primus.plugin("multiplex", require("@cocalc/primus-multiplex")); + primus.plugin("responder", require("@cocalc/primus-responder")); + + initWebsocketApi({ primus, manager }); + + const router = Router(); + const library: string = primus.library(); + + router.get("/.smc/primus.js", (_, res) => { + logger.debug("serving up primus.js to a specific client"); + res.send(library); + }); + logger.debug( + `waiting for clients to request primus.js (length=${library.length})...`, + ); + + return router; +} diff --git a/src/compute/compute/lib/listings.ts b/src/compute/compute/lib/listings.ts index d7181edd37..2e2a1277c1 100644 --- a/src/compute/compute/lib/listings.ts +++ b/src/compute/compute/lib/listings.ts @@ -60,4 +60,6 @@ export async function initListings({ existsSync, getLogger, }); + + return table; } diff --git a/src/compute/compute/lib/logger.ts b/src/compute/compute/lib/logger.ts new file mode 100644 index 0000000000..aee2a5a9f3 --- /dev/null +++ b/src/compute/compute/lib/logger.ts @@ -0,0 +1,6 @@ +import debug from "debug"; + +export function getLogger(name) { + const d = debug(`cocalc:compute:${name}`); + return { info: d, debug: d }; +} diff --git a/src/compute/compute/lib/manager.ts b/src/compute/compute/lib/manager.ts index c8f913054d..84463c694e 100644 --- a/src/compute/compute/lib/manager.ts +++ b/src/compute/compute/lib/manager.ts @@ -28,6 +28,7 @@ import { apiCall } from "@cocalc/api-client"; import { get_blob_store as initJupyterBlobStore } from "@cocalc/jupyter/blobs"; import { delay } from "awaiting"; import { executeCode } from "@cocalc/backend/execute-code"; +import { initHttpServer } from "./http-server"; const logger = debug("cocalc:compute:manager"); @@ -47,6 +48,9 @@ interface Options { // If true, doesn't do anything until the type of the file system that home is // mounted on is of this type, e.g., "fuse". waitHomeFilesystemType?: string; + + host?: string; + port?: number; } process.on("exit", () => { @@ -75,22 +79,27 @@ export function manager(opts: Options) { return new Manager(opts); } -class Manager { +export class Manager { private state: "new" | "init" | "ready" = "new"; private sync_db; - private project_id: string; - private home: string; + public project_id: string; + public home: string; + private host?: string; + private port?: number; private waitHomeFilesystemType?: string; - private compute_server_id: number; + public compute_server_id: number; private connections: { [path: string]: any } = {}; private websocket; - private client; + public listings; + public client; constructor({ project_id, compute_server_id = parseInt(process.env.COMPUTE_SERVER_ID ?? "0"), home = process.env.HOME ?? "/home/user", waitHomeFilesystemType, + host, + port, }: Options) { if (!project_id) { throw Error("project_id or process.env.PROJECT_ID must be given"); @@ -102,6 +111,8 @@ class Manager { // @ts-ignore -- can't true type, since constructed via plain javascript startup script. this.compute_server_id = parseInt(compute_server_id); this.home = home; + this.host = host; + this.port = port; this.waitHomeFilesystemType = waitHomeFilesystemType; const env = this.env(); for (const key in env) { @@ -115,6 +126,9 @@ class Manager { } this.log("initialize the Manager"); this.state = "init"; + + await this.initHttpServer(); + // Ping to start the project and ensure there is a hub connection to it. await pingProjectUntilSuccess(this.project_id); // wait for home direcotry file system to be mounted: @@ -174,7 +188,7 @@ class Manager { }; private initListings = async () => { - await initListings({ + this.listings = await initListings({ client: this.client, project_id: this.project_id, compute_server_id: this.compute_server_id, @@ -421,4 +435,14 @@ class Manager { throw Error(`unknown event '${data?.event}'`); } }; + + private initHttpServer = () => { + if (this.host != null && this.port != null) { + initHttpServer({ port: this.port, host: this.host, manager: this }); + } + }; + + getOpenFiles = (): string[] => { + return Object.keys(this.connections); + }; } diff --git a/src/compute/compute/package.json b/src/compute/compute/package.json index ebd6e823ab..0823f0a2ff 100644 --- a/src/compute/compute/package.json +++ b/src/compute/compute/package.json @@ -30,17 +30,25 @@ "dependencies": { "@cocalc/api-client": "workspace:*", "@cocalc/backend": "workspace:*", + "@cocalc/comm": "workspace:*", "@cocalc/compute": "link:", "@cocalc/jupyter": "workspace:*", + "@cocalc/primus-multiplex": "^1.1.0", + "@cocalc/primus-responder": "^1.0.5", + "@cocalc/static": "workspace:*", "@cocalc/sync": "workspace:*", "@cocalc/sync-client": "workspace:*", "@cocalc/sync-fs": "workspace:*", + "@cocalc/sync-server": "workspace:*", "@cocalc/terminal": "workspace:*", "@cocalc/util": "workspace:*", - "@types/ws": "^8.5.9", "awaiting": "^3.0.0", + "compression": "^1.7.4", "cookie": "^1.0.0", "debug": "^4.3.2", + "express": "^4.21.1", + "primus": "^8.0.7", + "serve-index": "^1.9.1", "websocketfs": "^0.17.4", "ws": "^8.18.0" }, @@ -50,8 +58,12 @@ "url": "https://github.com/sagemathinc/cocalc" }, "devDependencies": { + "@types/compression": "^1.7.5", "@types/cookie": "^0.6.0", + "@types/express": "^4.17.21", + "@types/ms": "^0.7.34", "@types/node": "^18.16.14", + "@types/ws": "^8.5.9", "typescript": "^5.6.3" } } diff --git a/src/compute/compute/tsconfig.json b/src/compute/compute/tsconfig.json index 9b192c7b00..31e04b938c 100644 --- a/src/compute/compute/tsconfig.json +++ b/src/compute/compute/tsconfig.json @@ -12,6 +12,7 @@ { "path": "../sync" }, { "path": "../sync-client" }, { "path": "../sync-fs" }, + { "path": "../sync-server" }, { "path": "../terminal" }, { "path": "../util" } ] diff --git a/src/compute/pnpm-lock.yaml b/src/compute/pnpm-lock.yaml index 2eea7010d4..9c3108c33c 100644 --- a/src/compute/pnpm-lock.yaml +++ b/src/compute/pnpm-lock.yaml @@ -6,6 +6,12 @@ settings: importers: + api-client: {} + + backend: {} + + comm: {} + compute: dependencies: '@cocalc/api-client': @@ -14,12 +20,24 @@ importers: '@cocalc/backend': specifier: workspace:* version: link:../backend + '@cocalc/comm': + specifier: workspace:* + version: link:../comm '@cocalc/compute': specifier: 'link:' version: 'link:' '@cocalc/jupyter': specifier: workspace:* version: link:../jupyter + '@cocalc/primus-multiplex': + specifier: ^1.1.0 + version: 1.1.0 + '@cocalc/primus-responder': + specifier: ^1.0.5 + version: 1.0.5 + '@cocalc/static': + specifier: workspace:* + version: link:../static '@cocalc/sync': specifier: workspace:* version: link:../sync @@ -29,24 +47,36 @@ importers: '@cocalc/sync-fs': specifier: workspace:* version: link:../sync-fs + '@cocalc/sync-server': + specifier: workspace:* + version: link:../sync-server '@cocalc/terminal': specifier: workspace:* version: link:../terminal '@cocalc/util': specifier: workspace:* version: link:../util - '@types/ws': - specifier: ^8.5.9 - version: 8.5.9 awaiting: specifier: ^3.0.0 version: 3.0.0 + compression: + specifier: ^1.7.4 + version: 1.7.4 cookie: specifier: ^1.0.0 version: 1.0.0 debug: specifier: ^4.3.2 version: 4.3.4 + express: + specifier: ^4.21.1 + version: 4.21.1 + primus: + specifier: ^8.0.7 + version: 8.0.9 + serve-index: + specifier: ^1.9.1 + version: 1.9.1 websocketfs: specifier: ^0.17.4 version: 0.17.4 @@ -54,38 +84,124 @@ importers: specifier: ^8.18.0 version: 8.18.0 devDependencies: + '@types/compression': + specifier: ^1.7.5 + version: 1.7.5 '@types/cookie': specifier: ^0.6.0 version: 0.6.0 + '@types/express': + specifier: ^4.17.21 + version: 4.17.21 + '@types/ms': + specifier: ^0.7.34 + version: 0.7.34 '@types/node': specifier: ^18.16.14 version: 18.16.14 + '@types/ws': + specifier: ^8.5.9 + version: 8.5.9 typescript: specifier: ^5.6.3 version: 5.6.3 + jupyter: {} + + static: {} + + sync: {} + + sync-client: {} + + sync-fs: {} + + sync-server: {} + + terminal: {} + + util: {} + packages: '@cocalc/fuse-native@2.4.1': resolution: {integrity: sha512-Z4mzgaJyz/vNIv5Z+8eK2D5n+xkcNbKZ4/yQI56685VD0SwOaD9WNRdvD7u6s1IiB2FoYqljdBjat/MwzNlb3g==} hasBin: true + '@cocalc/primus-multiplex@1.1.0': + resolution: {integrity: sha512-o8AOFVs996NQ2nvCjoSzNkBJRkPYi5XVPw0BbtMZUz+hnegSCfGawT59S++wta0CzGdFPWnyPtQqVJpobQlT4w==} + + '@cocalc/primus-responder@1.0.5': + resolution: {integrity: sha512-nFHqPo9zxbPNRkMzFH/ZbVrrp3RGb0vAVWSRG67+j8dwt3MN9pqZSEUOfWUt9AURsRJd9IR+gLnwrczQTNu6Zg==} + engines: {node: '>=0.10.28'} + '@isaacs/ttlcache@1.4.1': resolution: {integrity: sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==} engines: {node: '>=12'} + '@types/body-parser@1.19.5': + resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} + + '@types/compression@1.7.5': + resolution: {integrity: sha512-AAQvK5pxMpaT+nDvhHrsBhLSYG5yQdtkaJE1WYieSNY2mVFKAgmU4ks65rkZD5oqnGCFLyQpUr1CqI4DmUMyDg==} + + '@types/connect@3.4.38': + resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} + '@types/cookie@0.6.0': resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==} + '@types/express-serve-static-core@4.19.6': + resolution: {integrity: sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==} + + '@types/express@4.17.21': + resolution: {integrity: sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==} + + '@types/http-errors@2.0.4': + resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==} + + '@types/mime@1.3.5': + resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} + + '@types/ms@0.7.34': + resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} + '@types/node@18.16.14': resolution: {integrity: sha512-+ImzUB3mw2c5ISJUq0punjDilUQ5GnUim0ZRvchHIWJmOC0G+p0kzhXBqj6cDjK0QdPFwzrHWgrJp3RPvCG5qg==} + '@types/qs@6.9.16': + resolution: {integrity: sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==} + + '@types/range-parser@1.2.7': + resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} + + '@types/send@0.17.4': + resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==} + + '@types/serve-static@1.15.7': + resolution: {integrity: sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==} + '@types/ws@8.5.9': resolution: {integrity: sha512-jbdrY0a8lxfdTp/+r7Z4CkycbOFN8WX+IOchLJr3juT/xzbJ8URyTVSJ/hvNdadTgM1mnedb47n+Y31GsFnQlg==} '@wwa/statvfs@1.1.18': resolution: {integrity: sha512-C33QeTo2Nma9gMAJy3l1AQc0Qz5Lbf7mCY2C3F1W3noCdukODWH8nB8sjavdwjw9S7Qa+zrvQAfbbYCOzIphAw==} + accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + + access-control@1.0.1: + resolution: {integrity: sha512-H5aqjkogmFxfaOrfn/e42vyspHVXuJ8er63KuljJXpOyJ1ZO/U5CrHfO8BLKIy2w7mBM02L5quL0vbfQqrGQbA==} + + array-flatten@1.1.1: + resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} + + asyncemit@3.0.1: + resolution: {integrity: sha512-sVGuQSrkDjPJrnuMvWvltg4+UIO85qkDPGuSzVmj1LojoDR5DZPCVTSPRjYk1V0UPaeypFeMMoj22ZOG8egdyw==} + peerDependencies: + eventemitter3: '>=1.1.0' + awaiting@3.0.0: resolution: {integrity: sha512-19i4G7Hjxj9idgMlAM0BTRII8HfvsOdlr4D9cf3Dm1MZhvcKjBpzY8AMNEyIKyi+L9TIK15xZatmdcPG003yww==} engines: {node: '>=7.6.x'} @@ -93,6 +209,9 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + batch@0.6.1: + resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} + binarysearch@1.0.1: resolution: {integrity: sha512-FqhwdeXh1ZSAS/YpJ6lD9+SMf8JodCibe7c51Z9L1zAjHKUDTBisQgdmpfaL+m1qHvwAHnSLR8d9UHc79Hr34g==} @@ -102,23 +221,97 @@ packages: bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + body-parser@1.20.3: + resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + bytes@3.0.0: + resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==} + engines: {node: '>= 0.8'} + + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + call-bind@1.0.7: + resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} + engines: {node: '>= 0.4'} + chownr@1.1.4: resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + color-convert@1.9.3: + resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} + + color-name@1.1.3: + resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + color-string@1.9.1: + resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==} + + color@3.2.1: + resolution: {integrity: sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==} + + colornames@1.1.1: + resolution: {integrity: sha512-/pyV40IrsdulWv+wFPmERh9k/mjsPZ64yUMDmWrtj/k1nmgrzzIENWKdaVKyBbvFdQWqkcaRxr+polCo3VMe7A==} + + colorspace@1.1.4: + resolution: {integrity: sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==} + + compressible@2.0.18: + resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} + engines: {node: '>= 0.6'} + + compression@1.7.4: + resolution: {integrity: sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==} + engines: {node: '>= 0.8.0'} + + connected@0.0.2: + resolution: {integrity: sha512-J8DB7618GkIYjc1RCxSdG3vffhhYRwHNEckjOGfwAbabQIMgKsL5c54IaWtisulEwoOEbEODEUak4kyJP2GJ/Q==} + + content-disposition@0.5.4: + resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} + engines: {node: '>= 0.6'} + + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + + cookie-signature@1.0.6: + resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==} + cookie@0.5.0: resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==} engines: {node: '>= 0.6'} + cookie@0.7.1: + resolution: {integrity: sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==} + engines: {node: '>= 0.6'} + cookie@1.0.0: resolution: {integrity: sha512-bsSztFoaR8bw9MlFCrTHzc1wOKCUKOBsbgFdoDilZDkETAOOjKSqV7L+EQLbTaylwvZasd9vM4MGKotJaUfSpA==} engines: {node: '>=18'} + create-server@1.0.2: + resolution: {integrity: sha512-hie+Kyero+jxt6dwKhLKtN23qSNiMn8mNIEjTjwzaZwH2y4tr4nYloeFrpadqV+ZqV9jQ15t3AKotaK8dOo45w==} + cuint@0.2.2: resolution: {integrity: sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==} + debug@2.6.9: + resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + debug@4.3.4: resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'} @@ -136,39 +329,214 @@ packages: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} + define-data-property@1.1.4: + resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} + engines: {node: '>= 0.4'} + + depd@1.1.2: + resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} + engines: {node: '>= 0.6'} + + depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + + destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + detect-libc@2.0.3: resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==} engines: {node: '>=8'} + diagnostics@1.1.1: + resolution: {integrity: sha512-8wn1PmdunLJ9Tqbx+Fx/ZEuHfJf4NKSN2ZBj7SJC/OWRWha843+WsTjqMe1B5E3p28jqBlp+mJ2fPVxPyNgYKQ==} + + diagnostics@2.0.2: + resolution: {integrity: sha512-gvnlQHwkWTOeSM1iRNEwPcUuUwlhovzbuQzalKrTbcJhI5cvhtkRVZZqomwZt4pCl2dvbsugD6yyu+66rtMy3Q==} + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + + emits@3.0.0: + resolution: {integrity: sha512-WJSCMaN/qjIkzWy5Ayu0MDENFltcu4zTPPnWqdFPOVBtsENVTN+A3d76G61yuiVALsMK+76MejdPrwmccv/wag==} + + enabled@1.0.2: + resolution: {integrity: sha512-nnzgVSpB35qKrUN8358SjO1bYAmxoThECTWw9s3J0x5G8A9hokKHVDFzBjVpCoSryo6MhN8woVyascN5jheaNA==} + + enabled@2.0.0: + resolution: {integrity: sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==} + + encodeurl@1.0.2: + resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} + engines: {node: '>= 0.8'} + + encodeurl@2.0.0: + resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} + engines: {node: '>= 0.8'} + end-of-stream@1.4.4: resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + env-variable@0.0.6: + resolution: {integrity: sha512-bHz59NlBbtS0NhftmR8+ExBEekE7br0e01jw+kk0NDro7TtZzBYZ5ScGPs3OmwnpyfHTHOtr1Y6uedCdrIldtg==} + + es-define-property@1.0.0: + resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + + eventemitter3@4.0.7: + resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + + eventemitter3@5.0.1: + resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + expand-template@2.0.3: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} + express@4.21.1: + resolution: {integrity: sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==} + engines: {node: '>= 0.10.0'} + + extendible@0.1.1: + resolution: {integrity: sha512-AglckQA0TJV8/ZmhQcNmaaFcFFPXFIoZbfuoQOlGDK7Jh/roWotYzJ7ik1FBBCHBr8n7CgTR8lXXPAN8Rfb7rw==} + file-uri-to-path@1.0.0: resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + finalhandler@1.3.1: + resolution: {integrity: sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==} + engines: {node: '>= 0.8'} + + forwarded-for@1.1.0: + resolution: {integrity: sha512-1Yam9ht7GyMXMBvuwJfUYqpdtLVodtT5ee5JMBzGiSwVVeh37ZN8LuOWkNHd6ho2zUxpSZCHuQrt1Vjl2AxDNA==} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + fusing@1.0.0: + resolution: {integrity: sha512-gc2uPkiQy4zeMmLmdiCQXPe5zs+knBaoxGGe7R5Ve7fGEbrnnSOWiTKxq0elaO/RMBuTiZ1hzSGQa2+Sdl2h5w==} + + get-intrinsic@1.2.4: + resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} + engines: {node: '>= 0.4'} + github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + gopd@1.0.1: + resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} + + has-property-descriptors@1.0.2: + resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} + + has-proto@1.0.3: + resolution: {integrity: sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==} + engines: {node: '>= 0.4'} + + has-symbols@1.0.3: + resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} + engines: {node: '>= 0.4'} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + + http-errors@1.6.3: + resolution: {integrity: sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==} + engines: {node: '>= 0.6'} + + http-errors@2.0.0: + resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} + engines: {node: '>= 0.8'} + + iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + inherits@2.0.3: + resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} + inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + + is-arrayish@0.3.2: + resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==} + + kuler@1.0.1: + resolution: {integrity: sha512-J9nVUucG1p/skKul6DU3PUZrhs0LPulNaeUOox0IyXDi8S4CztTHs1gQphhuZmzXG7VOQSf6NJfKuzteQLv9gQ==} + + kuler@2.0.0: + resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} + lz4@0.6.5: resolution: {integrity: sha512-KSZcJU49QZOlJSItaeIU3p8WoAvkTmD9fJqeahQXNu1iQ/kR0/mQLdbrK8JY9MY8f6AhJoMrihp1nu1xDbscSQ==} engines: {node: '>= 0.10'} + media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + + merge-descriptors@1.0.3: + resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==} + + methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + + millisecond@0.1.2: + resolution: {integrity: sha512-BJ8XtxY+woL+5TkP6uS6XvOArm0JVrX2otkgtWZseHpIax0oOOPW3cnwhOjRqbEJg7YRO/BDF7fO/PTWNT3T9Q==} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + mime@1.6.0: + resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} + engines: {node: '>=4'} + hasBin: true + mimic-response@3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} @@ -179,12 +547,23 @@ packages: mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + ms@2.0.0: + resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + ms@2.1.2: resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + nan@2.20.0: resolution: {integrity: sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==} + nanoid@3.3.7: + resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + nanoresource@1.3.0: resolution: {integrity: sha512-OI5dswqipmlYfyL3k/YMm7mbERlh4Bd1KuKdMHpeoVD1iVxqxaTMKleB4qaA2mbQZ6/zMNSxCXv9M9P/YbqTuQ==} @@ -194,6 +573,10 @@ packages: napi-macros@2.2.2: resolution: {integrity: sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g==} + negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + node-abi@3.68.0: resolution: {integrity: sha512-7vbj10trelExNjFSBm5kTvZXXa7pZyKWx9RCKIyqe6I9Ev3IzGpQoqBP3a+cOdxY+pWj6VkP28n/2wWysBHD/A==} engines: {node: '>=10'} @@ -206,9 +589,33 @@ packages: resolution: {integrity: sha512-IRUxE4BVsHWXkV/SFOut4qTlagw2aM8T5/vnTsmrHJvVoKueJHRc/JaFND7QDDc61kLYUJ6qlZM3sqTSyx2dTw==} hasBin: true + node-uuid@1.4.8: + resolution: {integrity: sha512-TkCET/3rr9mUuRp+CpO7qfgT++aAxfDRaalQhwPFzI9BY/2rCDn6OfpZOVggi1AXfTPpfkTrg5f5WQx5G1uLxA==} + deprecated: Use uuid module instead + hasBin: true + + object-inspect@1.13.2: + resolution: {integrity: sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==} + engines: {node: '>= 0.4'} + + on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + + on-headers@1.0.2: + resolution: {integrity: sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==} + engines: {node: '>= 0.8'} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + + path-to-regexp@0.1.10: + resolution: {integrity: sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==} + port-get@1.0.4: resolution: {integrity: sha512-B8RcNfc8Ld+7C31DPaKIQz2aO9dqIs+4sUjhxJ2TSjEaidwyxu05WBbm08FJe+qkVvLiQqPbEAfNw1rB7JbjtA==} @@ -217,9 +624,31 @@ packages: engines: {node: '>=10'} hasBin: true + predefine@0.1.3: + resolution: {integrity: sha512-Nq6APFC5OtQRl5TmMk6RlGwl6UOCtEqa+5ZTbKFp6tMw4wdMUa7Rief0UNE3fV5BgQahJ70QmDgeOog8RE9FMw==} + + primus@8.0.9: + resolution: {integrity: sha512-gWsd6pWHAHGfyArl6DQU9iCAp4bAgFrintDpFbyA2r0wdzJ2n9SsffSaFqOKYZeE9wqKcBepnwBGoFKzNybqMA==} + + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + pump@3.0.2: resolution: {integrity: sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==} + qs@6.13.0: + resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==} + engines: {node: '>=0.6'} + + range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + + raw-body@2.5.2: + resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} + engines: {node: '>= 0.8'} + rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true @@ -228,20 +657,69 @@ packages: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} + safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + semver@7.6.3: resolution: {integrity: sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==} engines: {node: '>=10'} hasBin: true + send@0.19.0: + resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==} + engines: {node: '>= 0.8.0'} + + serve-index@1.9.1: + resolution: {integrity: sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==} + engines: {node: '>= 0.8.0'} + + serve-static@1.16.2: + resolution: {integrity: sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==} + engines: {node: '>= 0.8.0'} + + set-function-length@1.2.2: + resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==} + engines: {node: '>= 0.4'} + + setheader@1.0.2: + resolution: {integrity: sha512-A704nIwzqGed0CnJZIqDE+0udMPS839ocgf1R9OJ8aq8vw4U980HWeNaD9ec8VnmBni9lyGEWDedOWXT/C5kxA==} + + setprototypeof@1.1.0: + resolution: {integrity: sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==} + + setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + + side-channel@1.0.6: + resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} + engines: {node: '>= 0.4'} + simple-concat@1.0.1: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} simple-get@4.0.1: resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + simple-swizzle@0.2.2: + resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==} + + statuses@1.5.0: + resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==} + engines: {node: '>= 0.6'} + + statuses@2.0.1: + resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} + engines: {node: '>= 0.8'} + + storage-engine@3.0.7: + resolution: {integrity: sha512-V/jJykpPdsyDImLwu19syIAWn/Tb41tBDikQS+aQPH2h2OgqdLxwOg7wI9nPH3Y0Mh1ce566JZl2u+4eH1nAsg==} + string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} @@ -256,17 +734,43 @@ packages: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} engines: {node: '>=6'} + text-hex@1.0.0: + resolution: {integrity: sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==} + + toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + typescript@5.6.3: resolution: {integrity: sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==} engines: {node: '>=14.17'} hasBin: true + ultron@1.1.1: + resolution: {integrity: sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==} + + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + utils-merge@1.0.1: + resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} + engines: {node: '>= 0.4.0'} + + vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + websocket-sftp@0.8.2: resolution: {integrity: sha512-LHUzwTSQNB+Rud+OTe+lt9ojdWzE5nHQ2rnBs1JwQIXnh3bc0NVXk9pY7TMmFLkLYoBUKypa7Kl2CPfuxKLE4g==} engines: {node: '>=0.16.0'} @@ -302,12 +806,73 @@ snapshots: napi-macros: 2.2.2 node-gyp-build: 4.8.2 + '@cocalc/primus-multiplex@1.1.0': + dependencies: + escape-string-regexp: 4.0.0 + eventemitter3: 5.0.1 + predefine: 0.1.3 + + '@cocalc/primus-responder@1.0.5': + dependencies: + debug: 4.3.4 + node-uuid: 1.4.8 + transitivePeerDependencies: + - supports-color + '@isaacs/ttlcache@1.4.1': {} + '@types/body-parser@1.19.5': + dependencies: + '@types/connect': 3.4.38 + '@types/node': 18.16.14 + + '@types/compression@1.7.5': + dependencies: + '@types/express': 4.17.21 + + '@types/connect@3.4.38': + dependencies: + '@types/node': 18.16.14 + '@types/cookie@0.6.0': {} + '@types/express-serve-static-core@4.19.6': + dependencies: + '@types/node': 18.16.14 + '@types/qs': 6.9.16 + '@types/range-parser': 1.2.7 + '@types/send': 0.17.4 + + '@types/express@4.17.21': + dependencies: + '@types/body-parser': 1.19.5 + '@types/express-serve-static-core': 4.19.6 + '@types/qs': 6.9.16 + '@types/serve-static': 1.15.7 + + '@types/http-errors@2.0.4': {} + + '@types/mime@1.3.5': {} + + '@types/ms@0.7.34': {} + '@types/node@18.16.14': {} + '@types/qs@6.9.16': {} + + '@types/range-parser@1.2.7': {} + + '@types/send@0.17.4': + dependencies: + '@types/mime': 1.3.5 + '@types/node': 18.16.14 + + '@types/serve-static@1.15.7': + dependencies: + '@types/http-errors': 2.0.4 + '@types/node': 18.16.14 + '@types/send': 0.17.4 + '@types/ws@8.5.9': dependencies: '@types/node': 18.16.14 @@ -318,10 +883,29 @@ snapshots: node-addon-api: 8.1.0 prebuild-install: 7.1.2 + accepts@1.3.8: + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + + access-control@1.0.1: + dependencies: + millisecond: 0.1.2 + setheader: 1.0.2 + vary: 1.1.2 + + array-flatten@1.1.1: {} + + asyncemit@3.0.1(eventemitter3@5.0.1): + dependencies: + eventemitter3: 5.0.1 + awaiting@3.0.0: {} base64-js@1.5.1: {} + batch@0.6.1: {} + binarysearch@1.0.1: {} bindings@1.5.0: @@ -334,19 +918,109 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + body-parser@1.20.3: + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + on-finished: 2.4.1 + qs: 6.13.0 + raw-body: 2.5.2 + type-is: 1.6.18 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + buffer@5.7.1: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 + bytes@3.0.0: {} + + bytes@3.1.2: {} + + call-bind@1.0.7: + dependencies: + es-define-property: 1.0.0 + es-errors: 1.3.0 + function-bind: 1.1.2 + get-intrinsic: 1.2.4 + set-function-length: 1.2.2 + chownr@1.1.4: {} + color-convert@1.9.3: + dependencies: + color-name: 1.1.3 + + color-name@1.1.3: {} + + color-name@1.1.4: {} + + color-string@1.9.1: + dependencies: + color-name: 1.1.4 + simple-swizzle: 0.2.2 + + color@3.2.1: + dependencies: + color-convert: 1.9.3 + color-string: 1.9.1 + + colornames@1.1.1: {} + + colorspace@1.1.4: + dependencies: + color: 3.2.1 + text-hex: 1.0.0 + + compressible@2.0.18: + dependencies: + mime-db: 1.52.0 + + compression@1.7.4: + dependencies: + accepts: 1.3.8 + bytes: 3.0.0 + compressible: 2.0.18 + debug: 2.6.9 + on-headers: 1.0.2 + safe-buffer: 5.1.2 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + + connected@0.0.2: {} + + content-disposition@0.5.4: + dependencies: + safe-buffer: 5.2.1 + + content-type@1.0.5: {} + + cookie-signature@1.0.6: {} + cookie@0.5.0: {} + cookie@0.7.1: {} + cookie@1.0.0: {} + create-server@1.0.2: + dependencies: + connected: 0.0.2 + cuint@0.2.2: {} + debug@2.6.9: + dependencies: + ms: 2.0.0 + debug@4.3.4: dependencies: ms: 2.1.2 @@ -357,26 +1031,201 @@ snapshots: deep-extend@0.6.0: {} + define-data-property@1.1.4: + dependencies: + es-define-property: 1.0.0 + es-errors: 1.3.0 + gopd: 1.0.1 + + depd@1.1.2: {} + + depd@2.0.0: {} + + destroy@1.2.0: {} + detect-libc@2.0.3: {} + diagnostics@1.1.1: + dependencies: + colorspace: 1.1.4 + enabled: 1.0.2 + kuler: 1.0.1 + + diagnostics@2.0.2: + dependencies: + colorspace: 1.1.4 + enabled: 2.0.0 + kuler: 2.0.0 + storage-engine: 3.0.7 + + ee-first@1.1.1: {} + + emits@3.0.0: {} + + enabled@1.0.2: + dependencies: + env-variable: 0.0.6 + + enabled@2.0.0: {} + + encodeurl@1.0.2: {} + + encodeurl@2.0.0: {} + end-of-stream@1.4.4: dependencies: once: 1.4.0 + env-variable@0.0.6: {} + + es-define-property@1.0.0: + dependencies: + get-intrinsic: 1.2.4 + + es-errors@1.3.0: {} + + escape-html@1.0.3: {} + + escape-string-regexp@4.0.0: {} + + etag@1.8.1: {} + + eventemitter3@4.0.7: {} + + eventemitter3@5.0.1: {} + expand-template@2.0.3: {} + express@4.21.1: + dependencies: + accepts: 1.3.8 + array-flatten: 1.1.1 + body-parser: 1.20.3 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookie: 0.7.1 + cookie-signature: 1.0.6 + debug: 2.6.9 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 1.3.1 + fresh: 0.5.2 + http-errors: 2.0.0 + merge-descriptors: 1.0.3 + methods: 1.1.2 + on-finished: 2.4.1 + parseurl: 1.3.3 + path-to-regexp: 0.1.10 + proxy-addr: 2.0.7 + qs: 6.13.0 + range-parser: 1.2.1 + safe-buffer: 5.2.1 + send: 0.19.0 + serve-static: 1.16.2 + setprototypeof: 1.2.0 + statuses: 2.0.1 + type-is: 1.6.18 + utils-merge: 1.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + + extendible@0.1.1: {} + file-uri-to-path@1.0.0: {} + finalhandler@1.3.1: + dependencies: + debug: 2.6.9 + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.1 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + + forwarded-for@1.1.0: {} + + forwarded@0.2.0: {} + + fresh@0.5.2: {} + fs-constants@1.0.0: {} + function-bind@1.1.2: {} + + fusing@1.0.0: + dependencies: + emits: 3.0.0 + predefine: 0.1.3 + + get-intrinsic@1.2.4: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + has-proto: 1.0.3 + has-symbols: 1.0.3 + hasown: 2.0.2 + github-from-package@0.0.0: {} + gopd@1.0.1: + dependencies: + get-intrinsic: 1.2.4 + + has-property-descriptors@1.0.2: + dependencies: + es-define-property: 1.0.0 + + has-proto@1.0.3: {} + + has-symbols@1.0.3: {} + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + + http-errors@1.6.3: + dependencies: + depd: 1.1.2 + inherits: 2.0.3 + setprototypeof: 1.1.0 + statuses: 1.5.0 + + http-errors@2.0.0: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.1 + toidentifier: 1.0.1 + + iconv-lite@0.4.24: + dependencies: + safer-buffer: 2.1.2 + ieee754@1.2.1: {} + inherits@2.0.3: {} + inherits@2.0.4: {} ini@1.3.8: {} + ipaddr.js@1.9.1: {} + + is-arrayish@0.3.2: {} + + kuler@1.0.1: + dependencies: + colornames: 1.1.1 + + kuler@2.0.0: {} + lz4@0.6.5: dependencies: buffer: 5.7.1 @@ -384,16 +1233,38 @@ snapshots: nan: 2.20.0 xxhashjs: 0.2.2 + media-typer@0.3.0: {} + + merge-descriptors@1.0.3: {} + + methods@1.1.2: {} + + millisecond@0.1.2: {} + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + mime@1.6.0: {} + mimic-response@3.1.0: {} minimist@1.2.8: {} mkdirp-classic@0.5.3: {} + ms@2.0.0: {} + ms@2.1.2: {} + ms@2.1.3: {} + nan@2.20.0: {} + nanoid@3.3.7: {} + nanoresource@1.3.0: dependencies: inherits: 2.0.4 @@ -402,6 +1273,8 @@ snapshots: napi-macros@2.2.2: {} + negotiator@0.6.3: {} + node-abi@3.68.0: dependencies: semver: 7.6.3 @@ -410,10 +1283,24 @@ snapshots: node-gyp-build@4.8.2: {} + node-uuid@1.4.8: {} + + object-inspect@1.13.2: {} + + on-finished@2.4.1: + dependencies: + ee-first: 1.1.1 + + on-headers@1.0.2: {} + once@1.4.0: dependencies: wrappy: 1.0.2 + parseurl@1.3.3: {} + + path-to-regexp@0.1.10: {} + port-get@1.0.4: {} prebuild-install@7.1.2: @@ -431,11 +1318,46 @@ snapshots: tar-fs: 2.1.1 tunnel-agent: 0.6.0 + predefine@0.1.3: + dependencies: + extendible: 0.1.1 + + primus@8.0.9: + dependencies: + access-control: 1.0.1 + asyncemit: 3.0.1(eventemitter3@5.0.1) + create-server: 1.0.2 + diagnostics: 2.0.2 + eventemitter3: 5.0.1 + forwarded-for: 1.1.0 + fusing: 1.0.0 + nanoid: 3.3.7 + setheader: 1.0.2 + ultron: 1.1.1 + + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + pump@3.0.2: dependencies: end-of-stream: 1.4.4 once: 1.4.0 + qs@6.13.0: + dependencies: + side-channel: 1.0.6 + + range-parser@1.2.1: {} + + raw-body@2.5.2: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + rc@1.2.8: dependencies: deep-extend: 0.6.0 @@ -449,10 +1371,77 @@ snapshots: string_decoder: 1.3.0 util-deprecate: 1.0.2 + safe-buffer@5.1.2: {} + safe-buffer@5.2.1: {} + safer-buffer@2.1.2: {} + semver@7.6.3: {} + send@0.19.0: + dependencies: + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.0 + mime: 1.6.0 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.1 + transitivePeerDependencies: + - supports-color + + serve-index@1.9.1: + dependencies: + accepts: 1.3.8 + batch: 0.6.1 + debug: 2.6.9 + escape-html: 1.0.3 + http-errors: 1.6.3 + mime-types: 2.1.35 + parseurl: 1.3.3 + transitivePeerDependencies: + - supports-color + + serve-static@1.16.2: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 0.19.0 + transitivePeerDependencies: + - supports-color + + set-function-length@1.2.2: + dependencies: + define-data-property: 1.1.4 + es-errors: 1.3.0 + function-bind: 1.1.2 + get-intrinsic: 1.2.4 + gopd: 1.0.1 + has-property-descriptors: 1.0.2 + + setheader@1.0.2: + dependencies: + diagnostics: 1.1.1 + + setprototypeof@1.1.0: {} + + setprototypeof@1.2.0: {} + + side-channel@1.0.6: + dependencies: + call-bind: 1.0.7 + es-errors: 1.3.0 + get-intrinsic: 1.2.4 + object-inspect: 1.13.2 + simple-concat@1.0.1: {} simple-get@4.0.1: @@ -461,6 +1450,19 @@ snapshots: once: 1.4.0 simple-concat: 1.0.1 + simple-swizzle@0.2.2: + dependencies: + is-arrayish: 0.3.2 + + statuses@1.5.0: {} + + statuses@2.0.1: {} + + storage-engine@3.0.7: + dependencies: + enabled: 2.0.0 + eventemitter3: 4.0.7 + string_decoder@1.3.0: dependencies: safe-buffer: 5.2.1 @@ -482,14 +1484,31 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + text-hex@1.0.0: {} + + toidentifier@1.0.1: {} + tunnel-agent@0.6.0: dependencies: safe-buffer: 5.2.1 + type-is@1.6.18: + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + typescript@5.6.3: {} + ultron@1.1.1: {} + + unpipe@1.0.0: {} + util-deprecate@1.0.2: {} + utils-merge@1.0.1: {} + + vary@1.1.2: {} + websocket-sftp@0.8.2: dependencies: '@wwa/statvfs': 1.1.18 diff --git a/src/compute/pnpm-workspace.yaml b/src/compute/pnpm-workspace.yaml index 9f84f9cb8f..f8d2ae35e2 100644 --- a/src/compute/pnpm-workspace.yaml +++ b/src/compute/pnpm-workspace.yaml @@ -6,6 +6,8 @@ packages: - "sync" - "sync-fs" - "sync-client" + - "sync-server" - "util" - "terminal" - "compute" + - "static" diff --git a/src/compute/static b/src/compute/static new file mode 120000 index 0000000000..d4c106f6c1 --- /dev/null +++ b/src/compute/static @@ -0,0 +1 @@ +../packages/static/ \ No newline at end of file diff --git a/src/compute/sync-server b/src/compute/sync-server new file mode 120000 index 0000000000..50f34b5b37 --- /dev/null +++ b/src/compute/sync-server @@ -0,0 +1 @@ +../packages/sync-server \ No newline at end of file diff --git a/src/package.json b/src/package.json index 1cd544feec..944058d68f 100644 --- a/src/package.json +++ b/src/package.json @@ -14,7 +14,7 @@ "database": "cd dev/project && ./start_postgres.py", "database-remove-locks": "./scripts/database-remove-locks", "c": "LOGS=/tmp/ DEBUG='cocalc:*' ./scripts/c", - "version-check": "pip3 install typing_extensions mypy || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", + "version-check": "pip3 install typing_extensions mypy 2>/dev/null || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r test", "prettier-all": "cd packages/" diff --git a/src/packages/project/browser-websocket/eval-code.ts b/src/packages/backend/eval-code.ts similarity index 100% rename from src/packages/project/browser-websocket/eval-code.ts rename to src/packages/backend/eval-code.ts diff --git a/src/packages/backend/execute-code.ts b/src/packages/backend/execute-code.ts index f1208eee1e..e74eccb6bd 100644 --- a/src/packages/backend/execute-code.ts +++ b/src/packages/backend/execute-code.ts @@ -17,7 +17,6 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { EventEmitter } from "node:stream"; import shellEscape from "shell-escape"; - import getLogger from "@cocalc/backend/logger"; import { envToInt } from "@cocalc/backend/misc/env-to-number"; import { aggregate } from "@cocalc/util/aggregate"; @@ -37,6 +36,7 @@ import { import { Processes } from "@cocalc/util/types/project-info/types"; import { envForSpawn } from "./misc"; import { ProcessStats } from "./process-stats"; +import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; const log = getLogger("execute-code"); @@ -143,6 +143,15 @@ async function executeCodeNoAggregate( } else { throw new Error(`Async operation '${key}' does not exist.`); } + } else if (opts.ccNewFile && opts.command == "cc-new-file") { + // so we don't have to depend on having our cc-new-file script + // installed. We just don't support templates on compute server. + for (const path of opts.args ?? []) { + const target = join(opts.home ?? process.env.HOME ?? "", path); + await ensureContainingDirectoryExists(target); + await writeFile(target, ""); + } + return { exit_code: 0, stdout: "", stderr: "", type: "blocking" }; } opts.args ??= []; diff --git a/src/packages/project/browser-websocket/realpath.ts b/src/packages/backend/realpath.ts similarity index 73% rename from src/packages/project/browser-websocket/realpath.ts rename to src/packages/backend/realpath.ts index 176b5d8b94..7a819e206c 100644 --- a/src/packages/project/browser-websocket/realpath.ts +++ b/src/packages/backend/realpath.ts @@ -14,11 +14,15 @@ import { realpath as fs_realpath } from "node:fs/promises"; const HOME: string = process.env.SMC_LOCAL_HUB_HOME ?? process.env.HOME ?? "/home/user"; -export async function realpath(path: string): Promise { - const fullpath = HOME + "/" + path; +export default async function realpath( + path: string, + home?: string, +): Promise { + home = home ?? HOME; + const fullpath = home + "/" + path; const rpath = await fs_realpath(fullpath); - if (rpath == fullpath || !rpath.startsWith(HOME + "/")) { + if (rpath == fullpath || !rpath.startsWith(home + "/")) { return path; } - return rpath.slice((HOME + "/").length); + return rpath.slice((home + "/").length); } diff --git a/src/packages/frontend/account/account-page.tsx b/src/packages/frontend/account/account-page.tsx index f64dcf6737..3023a5b73d 100644 --- a/src/packages/frontend/account/account-page.tsx +++ b/src/packages/frontend/account/account-page.tsx @@ -15,6 +15,7 @@ import { Space } from "antd"; import { useIntl } from "react-intl"; import { SignOut } from "@cocalc/frontend/account/sign-out"; import { AntdTabItem, Col, Row, Tabs } from "@cocalc/frontend/antd-bootstrap"; +import { entryPoint } from "@cocalc/frontend/app-framework/entry-point"; import { React, redux, @@ -259,6 +260,10 @@ export const AccountPage: React.FC = () => { function RedirectToNextApp({}) { const isMountedRef = useIsMountedRef(); + if (entryPoint == "compute") { + // no login page for compute cocalc app + return; + } useEffect(() => { const f = () => { diff --git a/src/packages/frontend/app-framework/entry-point.ts b/src/packages/frontend/app-framework/entry-point.ts new file mode 100644 index 0000000000..437d3c7193 --- /dev/null +++ b/src/packages/frontend/app-framework/entry-point.ts @@ -0,0 +1,14 @@ +type EntryPoint = + | "next" // the next frontend app from @cocalc/next + | "app" // the full normal frontend app, loaded from the main website + | "embed" // an embedded version of the app, e.g., kiosk mode + | "compute"; // cocalc loaded from the compute server + +let entryPoint: EntryPoint = "next"; +export { entryPoint }; + +// called only from the entry points themselves. I wish I could all this +// stuff using rspack, but I couldn't figure out how. +export function setEntryPoint(x): void { + entryPoint = x; +} diff --git a/src/packages/frontend/app/active-content.tsx b/src/packages/frontend/app/active-content.tsx index 654ce667d8..e52f6236a5 100644 --- a/src/packages/frontend/app/active-content.tsx +++ b/src/packages/frontend/app/active-content.tsx @@ -6,9 +6,11 @@ import { AccountPage } from "@cocalc/frontend/account/account-page"; import { AdminPage } from "@cocalc/frontend/admin"; import { Alert } from "@cocalc/frontend/antd-bootstrap"; +import { entryPoint } from "@cocalc/frontend/app-framework/entry-point"; import { CSS, React, + redux, useActions, useTypedRedux, } from "@cocalc/frontend/app-framework"; @@ -64,10 +66,17 @@ export const ActiveContent: React.FC = React.memo(() => { v.push(
{x} -
+ , ); }); + if (entryPoint == "compute") { + const project_id = redux + .getStore("customize") + .getIn(["compute_server", "project_id"]) as string; + return ; + } + if (get_api_key) { // Only render the account page which has the message for allowing api access: return ; @@ -100,7 +109,7 @@ export const ActiveContent: React.FC = React.memo(() => { . - + , ); } } diff --git a/src/packages/frontend/app/connection-info.tsx b/src/packages/frontend/app/connection-info.tsx index 46c0e0fa3d..5f4099e3d9 100644 --- a/src/packages/frontend/app/connection-info.tsx +++ b/src/packages/frontend/app/connection-info.tsx @@ -50,7 +50,7 @@ export const ConnectionInfo: React.FC = React.memo(() => {

@@ -74,7 +74,7 @@ export const ConnectionInfo: React.FC = React.memo(() => {

diff --git a/src/packages/frontend/app/page.tsx b/src/packages/frontend/app/page.tsx index 3572b67fde..a8f89f848e 100644 --- a/src/packages/frontend/app/page.tsx +++ b/src/packages/frontend/app/page.tsx @@ -14,6 +14,7 @@ import { useIntl } from "react-intl"; import { Avatar } from "@cocalc/frontend/account/avatar/avatar"; import { alert_message } from "@cocalc/frontend/alerts"; import { Button } from "@cocalc/frontend/antd-bootstrap"; +import { entryPoint } from "@cocalc/frontend/app-framework/entry-point"; import { CSS, React, @@ -200,7 +201,12 @@ export const Page: React.FC = () => { } function render_sign_in_tab(): JSX.Element | null { - if (is_logged_in) return null; + if (is_logged_in) { + return null; + } + if (entryPoint == "compute") { + return null; + } let style: CSS | undefined = undefined; if (active_top_tab !== "account") { diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index 092196bf74..a85a155a50 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -7,11 +7,8 @@ Functionality that mainly involves working with a specific project. */ -import { join } from "path"; - import { redux } from "@cocalc/frontend/app-framework"; import computeServers from "@cocalc/frontend/compute/manager"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import { dialogs, getIntl } from "@cocalc/frontend/i18n"; import { ipywidgetsGetBufferUrl } from "@cocalc/frontend/jupyter/server-urls"; import { allow_project_to_run } from "@cocalc/frontend/project/client-side-throttle"; @@ -46,7 +43,6 @@ import { coerce_codomain_to_numbers, copy_without, defaults, - encode_path, is_valid_uuid_string, required, } from "@cocalc/util/misc"; @@ -54,6 +50,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { DirectoryListingEntry } from "@cocalc/util/types"; import httpApi from "./api"; import { WebappClient } from "./client"; +import rawUrl from "@cocalc/frontend/lib/raw-url"; export class ProjectClient { private client: WebappClient; @@ -89,12 +86,11 @@ export class ProjectClient { project_id: string; // string or array of strings path: string; // string or array of strings }): string { - const base_path = appBasePath; if (opts.path[0] === "/") { // absolute path to the root opts.path = HOME_ROOT + opts.path; // use root symlink, which is created by start_smc } - return encode_path(join(base_path, `${opts.project_id}/raw/${opts.path}`)); + return rawUrl(opts); } public async copy_path_between_projects(opts: { diff --git a/src/packages/frontend/compute/entry-point.ts b/src/packages/frontend/compute/entry-point.ts new file mode 100644 index 0000000000..7af4ebde9f --- /dev/null +++ b/src/packages/frontend/compute/entry-point.ts @@ -0,0 +1,64 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +/* +Entry point for compute server version of CoCalc... +*/ + +// Load/initialize Redux-based react functionality +import "@cocalc/frontend/client/client"; +import { redux } from "@cocalc/frontend/app-framework"; +import { setEntryPoint } from "@cocalc/frontend/app-framework/entry-point"; +import "@cocalc/frontend/jquery-plugins"; +import { init as initAccount } from "@cocalc/frontend/account"; +import { init as initApp } from "@cocalc/frontend/app/init"; +import { init as initProjects } from "@cocalc/frontend/projects"; +import { init as initFileUse } from "@cocalc/frontend/file-use/init"; +import { init as initMarkdown } from "@cocalc/frontend/markdown/markdown-input/main"; +import { init as initCrashBanner } from "@cocalc/frontend/crash-banner"; +import "codemirror"; +import { init as initLast } from "@cocalc/frontend/last"; +import { render } from "@cocalc/frontend/app/render"; + +export async function init() { + setEntryPoint("compute"); + initAccount(redux); + initApp(); + initFileUse(); + initProjects(); + initMarkdown(); + initLast(); + try { + await initEntryPointState(); + await render(); + } finally { + // don't insert the crash banner until the main app has rendered, + // or user would see the banner for a moment. + initCrashBanner(); + } + console.log("Loaded Compute Server Entry Point."); +} + +import { fromJS } from "immutable"; +async function initEntryPointState() { + console.log("initEntryPointState"); + const customizeStore = redux.getStore("customize"); + await customizeStore.async_wait({ + until: () => customizeStore.get("compute_server"), + }); + const project_id = customizeStore.getIn([ + "compute_server", + "project_id", + ]) as string; + const project_map = fromJS({ + [project_id]: { + title: "Compute Server Project (TODO)", + state: { time: new Date(), state: "running" }, + }, + }) as any; + const actions = redux.getActions("projects"); + actions.setState({ project_map }); + actions.open_project({ project_id: "81e0c408-ac65-4114-bad5-5f4b6539bd0e" }); +} diff --git a/src/packages/frontend/customize.tsx b/src/packages/frontend/customize.tsx index 129046db46..61ba24376a 100644 --- a/src/packages/frontend/customize.tsx +++ b/src/packages/frontend/customize.tsx @@ -180,6 +180,10 @@ export interface CustomizeState { insecure_test_mode?: boolean; i18n?: List; + + // in case we're connecting directly to a compute server, + // this has info about the configuration of that server. + compute_server?: TypedMap<{ project_id: string }>; } export class CustomizeStore extends Store { diff --git a/src/packages/frontend/embed/index.ts b/src/packages/frontend/embed/index.ts index 3e492c8d3f..12c7d501e1 100644 --- a/src/packages/frontend/embed/index.ts +++ b/src/packages/frontend/embed/index.ts @@ -13,6 +13,8 @@ console.log("Embed mode"); // Load/initialize Redux-based react functionality import "@cocalc/frontend/client/client"; import { redux } from "../app-framework"; +import { setEntryPoint } from "@cocalc/frontend/app-framework/entry-point"; + import "../jquery-plugins"; // Initialize app stores, actions, etc. @@ -22,7 +24,6 @@ import { init as initProjects } from "../projects"; import { init as initMarkdown } from "../markdown/markdown-input/main"; import { init as initCrashBanner } from "../crash-banner"; - // Do not delete this without first looking at https://github.com/sagemathinc/cocalc/issues/5390 // This import of codemirror forces the initial full load of codemirror // as part of the main webpack entry point. @@ -32,6 +33,7 @@ import { init as initLast } from "../last"; import { render } from "../app/render"; export async function init() { + setEntryPoint("embed"); initAccount(redux); initApp(); initProjects(); diff --git a/src/packages/frontend/entry-point.ts b/src/packages/frontend/entry-point.ts index 21a9db783e..f970bf1b8f 100644 --- a/src/packages/frontend/entry-point.ts +++ b/src/packages/frontend/entry-point.ts @@ -14,6 +14,7 @@ import { COCALC_MINIMAL } from "./fullscreen"; // Load/initialize Redux-based react functionality import { redux } from "./app-framework"; +import { setEntryPoint } from "@cocalc/frontend/app-framework/entry-point"; // Systemwide notifications that are broadcast to all users (and set by admins) import "./system-notifications"; @@ -50,6 +51,7 @@ import { init as initLast } from "./last"; import { render } from "./app/render"; export async function init() { + setEntryPoint("app"); initAccount(redux); initApp(); initProjects(); diff --git a/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx b/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx index 4edf6e656a..85039a7633 100644 --- a/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx @@ -1064,6 +1064,9 @@ addCommands({ label: labels.save, keyboard: `${IS_MACOS ? "⌘" : "control"} + S`, }, + // DO NOT just change the name from chatgpt to something else, e.g., + // this is explicitly used in /frontend/jupyter/llm/cell-tool.tsx + // to search for and run the command. chatgpt: { pos: 1, group: "show-frames", diff --git a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx index f0e8caa4d8..c2452be7c5 100644 --- a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx @@ -85,6 +85,10 @@ export interface FrameActions extends Actions { // optional, set in frame-editors/jupyter-editor/editor.ts → initMenus jupyter_actions?: JupyterActions; frame_actions?: NotebookFrameActions; + + // the menu bar and buttons - can be used to explicitly run any menu command + // programatically, etc. + manageCommands?; } interface EditorActions extends Actions { @@ -253,6 +257,7 @@ export function FrameTitleBar(props: FrameTitleBarProps) { intl, ], ); + props.actions.manageCommands = manageCommands; const has_unsaved_changes: boolean = useRedux([ props.editor_actions.name, diff --git a/src/packages/frontend/frame-editors/frame-tree/util.ts b/src/packages/frontend/frame-editors/frame-tree/util.ts index 0164bf7b86..0f8a25a427 100644 --- a/src/packages/frontend/frame-editors/frame-tree/util.ts +++ b/src/packages/frontend/frame-editors/frame-tree/util.ts @@ -8,9 +8,7 @@ Utility functions useful for frame-tree editors. */ import { path_split, separate_file_extension } from "@cocalc/util/misc"; -import { encode_path } from "@cocalc/util/misc"; -import { join } from "path"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; +import rawUrl from "@cocalc/frontend/lib/raw-url"; export function parse_path(path: string): { directory: string; @@ -22,10 +20,7 @@ export function parse_path(path: string): { return { directory: x.head, base: y.name, filename: x.tail }; } +// todo: rewrite everything that calls this... export function raw_url(project_id: string, path: string): string { - // we have to encode the path, since we query this raw server. see - // https://github.com/sagemathinc/cocalc/issues/5542 - // but actually, this is a problem for types of files, not just PDF - const path_enc = encode_path(path); - return join(appBasePath, project_id, "raw", path_enc); + return rawUrl({ project_id, path }); } diff --git a/src/packages/frontend/i18n/common.ts b/src/packages/frontend/i18n/common.ts index 53f50d35dc..17c5fa1d78 100644 --- a/src/packages/frontend/i18n/common.ts +++ b/src/packages/frontend/i18n/common.ts @@ -568,7 +568,7 @@ export const labels = defineMessages({ }, message_plural: { id: "labels.messsage.plural", - defaultMessage: "{num, plural, one {message} other {messages}}", + defaultMessage: "{num, plural, one {Message} other {Messages}}", }, reconnect: { id: "labels.reconnect", diff --git a/src/packages/frontend/i18n/trans/extracted.json b/src/packages/frontend/i18n/trans/extracted.json index 7c4479144d..816efe4991 100644 --- a/src/packages/frontend/i18n/trans/extracted.json +++ b/src/packages/frontend/i18n/trans/extracted.json @@ -1024,7 +1024,7 @@ "defaultMessage": "Translate" }, "jupyter.llm.cell-tool.assistant.title": { - "defaultMessage": "Use AI assitant on this cell" + "defaultMessage": "Use AI assistant on this cell" }, "jupyter.llm.cell-tool.explanation.bugfix": { "defaultMessage": "Explain the problem of the code in the cell and the selected language model will attempt to fix it. Usually, it will tell you if it found a problem and explain it to you." diff --git a/src/packages/frontend/jupyter/llm/cell-tool.tsx b/src/packages/frontend/jupyter/llm/cell-tool.tsx index 6b004974f0..7ff1bd4894 100644 --- a/src/packages/frontend/jupyter/llm/cell-tool.tsx +++ b/src/packages/frontend/jupyter/llm/cell-tool.tsx @@ -431,49 +431,65 @@ export function LLMCellTool({ actions, id, style, llmTools }: Props) { function renderDropdown() { return ( - ).map( - ([mode, action]) => { - return { - key: mode, - label: ( - - {" "} - {intl.formatMessage(action.label)} - - ), - onClick: () => setMode(mode as Mode), - }; - }, - ), - }} + - + - - + ).map( + ([mode, action]) => { + return { + key: mode, + label: ( + + {" "} + {intl.formatMessage(action.label)} + + ), + onClick: () => setMode(mode as Mode), + }; + }, + ), + }} + > + + + + ); } diff --git a/src/packages/frontend/jupyter/nbconvert.tsx b/src/packages/frontend/jupyter/nbconvert.tsx index 7b3aa82516..711b3041e1 100644 --- a/src/packages/frontend/jupyter/nbconvert.tsx +++ b/src/packages/frontend/jupyter/nbconvert.tsx @@ -9,7 +9,7 @@ NBConvert dialog -- for running nbconvert import { Button, Modal } from "antd"; import * as immutable from "immutable"; import React, { useEffect, useRef } from "react"; - +import rawUrl from "@cocalc/frontend/lib/raw-url"; import { redux } from "@cocalc/frontend/app-framework"; import { A, Icon, Loading, TimeAgo } from "@cocalc/frontend/components"; import * as misc from "@cocalc/util/misc"; @@ -196,7 +196,7 @@ export const NBConvert: React.FC = React.memo( ext = info.ext; } const targetPath = misc.change_filename_extension(path, ext); - const url = actions.store.get_raw_link(targetPath); + const url = rawUrl({ path: targetPath, project_id }); return { targetPath, url, info }; } diff --git a/src/packages/frontend/jupyter/use-kernels-info.ts b/src/packages/frontend/jupyter/use-kernels-info.ts index 70c5662be6..9b51a3ed9d 100644 --- a/src/packages/frontend/jupyter/use-kernels-info.ts +++ b/src/packages/frontend/jupyter/use-kernels-info.ts @@ -6,7 +6,6 @@ import { fromJS } from "immutable"; import { useEffect, useMemo, useState } from "react"; import useAsyncEffect from "use-async-effect"; - import { getKernelInfo } from "@cocalc/frontend/components/run-button/kernel-info"; import { useProjectContext } from "@cocalc/frontend/project/context"; import { diff --git a/src/packages/frontend/lib/raw-url.ts b/src/packages/frontend/lib/raw-url.ts index 8287c85cb0..d070fc6eff 100644 --- a/src/packages/frontend/lib/raw-url.ts +++ b/src/packages/frontend/lib/raw-url.ts @@ -2,10 +2,14 @@ The raw URL is the following, of course encoded as a URL: .../{project_id}/raw/{full relative path in the project to file} + +On a compute server though the project_id is not redundant (there is only one project), +so not in the URL. */ import { join } from "path"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; +import { encode_path } from "@cocalc/util/misc"; interface Options { project_id: string; @@ -13,14 +17,9 @@ interface Options { } export default function rawURL({ project_id, path }: Options): string { - return join(appBasePath, project_id, "raw", encodePath(path)); -} - -export function encodePath(path: string) { - const segments = path.split("/"); - const encoded: string[] = []; - for (const segment of segments) { - encoded.push(encodeURIComponent(segment)); - } - return encoded.join("/"); + // we have to encode the path, since we query this raw server. see + // https://github.com/sagemathinc/cocalc/issues/5542 + // but actually, this is a problem for types of files, not just PDF + const path_enc = encode_path(path); + return join(appBasePath, project_id, "raw", path_enc); } diff --git a/src/packages/frontend/misc/process-links/generic.ts b/src/packages/frontend/misc/process-links/generic.ts index c1fc5327d2..13a38eee12 100644 --- a/src/packages/frontend/misc/process-links/generic.ts +++ b/src/packages/frontend/misc/process-links/generic.ts @@ -13,9 +13,9 @@ Define a jQuery plugin that processes links. import { join } from "path"; import { is_valid_uuid_string as isUUID } from "@cocalc/util/misc"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import { isCoCalcURL } from "@cocalc/frontend/lib/cocalc-urls"; import Fragment, { FragmentId } from "@cocalc/frontend/misc/fragment-id"; +import rawUrl from "@cocalc/frontend/lib/raw-url"; type jQueryAPI = Function; @@ -191,15 +191,14 @@ function processMediaTag( // absolute path or data: url newSrc = src; } else if (opts.projectId != null && opts.filePath != null) { - let projectId: string; const i = src.indexOf("/projects/"); const j = src.indexOf("/files/"); if (isCoCalcURL(src) && i !== -1 && j !== -1 && j > i) { // the href is inside the app, points to the current project or another one // j-i should be 36, unless we ever start to have different (vanity) project_ids const path = src.slice(j + "/files/".length); - projectId = src.slice(i + "/projects/".length, j); - newSrc = join(appBasePath, projectId, "raw", path); + const project_id = src.slice(i + "/projects/".length, j); + newSrc = rawUrl({ project_id, path }); y.attr(attr, newSrc); return; } @@ -209,7 +208,7 @@ function processMediaTag( } // we do not have an absolute url, hence we assume it is a // relative URL to a file in a project - newSrc = join(appBasePath, opts.projectId, "raw", opts.filePath, src); + newSrc = rawUrl({ project_id: opts.projectId, path: opts.filePath }); } if (newSrc != null) { y.attr(attr, newSrc); diff --git a/src/packages/frontend/project/explorer/download.tsx b/src/packages/frontend/project/explorer/download.tsx index e41544f538..b64b53c910 100644 --- a/src/packages/frontend/project/explorer/download.tsx +++ b/src/packages/frontend/project/explorer/download.tsx @@ -8,6 +8,7 @@ import CheckedFiles from "./checked-files"; import ShowError from "@cocalc/frontend/components/error"; import { PRE_STYLE } from "./action-box"; import { Icon } from "@cocalc/frontend/components/icon"; +import rawUrl from "@cocalc/frontend/lib/raw-url"; export default function Download({}) { const inputRef = useRef(null); @@ -36,14 +37,13 @@ export default function Download({}) { setArchiveMode(true); return; } - const file = checked_files.first(); + const path = checked_files.first(); const isdir = redux.getProjectStore(project_id).get("displayed_listing") - ?.file_map?.[path_split(file).tail]?.isdir; - console.log({ isdir }); + ?.file_map?.[path_split(path).tail]?.isdir; setArchiveMode(!!isdir); if (!isdir) { - const store = actions?.get_store(); - setUrl(store?.get_raw_link(file) ?? ""); + const url = rawUrl({ project_id, path }); + setUrl(`${document.location.origin}${url}`); } }, [checked_files, current_path]); diff --git a/src/packages/frontend/project/explorer/file-listing/file-row.tsx b/src/packages/frontend/project/explorer/file-listing/file-row.tsx index 25b4b58066..8962a9a784 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-row.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-row.tsx @@ -21,7 +21,7 @@ import { ProjectActions } from "@cocalc/frontend/project_actions"; import track from "@cocalc/frontend/user-tracking"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; -import { url_href } from "../../utils"; +import rawUrl from "@cocalc/frontend/lib/raw-url"; import { FileCheckbox } from "./file-checkbox"; import { PublicButton } from "./public-button"; import { generate_click_for } from "./utils"; @@ -335,7 +335,10 @@ export const FileRow: React.FC = React.memo((props) => { // See https://github.com/sagemathinc/cocalc/issues/1020 // support right-click → copy url for the download button - const url = url_href(props.actions.project_id, full_path()); + const url = rawUrl({ + project_id: props.actions.project_id, + path: full_path(), + }); return ( ) => { ? item.isopen ? { fontWeight: "bold" } : item.isdir - ? undefined - : { color: COLORS.FILE_EXT } + ? undefined + : { color: COLORS.FILE_EXT } : undefined; return ( @@ -287,8 +287,8 @@ export const FileListItem = React.memo((props: Readonly) => { ? "check-square" : "square" : item.isdir - ? "folder-open" - : file_options(item.name)?.icon ?? "file"); + ? "folder-open" + : (file_options(item.name)?.icon ?? "file")); return ( ) => { const actionNames = multiple ? ACTION_BUTTONS_MULTI : isdir - ? ACTION_BUTTONS_DIR - : ACTION_BUTTONS_FILE; + ? ACTION_BUTTONS_DIR + : ACTION_BUTTONS_FILE; for (const name of actionNames) { if (name === "download" && !item.isdir) continue; const disabled = @@ -500,7 +500,7 @@ export const FileListItem = React.memo((props: Readonly) => { const full_path = path_to_file(current_path, name); const ext = (filename_extension(name) ?? "").toLowerCase(); const showView = VIEWABLE_FILE_EXT.includes(ext); - const url = url_href(project_id, full_path); + const url = rawUrl({ project_id, path: full_path }); ctx.push({ key: "divide-download", type: "divider" }); @@ -533,10 +533,10 @@ export const FileListItem = React.memo((props: Readonly) => { ? FILE_ITEM_ACTIVE_STYLE_2 : {} : item.isopen - ? item.isactive - ? FILE_ITEM_ACTIVE_STYLE - : FILE_ITEM_OPENED_STYLE - : {}; + ? item.isactive + ? FILE_ITEM_ACTIVE_STYLE + : FILE_ITEM_OPENED_STYLE + : {}; return ( diff --git a/src/packages/frontend/project/page/flyouts/files-bottom.tsx b/src/packages/frontend/project/page/flyouts/files-bottom.tsx index 35cd72f74a..a2f49b9306 100644 --- a/src/packages/frontend/project/page/flyouts/files-bottom.tsx +++ b/src/packages/frontend/project/page/flyouts/files-bottom.tsx @@ -29,7 +29,7 @@ import { DirectoryListing, DirectoryListingEntry, } from "@cocalc/frontend/project/explorer/types"; -import { url_href } from "@cocalc/frontend/project/utils"; +import rawUrl from "@cocalc/frontend/lib/raw-url"; import { filename_extension, human_readable_size, @@ -191,7 +191,7 @@ export function FilesBottom({ const ext = (filename_extension(name) ?? "").toLowerCase(); const showView = VIEWABLE_FILE_EXT.includes(ext); // the "href" part makes the link right-click copyable - const url = url_href(project_id, full_path); + const url = rawUrl({ project_id, path: full_path }); const showDownload = !student_project_functionality.disableActions; const sizeStr = human_readable_size(size); @@ -309,7 +309,7 @@ export function FilesBottom({ const name = singleFile.name; const iconName = singleFile.isdir ? "folder" - : file_options(name)?.icon ?? "file"; + : (file_options(name)?.icon ?? "file"); return (
{trunc_middle(name, 20)} diff --git a/src/packages/frontend/project/page/home-page/ai-generate-document.tsx b/src/packages/frontend/project/page/home-page/ai-generate-document.tsx index 74a0743500..991352082c 100644 --- a/src/packages/frontend/project/page/home-page/ai-generate-document.tsx +++ b/src/packages/frontend/project/page/home-page/ai-generate-document.tsx @@ -25,7 +25,6 @@ import { delay } from "awaiting"; import { debounce, isEmpty, throttle } from "lodash"; import { useEffect, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { useLanguageModelSetting } from "@cocalc/frontend/account/useLanguageModelSetting"; import { CSS, diff --git a/src/packages/frontend/project/page/project-status-hook.ts b/src/packages/frontend/project/page/project-status-hook.ts index be13c434cd..56d8e4c296 100644 --- a/src/packages/frontend/project/page/project-status-hook.ts +++ b/src/packages/frontend/project/page/project-status-hook.ts @@ -23,7 +23,9 @@ export function useProjectStatus(actions?: ProjectActions): void { } function connect() { - if (project_id == null) return; + if (project_id == null) { + return; + } const status_sync = webapp_client.project_client.project_status(project_id); statusRef.current = status_sync; const update = () => { diff --git a/src/packages/frontend/project/utils.ts b/src/packages/frontend/project/utils.ts index 9ed7ac5038..eade28101b 100644 --- a/src/packages/frontend/project/utils.ts +++ b/src/packages/frontend/project/utils.ts @@ -8,7 +8,6 @@ import * as dogNames from "dog-names"; import * as os_path from "path"; import { generate as heroku } from "project-name-generator"; import * as superb from "superb"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import { file_options } from "@cocalc/frontend/editor-tmp"; import { BASE_URL } from "@cocalc/frontend/misc"; import { webapp_client } from "@cocalc/frontend/webapp-client"; @@ -25,6 +24,7 @@ import { unreachable, uuid, } from "@cocalc/util/misc"; +import rawUrl from "@cocalc/frontend/lib/raw-url"; export function randomPetName() { return Math.random() > 0.5 ? catNames.random() : dogNames.allRandom(); @@ -289,14 +289,9 @@ export function url_fullpath(project_id: string, path: string): string { ); } -// returns the URL for the file at the given path -export function url_href(project_id: string, path: string): string { - return os_path.join(appBasePath, project_id, "raw", encode_path(path)); -} - // returns the download URL for a file at a given path export function download_href(project_id: string, path: string): string { - return `${url_href(project_id, path)}?download`; + return `${rawUrl({ project_id, path })}?download`; } export function in_snapshot_path(path: string): boolean { diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index ed9712c5bf..53ca9d29a1 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -52,7 +52,7 @@ export class API { try { this.cachedVersion = await this.call({ cmd: "version" }, 15000); } catch (err) { - if (err.message.includes('command "version" not implemented')) { + if (err.message?.includes('command "version" not implemented')) { this.cachedVersion = 0; } else { throw err; diff --git a/src/packages/frontend/project/websocket/connect.ts b/src/packages/frontend/project/websocket/connect.ts index 2b04449565..5e28edd24c 100644 --- a/src/packages/frontend/project/websocket/connect.ts +++ b/src/packages/frontend/project/websocket/connect.ts @@ -16,8 +16,8 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { callback, delay } from "awaiting"; import { ajax, globalEval } from "jquery"; import { join } from "path"; - import { redux } from "@cocalc/frontend/app-framework"; +import { entryPoint } from "@cocalc/frontend/app-framework/entry-point"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import { webapp_client } from "@cocalc/frontend/webapp-client"; import { allow_project_to_run } from "../client-side-throttle"; @@ -88,17 +88,20 @@ async function connection_to_project0(project_id: string): Promise { throw Error("currently reading one already"); } - if (!webapp_client.is_signed_in()) { - // At least wait until main client is signed in, since nothing - // will work until that is the case anyways. - await once(webapp_client, "signed_in"); - } + // for compute entry point, no sign in needed and project is assumed running. + if (entryPoint != "compute") { + if (!webapp_client.is_signed_in()) { + // At least wait until main client is signed in, since nothing + // will work until that is the case anyways. + await once(webapp_client, "signed_in"); + } - log("wait_for_project_to_start..."); - await wait_for_project_to_start(project_id); - log("wait_for_project_to_start: done"); + log("wait_for_project_to_start..."); + await wait_for_project_to_start(project_id); + log("wait_for_project_to_start: done"); + } - // Now project is thought to be running, so maybe this will work: + // Now user is signed in and project is thought to be running, so maybe this will work: try { if (do_eval) { READING_PRIMUS_JS = true; diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 4588efbcdc..8c31010dac 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -84,7 +84,6 @@ import { download_href, in_snapshot_path, normalize, - url_href, } from "@cocalc/frontend/project/utils"; import { API } from "@cocalc/frontend/project/websocket/api"; import { @@ -101,6 +100,7 @@ import { ProjectStoreState, } from "@cocalc/frontend/project_store"; import { webapp_client } from "@cocalc/frontend/webapp-client"; +import rawUrl from "@cocalc/frontend/lib/raw-url"; const { defaults, required } = misc; @@ -2571,7 +2571,7 @@ export class ProjectActions extends Actions { url = download_href(this.project_id, opts.path); download_file(url); } else { - url = url_href(this.project_id, opts.path); + url = rawUrl({ project_id: this.project_id, path: opts.path }); const tab = open_new_tab(url); if (tab != null && opts.print) { // "?" since there might be no print method -- could depend on browser API diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index aa1f814246..b1dfe0fa5c 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -16,7 +16,6 @@ if (typeof window !== "undefined" && window !== null) { } import * as immutable from "immutable"; - import { alert_message } from "@cocalc/frontend/alerts"; import { AppRedux, @@ -559,12 +558,6 @@ export class ProjectStore extends Store { }; }; - get_raw_link = (path) => { - let url = document.URL; - url = url.slice(0, url.indexOf("/projects/")); - return `${url}/${this.project_id}/raw/${misc.encode_path(path)}`; - }; - // returns false, if this project isn't capable of opening a file with the given extension async can_open_file_ext( ext: string, diff --git a/src/packages/frontend/projects/projects-nav.tsx b/src/packages/frontend/projects/projects-nav.tsx index 774ba952e1..1ad386371a 100644 --- a/src/packages/frontend/projects/projects-nav.tsx +++ b/src/packages/frontend/projects/projects-nav.tsx @@ -5,7 +5,7 @@ import type { TabsProps } from "antd"; import { Avatar, Popover, Tabs, Tooltip } from "antd"; - +import { entryPoint } from "@cocalc/frontend/app-framework/entry-point"; import { redux, useActions, @@ -207,6 +207,9 @@ function ProjectTab({ project_id }: ProjectTabProps) { } function onMouseUp(e: React.MouseEvent) { + if (entryPoint == "compute") { + return; + } // if middle mouse button has been clicked, close the project if (e.button === 1) { e.stopPropagation(); @@ -215,6 +218,31 @@ function ProjectTab({ project_id }: ProjectTabProps) { } } + const ws =
{renderWebsocketIndicator()}
; + + if (entryPoint == "compute") { + return ( +
+ {ws} +
+ {icon} + {renderNoInternet()} + {renderAvatar()}{" "} + {title} +
+
+ ); + } + const body = (
{renderWebsocketIndicator()}
@@ -263,6 +291,7 @@ export function ProjectsNav(props: ProjectsNavProps) { return { label: , key: project_id, + closable: entryPoint != "compute", }; }); }, [openProjects]); @@ -318,7 +347,7 @@ export function ProjectsNav(props: ProjectsNavProps) { onChange={(project_id) => { actions.set_active_tab(project_id); }} - type={"editable-card"} + type={entryPoint == "compute" ? "card" : "editable-card"} renderTabBar={renderTabBar} items={items} /> diff --git a/src/packages/frontend/projects/store.ts b/src/packages/frontend/projects/store.ts index e9d5b400bf..06645c383f 100644 --- a/src/packages/frontend/projects/store.ts +++ b/src/packages/frontend/projects/store.ts @@ -5,7 +5,12 @@ import { List, Map, Set } from "immutable"; import { fromPairs, isEmpty } from "lodash"; import LRU from "lru-cache"; -import { redux, Store, TypedMap } from "@cocalc/frontend/app-framework"; +import { entryPoint } from "@cocalc/frontend/app-framework/entry-point"; +import { + redux, + Store, + TypedMap, +} from "@cocalc/frontend/app-framework"; import { StudentProjectFunctionality } from "@cocalc/frontend/course/configuration/customize-student-project-functionality"; import { CUSTOM_IMG_PREFIX } from "@cocalc/frontend/custom-software/util"; import { WebsocketState } from "@cocalc/frontend/project/websocket/websocket-state"; @@ -243,6 +248,10 @@ export class ProjectsStore extends Store { 'admin' - user is not owner/collaborator but is an admin, hence has rights. */ public get_my_group(project_id: string): UserGroup | undefined { + if (entryPoint == "compute") { + // in compute server mode there is only one project. + return "owner"; + } const account_store = redux.getStore("account"); if (account_store == null) { return; diff --git a/src/packages/frontend/user-tracking.ts b/src/packages/frontend/user-tracking.ts index 21bbc8b254..bf7fd50a9e 100644 --- a/src/packages/frontend/user-tracking.ts +++ b/src/packages/frontend/user-tracking.ts @@ -10,6 +10,7 @@ import { query, server_time } from "./frame-editors/generic/client"; import { analytics_cookie_name as analytics, uuid } from "@cocalc/util/misc"; import { redux } from "./app-framework"; +import { entryPoint } from "@cocalc/frontend/app-framework/entry-point"; import { version } from "@cocalc/util/smc-version"; import { get_cookie } from "./misc"; import { webapp_client } from "./webapp-client"; @@ -41,8 +42,12 @@ export async function log(eventName: string, payload: any): Promise { // shows a warning in the console when it can't report to the backend. export default async function track( event: string, - value: object + value: object, ): Promise { + if (entryPoint == "compute") { + // for now, we do no tracking of users when using compute servers directly. + return; + } // Replace all dashes with underscores in the event argument for consistency event = event.replace(/-/g, "_"); diff --git a/src/packages/hub/servers/express-app.ts b/src/packages/hub/servers/express-app.ts index fcbbb4aded..690954737a 100644 --- a/src/packages/hub/servers/express-app.ts +++ b/src/packages/hub/servers/express-app.ts @@ -5,7 +5,6 @@ The main hub express app. import compression from "compression"; import cookieParser from "cookie-parser"; import express from "express"; -import ms from "ms"; import { join } from "path"; import { parse as parseURL } from "url"; import webpackDevMiddleware from "webpack-dev-middleware"; @@ -32,10 +31,7 @@ import initStripeWebhook from "./app/webhooks/stripe"; import { database } from "./database"; import initHttpServer from "./http"; import initRobots from "./robots"; - -// Used for longterm caching of files. This should be in units of seconds. -const MAX_AGE = Math.round(ms("10 days") / 1000); -const SHORT_AGE = Math.round(ms("10 seconds") / 1000); +import { cacheShortTerm, cacheLongTerm } from "@cocalc/util/http-caching"; interface Options { projectControl; @@ -166,30 +162,6 @@ export default async function init(opts: Options): Promise<{ return { httpServer, router }; } -function cacheShortTerm(res) { - res.setHeader( - "Cache-Control", - `public, max-age=${SHORT_AGE}, must-revalidate`, - ); - res.setHeader( - "Expires", - new Date(Date.now().valueOf() + SHORT_AGE).toUTCString(), - ); -} - -// Various files such as the webpack static content should be cached long-term, -// and we use this function to set appropriate headers at various points below. -function cacheLongTerm(res) { - res.setHeader( - "Cache-Control", - `public, max-age=${MAX_AGE}, must-revalidate'`, - ); - res.setHeader( - "Expires", - new Date(Date.now().valueOf() + MAX_AGE).toUTCString(), - ); -} - async function initStatic(router) { let compiler: any = null; if ( @@ -215,7 +187,7 @@ async function initStatic(router) { router.use("/static", webpackHotMiddleware(compiler, {})); } else { router.use( - join("/static", STATIC_PATH, "app.html"), + join("/static", "app.html"), express.static(join(STATIC_PATH, "app.html"), { setHeaders: cacheShortTerm, }), diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index d5bccb0d07..f02d32608f 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -159,7 +159,7 @@ export abstract class JupyterActions extends Actions { } // Only use this on the frontend, of course. - protected getFrameActions() { + getFrameActions() { return this.redux.getEditorActions(this.project_id, this.path); } diff --git a/src/packages/jupyter/redux/store.ts b/src/packages/jupyter/redux/store.ts index 82525e3f03..073fe23d02 100644 --- a/src/packages/jupyter/redux/store.ts +++ b/src/packages/jupyter/redux/store.ts @@ -136,7 +136,10 @@ export class JupyterStore extends Store { // TODO: We use unsafe_getIn because maybe the cell type isn't spelled out yet, or our typescript isn't good enough. const type = this.unsafe_getIn(["cells", id, "cell_type"], "code"); if (type != "markdown" && type != "code" && type != "raw") { - throw Error(`invalid cell type ${type} for cell ${id}`); + console.warn( + `Jupyter: invalid cell type ${type} for cell ${id} -- falling back to raw`, + ); + return "raw"; } return type; } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index a6d4d7d987..5c598d5aea 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1188,6 +1188,9 @@ importers: '@cocalc/sync-fs': specifier: workspace:* version: link:../sync-fs + '@cocalc/sync-server': + specifier: workspace:* + version: link:../sync-server '@cocalc/terminal': specifier: workspace:* version: link:../terminal @@ -1224,9 +1227,6 @@ importers: debug: specifier: ^4.3.2 version: 4.3.7(supports-color@9.4.0) - diskusage: - specifier: ^1.1.3 - version: 1.2.0 expect: specifier: ^26.6.2 version: 26.6.2 @@ -1900,6 +1900,55 @@ importers: specifier: ^18.16.14 version: 18.19.50 + sync-server: + dependencies: + '@cocalc/backend': + specifier: workspace:* + version: link:../backend + '@cocalc/comm': + specifier: workspace:* + version: link:../comm + '@cocalc/jupyter': + specifier: workspace:* + version: link:../jupyter + '@cocalc/sync': + specifier: workspace:* + version: link:../sync + '@cocalc/sync-server': + specifier: workspace:* + version: 'link:' + '@cocalc/terminal': + specifier: workspace:* + version: link:../terminal + '@cocalc/util': + specifier: workspace:* + version: link:../util + awaiting: + specifier: ^3.0.0 + version: 3.0.0 + diskusage: + specifier: ^1.1.3 + version: 1.2.0 + events: + specifier: 3.3.0 + version: 3.3.0 + json-stable-stringify: + specifier: ^1.0.1 + version: 1.1.1 + lodash: + specifier: ^4.17.21 + version: 4.17.21 + devDependencies: + '@types/json-stable-stringify': + specifier: ^1.0.32 + version: 1.0.36 + '@types/lodash': + specifier: ^4.14.202 + version: 4.17.9 + '@types/node': + specifier: ^18.16.14 + version: 18.19.55 + terminal: dependencies: '@cocalc/api-client': @@ -1957,6 +2006,9 @@ importers: '@types/debug': specifier: ^4.1.12 version: 4.1.12 + '@types/ms': + specifier: ^0.7.31 + version: 0.7.34 async: specifier: ^1.5.2 version: 1.5.2 @@ -1993,6 +2045,9 @@ importers: lru-cache: specifier: ^7.18.3 version: 7.18.3 + ms: + specifier: 2.1.2 + version: 2.1.2 prop-types: specifier: ^15.7.2 version: 15.8.1 @@ -4110,9 +4165,6 @@ packages: '@types/mocha@10.0.8': resolution: {integrity: sha512-HfMcUmy9hTMJh66VNcmeC9iVErIZJli2bszuXc6julh5YGuRb/W5OnkHjwLNYdFlMis0sY3If5SEAp+PktdJjw==} - '@types/ms@0.7.31': - resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} - '@types/ms@0.7.34': resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} @@ -8711,9 +8763,6 @@ packages: nan@2.17.0: resolution: {integrity: sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==} - nan@2.19.0: - resolution: {integrity: sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==} - nan@2.20.0: resolution: {integrity: sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==} @@ -12947,7 +12996,7 @@ snapshots: dependencies: '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.1 - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/yargs': 15.0.19 chalk: 4.1.2 @@ -12956,7 +13005,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.1 - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/yargs': 17.0.24 chalk: 4.1.2 @@ -14153,7 +14202,7 @@ snapshots: '@types/connect@3.4.35': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/cookie@0.3.3': {} @@ -14161,7 +14210,7 @@ snapshots: '@types/debug@4.1.12': dependencies: - '@types/ms': 0.7.31 + '@types/ms': 0.7.34 '@types/dot-object@2.1.6': {} @@ -14182,7 +14231,7 @@ snapshots: '@types/express-serve-static-core@4.19.0': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/qs': 6.9.7 '@types/range-parser': 1.2.4 '@types/send': 0.17.4 @@ -14218,7 +14267,7 @@ snapshots: '@types/glob@7.2.0': dependencies: '@types/minimatch': 5.1.2 - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/graceful-fs@4.1.9': dependencies: @@ -14270,11 +14319,11 @@ snapshots: '@types/keyv@3.1.4': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/ldapjs@2.2.5': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/linkify-it@5.0.0': {} @@ -14284,7 +14333,7 @@ snapshots: '@types/lz4@0.6.4': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/mapbox__point-geometry@0.1.4': {} @@ -14313,15 +14362,13 @@ snapshots: '@types/mocha@10.0.8': {} - '@types/ms@0.7.31': {} - '@types/ms@0.7.34': {} '@types/node-cleanup@2.1.5': {} '@types/node-fetch@2.6.11': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.55 form-data: 4.0.0 '@types/node-forge@1.3.11': @@ -14354,7 +14401,7 @@ snapshots: '@types/oauth@0.9.1': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/parse5@6.0.3': {} @@ -14424,13 +14471,13 @@ snapshots: '@types/request@2.48.12': dependencies: '@types/caseless': 0.12.5 - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/tough-cookie': 4.0.5 form-data: 2.5.1 '@types/responselike@1.0.3': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/retry@0.12.0': {} @@ -14447,7 +14494,7 @@ snapshots: '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/serve-index@1.9.4': dependencies: @@ -14456,7 +14503,7 @@ snapshots: '@types/serve-static@1.15.0': dependencies: '@types/mime': 3.0.1 - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/serve-static@1.15.7': dependencies: @@ -14503,16 +14550,16 @@ snapshots: '@types/xml-crypto@1.4.6': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.55 xpath: 0.0.27 '@types/xml-encryption@1.2.4': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/xml2js@0.4.14': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.55 '@types/yargs-parser@21.0.0': {} @@ -15446,7 +15493,7 @@ snapshots: canvas@2.11.2(encoding@0.1.13): dependencies: '@mapbox/node-pre-gyp': 1.0.11(encoding@0.1.13) - nan: 2.19.0 + nan: 2.20.0 simple-get: 3.1.1 transitivePeerDependencies: - encoding @@ -18852,7 +18899,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 18.19.50 + '@types/node': 18.19.55 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -19773,9 +19820,6 @@ snapshots: nan@2.17.0: {} - nan@2.19.0: - optional: true - nan@2.20.0: {} nanoid@3.3.6: {} @@ -20913,7 +20957,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 18.19.50 + '@types/node': 18.19.55 long: 5.2.3 protocol-buffers-schema@3.6.0: {} diff --git a/src/packages/project/autorenice.ts b/src/packages/project/autorenice.ts index de83fc5e89..f01f80cf18 100644 --- a/src/packages/project/autorenice.ts +++ b/src/packages/project/autorenice.ts @@ -13,7 +13,10 @@ import { reverse, sortBy } from "lodash"; import { setPriority } from "node:os"; import { getLogger } from "./logger"; -import { ProjectInfoServer, get_ProjectInfoServer } from "./project-info"; +import { + ProjectInfoServer, + get_ProjectInfoServer, +} from "@cocalc/sync-server/monitor/activity"; import { Process, Processes, diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts index bf45024e51..fce0452d7f 100644 --- a/src/packages/project/browser-websocket/api.ts +++ b/src/packages/project/browser-websocket/api.ts @@ -21,18 +21,18 @@ import { nbconvert as jupyter_nbconvert } from "../jupyter/convert"; import { lean, lean_channel } from "../lean/server"; import { jupyter_strip_notebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; import { jupyter_run_notebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; -import { synctable_channel } from "../sync/server"; -import { syncdoc_call } from "../sync/sync-doc"; +import { synctable_channel } from "@cocalc/sync-server/server/server"; +import { callSyncDoc } from "@cocalc/sync-server/server/syncdocs-manager"; import { terminal } from "@cocalc/terminal"; -import { x11_channel } from "../x11/server"; +import { x11_channel } from "@cocalc/sync-server/x11"; import { canonical_paths } from "./canonical-path"; import { delete_files } from "@cocalc/backend/files/delete-files"; -import { eval_code } from "./eval-code"; +import { eval_code } from "@cocalc/backend/eval-code"; import computeFilesystemCache from "./compute-filesystem-cache"; import { move_files } from "@cocalc/backend/files/move-files"; import { rename_file } from "@cocalc/backend/files/rename-file"; -import { realpath } from "./realpath"; -import { project_info_ws } from "../project-info"; +import realpath from "@cocalc/backend/realpath"; +import { project_info_ws } from "@cocalc/sync-server/monitor/activity"; import query from "./query"; import { browser_symmetric_channel } from "./symmetric_channel"; import type { Mesg } from "@cocalc/comm/websocket/types"; @@ -167,7 +167,7 @@ async function handleApiCall(data: Mesg, spark): Promise { data.options, ); case "syncdoc_call": - return await syncdoc_call(data.path, data.mesg); + return await callSyncDoc(data.path, data.mesg); case "symmetric_channel": return await browser_symmetric_channel(client, primus, log, data.name); case "realpath": diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index f3cab0492c..df84003786 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -45,9 +45,9 @@ import initJupyter from "./jupyter/init"; import * as kucalc from "./kucalc"; import { getLogger } from "./logger"; import * as sage_session from "./sage_session"; -import { getListingsTable } from "@cocalc/project/sync/listings"; -import { get_synctable } from "./sync/open-synctables"; -import { get_syncdoc } from "./sync/sync-doc"; +import { getListingsTable } from "@cocalc/sync-server/server/listings"; +import { get_synctable } from "@cocalc/sync-server/server/open-synctables"; +import { getSyncDoc } from "@cocalc/sync-server/server/syncdocs-manager"; const winston = getLogger("client"); @@ -506,7 +506,7 @@ export class Client extends EventEmitter implements ProjectClientInterface { // Get the synchronized doc with the given path. Returns undefined // if currently no such sync-doc. public syncdoc({ path }: { path: string }): SyncDoc | undefined { - return get_syncdoc(path); + return getSyncDoc(path); } public symmetric_channel(name) { diff --git a/src/packages/project/jupyter/http-server.ts b/src/packages/project/jupyter/http-server.ts index 29cd44eb65..4e04b6fcef 100644 --- a/src/packages/project/jupyter/http-server.ts +++ b/src/packages/project/jupyter/http-server.ts @@ -23,7 +23,7 @@ import { BlobStoreSqlite, } from "@cocalc/jupyter/blobs"; import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data"; -import { get_ProjectStatusServer } from "@cocalc/project/project-status/server"; +import { get_ProjectStatusServer } from "@cocalc/sync-server/monitor/status-and-alerts"; import { delay } from "awaiting"; const log = getLogger("jupyter-http-server"); diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 044e8c8f4e..f960880c4f 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -28,6 +28,7 @@ "@cocalc/sync": "workspace:*", "@cocalc/sync-client": "workspace:*", "@cocalc/sync-fs": "workspace:*", + "@cocalc/sync-server": "workspace:*", "@cocalc/terminal": "workspace:*", "@cocalc/util": "workspace:*", "@nteract/messaging": "^7.0.20", @@ -40,7 +41,6 @@ "compression": "^1.7.4", "daemonize-process": "^3.0.0", "debug": "^4.3.2", - "diskusage": "^1.1.3", "expect": "^26.6.2", "express": "^4.21.1", "express-rate-limit": "^7.4.0", diff --git a/src/packages/project/project-status/index.ts b/src/packages/project/project-status/index.ts deleted file mode 100644 index b30058a77f..0000000000 --- a/src/packages/project/project-status/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -export { get_ProjectStatusServer, ProjectStatusServer } from "./server"; diff --git a/src/packages/project/servers/browser/http-server.ts b/src/packages/project/servers/browser/http-server.ts index 20a6db64a1..5a70774ccb 100644 --- a/src/packages/project/servers/browser/http-server.ts +++ b/src/packages/project/servers/browser/http-server.ts @@ -16,7 +16,6 @@ import express from "express"; import { createServer } from "http"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; - import basePath from "@cocalc/backend/base-path"; import initWebsocket from "@cocalc/project/browser-websocket/server"; import initWebsocketFs from "../websocketfs"; diff --git a/src/packages/project/servers/browser/static.ts b/src/packages/project/servers/browser/static.ts index 1fe3c2331d..ef73d64cba 100644 --- a/src/packages/project/servers/browser/static.ts +++ b/src/packages/project/servers/browser/static.ts @@ -1,10 +1,19 @@ -import { Application, static as staticServer } from "express"; +/* +Serve static files from the HOME directory. + +Security: Authentication is not handled at this level. + +NOTE: There is a very similar server in src/compute/compute/lib/http-server/static.ts +*/ + +import { type Application, static as staticServer } from "express"; import index from "serve-index"; import { getLogger } from "@cocalc/project/logger"; +const log = getLogger("serve-static-files-to-browser"); + export default function init(app: Application, base: string) { - const winston = getLogger("serve-static-files-to-browser"); - winston.info(`initialize with base="${base}"`); + log.info(`initialize with base="${base}"`); // Setup the static raw HTTP server. This must happen after anything above, // since it serves all URL's (so it has to be the fallback). app.use(base, (req, res, next) => { @@ -24,7 +33,7 @@ export default function init(app: Application, base: string) { if (HOME == null) { throw Error("HOME env variable must be defined"); } - winston.info(`serving up HOME="${HOME}"`); + log.info(`serving up HOME="${HOME}"`); app.use(base, index(HOME, { hidden: true, icons: true })); app.use(base, staticServer(HOME, { dotfiles: "allow" })); diff --git a/src/packages/project/tsconfig.json b/src/packages/project/tsconfig.json index 4e6d052ed7..c2a3f0e822 100644 --- a/src/packages/project/tsconfig.json +++ b/src/packages/project/tsconfig.json @@ -16,6 +16,7 @@ { "path": "../sync" }, { "path": "../sync-client" }, { "path": "../sync-fs" }, + { "path": "../sync-server" }, { "path": "../terminal" }, { "path": "../util" } ] diff --git a/src/packages/static/package.json b/src/packages/static/package.json index 440ebd1189..ccaae11ec7 100644 --- a/src/packages/static/package.json +++ b/src/packages/static/package.json @@ -18,7 +18,7 @@ "build0": "pnpm run copy-css && cd src && ../../node_modules/.bin/tsc --build", "build": "pnpm run build0 && ./production-build.py", "build-dev": "pnpm run build0 && NODE_ENV=development pnpm rspack build", - "watch": "NODE_ENV=development pnpm rspack build -w", + "watch": "../node_modules/.bin/tsc && NODE_ENV=development pnpm rspack build -w", "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, diff --git a/src/packages/static/src/plugins/app-loader.ts b/src/packages/static/src/plugins/app-loader.ts index f65eb15313..f18f9678ad 100644 --- a/src/packages/static/src/plugins/app-loader.ts +++ b/src/packages/static/src/plugins/app-loader.ts @@ -27,4 +27,15 @@ export default function appLoaderPlugin( chunks: ["load", "embed"], }), ); + + registerPlugin( + "Compute -- generates the compute.html file", + new rspack.HtmlRspackPlugin({ + title, + filename: "compute.html", + template: resolve(__dirname, "../app.html"), + hash: PRODMODE, + chunks: ["load", "compute"], + }), + ); } diff --git a/src/packages/static/src/rspack.config.ts b/src/packages/static/src/rspack.config.ts index c268b68eea..284a31bf2c 100644 --- a/src/packages/static/src/rspack.config.ts +++ b/src/packages/static/src/rspack.config.ts @@ -168,6 +168,12 @@ export default function getConfig({ middleware }: Options = {}): any { ]), dependOn: "load", }, + compute: { + import: insertHotMiddlewareUrl([ + resolve("dist-ts/src/webapp-compute.js"), + ]), + dependOn: "load", + }, }, /* Why chunkhash below, rather than contenthash? This says contenthash is a special thing for css and other text files only (??): diff --git a/src/packages/static/src/webapp-compute.ts b/src/packages/static/src/webapp-compute.ts new file mode 100644 index 0000000000..429053595f --- /dev/null +++ b/src/packages/static/src/webapp-compute.ts @@ -0,0 +1,11 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +import "./webapp-libraries"; +import { init } from "@cocalc/frontend/compute/entry-point"; +import { startedUp } from "./webapp-error"; + +init(); +startedUp(); diff --git a/src/packages/sync-client/lib/api.ts b/src/packages/sync-client/lib/api.ts index d3823e74e3..86721528fd 100644 --- a/src/packages/sync-client/lib/api.ts +++ b/src/packages/sync-client/lib/api.ts @@ -32,7 +32,7 @@ export default class API implements API_Interface { try { this.cachedVersion = await this.call({ cmd: "version" }, 15000); } catch (err) { - if (err.message.includes('command "version" not implemented')) { + if (err.message?.includes('command "version" not implemented')) { this.cachedVersion = 0; } else { throw err; diff --git a/src/packages/sync-fs/lib/index.ts b/src/packages/sync-fs/lib/index.ts index 996b551595..c804fcee4d 100644 --- a/src/packages/sync-fs/lib/index.ts +++ b/src/packages/sync-fs/lib/index.ts @@ -28,7 +28,6 @@ import { executeCode } from "@cocalc/backend/execute-code"; import { delete_files } from "@cocalc/backend/files/delete-files"; import { move_files } from "@cocalc/backend/files/move-files"; import { rename_file } from "@cocalc/backend/files/rename-file"; -import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; const EXPLICIT_HIDDEN_EXCLUDES = [".cache", ".local"]; @@ -318,17 +317,11 @@ class SyncFS { return await getListing(data.path, data.hidden, this.mount); case "exec": - if (data.opts.command == "cc-new-file") { - // so we don't have to depend on having our cc-new-file script - // installed. We just don't support templates on compute server. - for (const path of data.opts.args ?? []) { - const target = join(this.mount, path); - await ensureContainingDirectoryExists(target); - await writeFile(target, ""); - } - return { status: 0, stdout: "", stderr: "" }; - } - return await executeCode({ ...data.opts, home: this.mount }); + return await executeCode({ + ...data.opts, + home: this.mount, + ccNewFile: true, + }); case "delete_files": return await delete_files(data.paths, this.mount); diff --git a/src/packages/sync-server/README.md b/src/packages/sync-server/README.md new file mode 100644 index 0000000000..2997e75041 --- /dev/null +++ b/src/packages/sync-server/README.md @@ -0,0 +1,12 @@ +# Sync Server + +The project home base @cocalc/project serves synctables to browser clients +and compute servers. Compute servers themselves also server synctables +to browsers, so that browsers can connect directly to compute servers +for reduced latency. + +All connections work over WebSockets, and the code here is meant to manage +all of this. It makes sure that there is only one actual synctable with +given defining parameters and so on. + +**TODO** \ No newline at end of file diff --git a/src/packages/sync-server/monitor/README.md b/src/packages/sync-server/monitor/README.md new file mode 100644 index 0000000000..aa4827c52f --- /dev/null +++ b/src/packages/sync-server/monitor/README.md @@ -0,0 +1,9 @@ +The modules here all collect and update tables that provide various +info about the sync-server, which is relevant to the sync client. + +- process +- disk usage +- cpu usage +- X11 ports + +etc \ No newline at end of file diff --git a/src/packages/project/project-info/index.ts b/src/packages/sync-server/monitor/activity/index.ts similarity index 100% rename from src/packages/project/project-info/index.ts rename to src/packages/sync-server/monitor/activity/index.ts diff --git a/src/packages/project/project-info/project-info.ts b/src/packages/sync-server/monitor/activity/project-info.ts similarity index 76% rename from src/packages/project/project-info/project-info.ts rename to src/packages/sync-server/monitor/activity/project-info.ts index cb897d01f5..398f33bb68 100644 --- a/src/packages/project/project-info/project-info.ts +++ b/src/packages/sync-server/monitor/activity/project-info.ts @@ -5,6 +5,14 @@ /* Project information + + +NOTE: +It seems like project_info_ws sets up up an entire websocket channel, but literally the only +thing it does is implement one command to send a kill signal to a process. There's an exec +api that could do the same thing already. Maybe there was a big plan to add a much more sophisticated +API, but it hasn't happened yet? + */ import { ProjectInfoCmds } from "@cocalc/util/types/project-info/types"; @@ -21,8 +29,8 @@ export function get_ProjectInfoServer(): ProjectInfoServer { } export async function project_info_ws( - primus: any, - logger: { debug: Function } + primus, + logger: { debug: Function }, ): Promise { const L = (...msg) => logger.debug("project_info:", ...msg); const name = `project_info`; @@ -32,7 +40,7 @@ export async function project_info_ws( L(`deregistering ${spark.id}`); } - channel.on("connection", function (spark: any): void { + channel.on("connection", (spark): void => { // Now handle the connection L(`channel: new connection from ${spark.address.ip} -- ${spark.id}`); @@ -43,7 +51,7 @@ export async function project_info_ws( spark.on("close", () => close("close")); spark.on("end", () => close("end")); - spark.on("data", function (data: ProjectInfoCmds) { + spark.on("data", (data: ProjectInfoCmds) => { // we assume only ProjectInfoCmds should come in, but better check what this is if (typeof data === "object") { switch (data.cmd) { @@ -58,7 +66,7 @@ export async function project_info_ws( }); }); - channel.on("disconnection", function (spark: any): void { + channel.on("disconnection", (spark): void => { L(`channel: disconnection from ${spark.address.ip} -- ${spark.id}`); deregister(spark); }); diff --git a/src/packages/project/project-info/server.ts b/src/packages/sync-server/monitor/activity/server.ts similarity index 97% rename from src/packages/project/project-info/server.ts rename to src/packages/sync-server/monitor/activity/server.ts index 2d7466be87..de730f2dfa 100644 --- a/src/packages/project/project-info/server.ts +++ b/src/packages/sync-server/monitor/activity/server.ts @@ -13,7 +13,6 @@ import type { DiskUsage as DF_DiskUsage } from "diskusage"; import { check as df } from "diskusage"; import { EventEmitter } from "node:events"; import { readFile } from "node:fs/promises"; - import { ProcessStats } from "@cocalc/backend/process-stats"; import { get_kernel_by_pid } from "@cocalc/jupyter/kernel"; import { pidToPath as terminalPidToPath } from "@cocalc/terminal"; @@ -25,11 +24,10 @@ import { Processes, ProjectInfo, } from "@cocalc/util/types/project-info/types"; -import { get_path_for_pid as x11_pid2path } from "../x11/server"; -//import { get_sage_path } from "../sage_session" -import { getLogger } from "../logger"; +import { get_path_for_pid as x11_pid2path } from "@cocalc/sync-server/x11"; +import { getLogger } from "@cocalc/backend/logger"; -const L = getLogger("project-info:server").debug; +const L = getLogger("sync-server:project-info:server").debug; // function is_in_dev_project() { // return process.env.SMC_LOCAL_HUB_HOME != null; diff --git a/src/packages/project/project-info/utils.ts b/src/packages/sync-server/monitor/activity/utils.ts similarity index 100% rename from src/packages/project/project-info/utils.ts rename to src/packages/sync-server/monitor/activity/utils.ts diff --git a/src/packages/project/project-status/server.ts b/src/packages/sync-server/monitor/status-and-alerts.ts similarity index 93% rename from src/packages/project/project-status/server.ts rename to src/packages/sync-server/monitor/status-and-alerts.ts index 53a7c44703..9daad6d5f6 100644 --- a/src/packages/project/project-status/server.ts +++ b/src/packages/sync-server/monitor/status-and-alerts.ts @@ -8,20 +8,20 @@ Project status server, doing the heavy lifting of telling the client what's going on in the project, especially if there is a problem. Under the hood, it subscribes to the ProjectInfoServer, which updates -various statistics at a high-frequency. Therefore, this here filters +various statistics at a high-frequency. Therefore, this filters that information to a low-frequency low-volume stream of important status updates. -Hence in particular, information like cpu, memory and disk are smoothed out and throttled. +In particular, information like cpu, memory and disk are smoothed out and throttled. */ -import { getLogger } from "@cocalc/project/logger"; +import { getLogger } from "@cocalc/backend/logger"; import { how_long_ago_m, round1 } from "@cocalc/util/misc"; import { version as smcVersion } from "@cocalc/util/smc-version"; import { delay } from "awaiting"; import { EventEmitter } from "events"; import { isEqual } from "lodash"; -import { get_ProjectInfoServer, ProjectInfoServer } from "../project-info"; +import { get_ProjectInfoServer, ProjectInfoServer } from "@cocalc/sync-server/monitor/activity"; import { ProjectInfo } from "@cocalc/util/types/project-info/types"; import { ALERT_DISK_FREE, @@ -42,7 +42,7 @@ import { cgroup_stats } from "@cocalc/comm/project-status/utils"; // return next; //} -const winston = getLogger("ProjectStatusServer"); +const logger = getLogger("project:project-status-server"); function quantize(val, order) { const q = Math.round(Math.pow(10, order)); @@ -58,7 +58,6 @@ interface Elevated { } export class ProjectStatusServer extends EventEmitter { - private readonly dbg: Function; private running = false; private readonly testing: boolean; private readonly project_info: ProjectInfoServer; @@ -83,14 +82,13 @@ export class ProjectStatusServer extends EventEmitter { constructor(testing = false) { super(); this.testing = testing; - this.dbg = (...msg) => winston.debug(...msg); this.project_info = get_ProjectInfoServer(); } private async init(): Promise { this.project_info.start(); this.project_info.on("info", (info) => { - //this.dbg(`got info timestamp=${info.timestamp}`); + //logger.debug(`got info timestamp=${info.timestamp}`); this.info = info; this.update(); this.emitInfo(); @@ -100,14 +98,14 @@ export class ProjectStatusServer extends EventEmitter { // checks if there the current state (after update()) should be emitted private emitInfo(): void { if (this.lastEmit === 0) { - this.dbg("emitInfo[last=0]", this.status); + logger.debug("emitInfo[last=0]", this.status); this.doEmit(); return; } // if alert changed, emit immediately if (!isEqual(this.last?.alerts, this.status?.alerts)) { - this.dbg("emitInfo[alert]", this.status); + logger.debug("emitInfo[alert]", this.status); this.doEmit(); } else { // deep comparison check via lodash and we rate limit @@ -115,7 +113,7 @@ export class ProjectStatusServer extends EventEmitter { this.lastEmit + 1000 * STATUS_UPDATES_INTERVAL_S > Date.now(); const changed = !isEqual(this.status, this.last); if (!recent && changed) { - this.dbg("emitInfo[changed]", this.status); + logger.debug("emitInfo[changed]", this.status); this.doEmit(); } } @@ -202,7 +200,7 @@ export class ProjectStatusServer extends EventEmitter { } } pids.sort(); // to make this stable across iterations - //this.dbg("alert_cpu_processes", pids, ecp); + //logger.debug("alert_cpu_processes", pids, ecp); return pids; } @@ -298,7 +296,7 @@ export class ProjectStatusServer extends EventEmitter { public async start(): Promise { if (this.running) { - this.dbg( + logger.debug( "project-status/server: already running, cannot be started twice", ); } else { @@ -307,7 +305,7 @@ export class ProjectStatusServer extends EventEmitter { } private async _start(): Promise { - this.dbg("start"); + logger.debug("start"); if (this.running) { throw Error("Cannot start ProjectStatusServer twice"); } diff --git a/src/packages/project/usage-info/const.ts b/src/packages/sync-server/monitor/usage/const.ts similarity index 100% rename from src/packages/project/usage-info/const.ts rename to src/packages/sync-server/monitor/usage/const.ts diff --git a/src/packages/project/usage-info/index.ts b/src/packages/sync-server/monitor/usage/index.ts similarity index 100% rename from src/packages/project/usage-info/index.ts rename to src/packages/sync-server/monitor/usage/index.ts diff --git a/src/packages/project/usage-info/server.ts b/src/packages/sync-server/monitor/usage/server.ts similarity index 96% rename from src/packages/project/usage-info/server.ts rename to src/packages/sync-server/monitor/usage/server.ts index 9f0c4ff9bb..f26ada4010 100644 --- a/src/packages/project/usage-info/server.ts +++ b/src/packages/sync-server/monitor/usage/server.ts @@ -13,14 +13,13 @@ from the ProjectInfoServer (which collects data about everything) import { delay } from "awaiting"; import { EventEmitter } from "node:events"; - -import { getLogger } from "../logger"; -import { ProjectInfoServer, get_ProjectInfoServer } from "../project-info"; +import { getLogger } from "@cocalc/backend/logger"; +import { ProjectInfoServer, get_ProjectInfoServer } from "@cocalc/sync-server/monitor/activity"; import { Process, ProjectInfo } from "@cocalc/util/types/project-info/types"; import type { UsageInfo } from "@cocalc/util/types/project-usage-info"; import { throttle } from "lodash"; -const L = getLogger("usage-info:server").debug; +const L = getLogger("sync-server:usage:server").debug; const throttled_dbg = throttle((...args) => L(...args), 10000); diff --git a/src/packages/sync-server/package.json b/src/packages/sync-server/package.json new file mode 100644 index 0000000000..e510f4d6df --- /dev/null +++ b/src/packages/sync-server/package.json @@ -0,0 +1,55 @@ +{ + "name": "@cocalc/sync-server", + "version": "0.1.0", + "description": "CoCalc realtime synchronization framework -- server component", + "exports": { + "./server/*": "./dist/server/*.js", + "./x11": "./dist/x11.js", + "./monitor/activity": "./dist/monitor/activity/index.js", + "./monitor/usage": "./dist/monitor/usage/index.js", + "./monitor/status-and-alerts": "./dist/monitor/status-and-alerts.js" + }, + "scripts": { + "clean": "rm -rf dist", + "preinstall": "npx only-allow pnpm", + "build": "../node_modules/.bin/tsc --build", + "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", + "prepublishOnly": "pnpm test" + }, + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], + "author": "SageMath, Inc.", + "keywords": [ + "cocalc", + "realtime synchronization" + ], + "license": "SEE LICENSE.md", + "dependencies": { + "@cocalc/backend": "workspace:*", + "@cocalc/comm": "workspace:*", + "@cocalc/jupyter": "workspace:*", + "@cocalc/sync": "workspace:*", + "@cocalc/sync-server": "workspace:*", + "@cocalc/terminal": "workspace:*", + "@cocalc/util": "workspace:*", + "awaiting": "^3.0.0", + "diskusage": "^1.1.3", + "events": "3.3.0", + "json-stable-stringify": "^1.0.1", + "lodash": "^4.17.21" + }, + "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/sync-server", + "repository": { + "type": "git", + "url": "https://github.com/sagemathinc/cocalc" + }, + "devDependencies": { + "@types/json-stable-stringify": "^1.0.32", + "@types/lodash": "^4.14.202", + "@types/node": "^18.16.14" + } +} diff --git a/src/packages/project/sync/compute-server-open-file-tracking.ts b/src/packages/sync-server/server/compute-server-open-file-tracking.ts similarity index 96% rename from src/packages/project/sync/compute-server-open-file-tracking.ts rename to src/packages/sync-server/server/compute-server-open-file-tracking.ts index 4ebb773a59..e9ce20cb6d 100644 --- a/src/packages/project/sync/compute-server-open-file-tracking.ts +++ b/src/packages/sync-server/server/compute-server-open-file-tracking.ts @@ -1,10 +1,10 @@ /* Manage the state of open files in the compute servers syncdb sync'd file. -TODO: terminals aren't handled at all here, since they don't have a syncdoc. +NOTE: terminals aren't handled at all here, since they don't have a syncdoc. */ -import type { SyncDocs } from "./sync-doc"; +import type { SyncDocs } from "@cocalc/sync-server/server/syncdocs-manager"; import type { SyncDB } from "@cocalc/sync/editor/db/sync"; import { once } from "@cocalc/util/async-utils"; import { meta_file, auxFileToOriginal } from "@cocalc/util/misc"; diff --git a/src/packages/project/sync/listings.ts b/src/packages/sync-server/server/listings.ts similarity index 92% rename from src/packages/project/sync/listings.ts rename to src/packages/sync-server/server/listings.ts index 796f0a01a6..a477c76fa4 100644 --- a/src/packages/project/sync/listings.ts +++ b/src/packages/sync-server/server/listings.ts @@ -9,7 +9,7 @@ import { } from "@cocalc/sync/listings"; import getListing from "@cocalc/backend/get-listing"; import { Watcher } from "@cocalc/backend/path-watcher"; -import { close_all_syncdocs_in_tree } from "./sync-doc"; +import { closeAllSyncDocsInTree } from "@cocalc/sync-server/server/syncdocs-manager"; import { getLogger } from "@cocalc/backend/logger"; import { existsSync } from "fs"; @@ -24,7 +24,7 @@ export function registerListingsTable(table, query): void { const onDeletePath = async (path) => { // Also we need to close *all* syncdocs that are going to be deleted, // and wait until closing is done before we return. - await close_all_syncdocs_in_tree(path); + await closeAllSyncDocsInTree(path); }; const createWatcher = (path: string, debounce: number) => diff --git a/src/packages/project/sync/open-synctables.ts b/src/packages/sync-server/server/open-synctables.ts similarity index 100% rename from src/packages/project/sync/open-synctables.ts rename to src/packages/sync-server/server/open-synctables.ts diff --git a/src/packages/project/sync/project-info.ts b/src/packages/sync-server/server/project-info.ts similarity index 91% rename from src/packages/project/sync/project-info.ts rename to src/packages/sync-server/server/project-info.ts index c31d0eb2ee..5ed1ec71c3 100644 --- a/src/packages/project/sync/project-info.ts +++ b/src/packages/sync-server/server/project-info.ts @@ -6,9 +6,11 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { close } from "@cocalc/util/misc"; import { SyncTable } from "@cocalc/sync/table"; -import { get_ProjectInfoServer } from "../project-info"; +import { + get_ProjectInfoServer, + type ProjectInfoServer, +} from "@cocalc/sync-server/monitor/activity"; import { ProjectInfo } from "@cocalc/util/types/project-info/types"; -import { ProjectInfoServer } from "../project-info"; class ProjectInfoTable { private table: SyncTable; @@ -21,7 +23,7 @@ class ProjectInfoTable { constructor( table: SyncTable, logger: { debug: Function }, - project_id: string + project_id: string, ) { this.project_id = project_id; this.logger = logger; @@ -48,7 +50,7 @@ class ProjectInfoTable { this.log( `ProjectInfoTable state = '${ this.state - }' and table is '${this.table?.get_state()}'` + }' and table is '${this.table?.get_state()}'`, ); } } @@ -72,12 +74,12 @@ let project_info_table: ProjectInfoTable | undefined = undefined; export function register_project_info_table( table: SyncTable, logger: any, - project_id: string + project_id: string, ): void { logger.debug("register_project_info_table"); if (project_info_table != null) { logger.debug( - "register_project_info_table: cleaning up an already existing one" + "register_project_info_table: cleaning up an already existing one", ); project_info_table.close(); } diff --git a/src/packages/project/sync/project-status.ts b/src/packages/sync-server/server/project-status.ts similarity index 76% rename from src/packages/project/sync/project-status.ts rename to src/packages/sync-server/server/project-status.ts index 14fb1a73b8..7366638669 100644 --- a/src/packages/project/sync/project-status.ts +++ b/src/packages/sync-server/server/project-status.ts @@ -9,26 +9,23 @@ import { SyncTable } from "@cocalc/sync/table"; import { get_ProjectStatusServer, ProjectStatusServer, -} from "../project-status"; -import { ProjectStatus } from "@cocalc/comm/project-status/types"; +} from "@cocalc/sync-server/monitor/status-and-alerts"; +import type { ProjectStatus } from "@cocalc/comm/project-status/types"; +import { getLogger } from "@cocalc/backend/logger"; + +const logger = getLogger("project:project-status"); class ProjectStatusTable { private table: SyncTable; - private logger: { debug: Function }; private project_id: string; private state: "ready" | "closed" = "ready"; private readonly publish: (status: ProjectStatus) => Promise; private readonly status_server: ProjectStatusServer; - constructor( - table: SyncTable, - logger: { debug: Function }, - project_id: string, - ) { + constructor(table: SyncTable, project_id: string) { this.status_handler = this.status_handler.bind(this); this.project_id = project_id; - this.logger = logger; - this.log("register"); + logger.debug("register"); this.publish = reuseInFlight(this.publish_impl.bind(this)); this.table = table; this.table.on("closed", () => this.close()); @@ -39,7 +36,7 @@ class ProjectStatusTable { } private status_handler(status): void { - this.log?.("status_server event 'status'", status.timestamp); + logger.debug("status_server event 'status'", status.timestamp); this.publish?.(status); } @@ -50,10 +47,10 @@ class ProjectStatusTable { try { await this.table.save(); } catch (err) { - this.log(`error saving ${err}`); + logger.debug(`error saving ${err}`); } - } else if (this.log != null) { - this.log( + } else { + logger.debug( `ProjectStatusTable '${ this.state }' and table is ${this.table?.get_state()}`, @@ -62,24 +59,18 @@ class ProjectStatusTable { } public close(): void { - this.log("close"); + logger.debug("close"); this.status_server?.off("status", this.status_handler); this.table?.close_no_async(); close(this); this.state = "closed"; } - - private log(...args): void { - if (this.logger == null) return; - this.logger.debug("project_status", ...args); - } } let project_status_table: ProjectStatusTable | undefined = undefined; export function register_project_status_table( table: SyncTable, - logger: any, project_id: string, ): void { logger.debug("register_project_status_table"); @@ -89,7 +80,7 @@ export function register_project_status_table( ); project_status_table.close(); } - project_status_table = new ProjectStatusTable(table, logger, project_id); + project_status_table = new ProjectStatusTable(table, project_id); } export function get_project_status_table(): ProjectStatusTable | undefined { diff --git a/src/packages/project/sync/server.ts b/src/packages/sync-server/server/server.ts similarity index 97% rename from src/packages/project/sync/server.ts rename to src/packages/sync-server/server/server.ts index 188118a899..93269e1560 100644 --- a/src/packages/project/sync/server.ts +++ b/src/packages/sync-server/server/server.ts @@ -10,7 +10,7 @@ between project and browser client. TODO: - [ ] If initial query fails, need to raise exception. Right now it gets -silently swallowed in persistent mode... + silently swallowed in persistent mode... */ // How long to wait from when we hit 0 clients until closing this channel. @@ -51,8 +51,10 @@ import { // @ts-ignore -- typescript nonsense. const _ = set_debug; -import { init_syncdoc, getSyncDocFromSyncTable } from "./sync-doc"; -import { key, register_synctable } from "./open-synctables"; +import { + key, + register_synctable, +} from "@cocalc/sync-server/server/open-synctables"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { once } from "@cocalc/util/async-utils"; import { delay } from "awaiting"; @@ -62,8 +64,12 @@ import { register_project_info_table } from "./project-info"; import { register_project_status_table } from "./project-status"; import { register_usage_info_table } from "./usage-info"; import type { MergeType } from "@cocalc/sync/table/synctable"; -import Client from "@cocalc/sync-client"; +import type { Client } from "@cocalc/sync/client/types"; import { getJupyterRedux } from "@cocalc/jupyter/kernel"; +import { + initSyncDoc, + getSyncDocFromSyncTable, +} from "@cocalc/sync-server/server/syncdocs-manager"; type Query = { [key: string]: any }; @@ -237,7 +243,7 @@ class SyncTableChannel { } if (this.synctable.table === "syncstrings") { this.log("init_synctable -- syncstrings: also initialize syncdoc..."); - init_syncdoc(this.client, this.synctable); + initSyncDoc(this.client, this.synctable); } this.synctable.on( @@ -460,6 +466,10 @@ class SyncTableChannel { if (this.closed || this.closing) { return; // closing or already closed } + if (this.synctable == null) { + this.log("save_if_possible: not initialized yet"); + return; + } this.log("save_if_possible: saves changes to database"); await this.synctable.save(); if (this.synctable.table === "syncstrings") { @@ -517,8 +527,8 @@ class SyncTableChannel { this.log("close: closing"); this.closing = true; delete synctable_channels[this.name]; - this.channel.destroy(); - this.synctable.close_no_async(); + this.channel?.destroy(); + this.synctable?.close_no_async(); this.log("close: closed"); close(this); // don't call this.log after this! this.closed = true; @@ -592,7 +602,6 @@ async function synctable_channel0( } else if (query?.project_status != null) { register_project_status_table( synctable_channels[name].get_synctable(), - logger, client.client_id(), ); } else if (query?.usage_info != null) { diff --git a/src/packages/project/sync/sync-doc.ts b/src/packages/sync-server/server/syncdocs-manager.ts similarity index 81% rename from src/packages/project/sync/sync-doc.ts rename to src/packages/sync-server/server/syncdocs-manager.ts index 660fd44407..eac66e49d6 100644 --- a/src/packages/project/sync/sync-doc.ts +++ b/src/packages/sync-server/server/syncdocs-manager.ts @@ -4,7 +4,13 @@ */ /* -Backend project support for using syncdocs. +Backend support for using syncdocs. If a client opens a synctable with +table='syncstrings', then a corresponding SyncDoc gets created here, so +that its possible to edit the actual file on disk that corresponds to +that entry in the syncstrings table. This is done automatically, rather +than requiring the frontend client to somehow configure this. + +--- This is mainly responsible for: @@ -16,12 +22,12 @@ This is mainly responsible for: import { SyncTable } from "@cocalc/sync/table"; import { SyncDB } from "@cocalc/sync/editor/db/sync"; import { SyncString } from "@cocalc/sync/editor/string/sync"; -import type Client from "@cocalc/sync-client"; +import type { Client } from "@cocalc/sync/client/types"; import { once } from "@cocalc/util/async-utils"; import { filename_extension, original_path } from "@cocalc/util/misc"; -import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import { EventEmitter } from "events"; import { COMPUTER_SERVER_DB_NAME } from "@cocalc/util/compute/manager"; +import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import computeServerOpenFileTracking from "./compute-server-open-file-tracking"; import { getLogger } from "@cocalc/backend/logger"; @@ -144,7 +150,7 @@ const syncDocs = new SyncDocs(); // instead of syncdoc_metadata (say) because it was created when we only // used strings for sync. -export function init_syncdoc(client: Client, synctable: SyncTable): void { +export function initSyncDoc(client: Client, synctable: SyncTable): void { if (synctable.get_table() !== "syncstrings") { throw Error("table must be 'syncstrings'"); } @@ -153,34 +159,38 @@ export function init_syncdoc(client: Client, synctable: SyncTable): void { } // It's the right type of table and not closed. Now do // the real setup work (without blocking). - init_syncdoc_async(client, synctable); + initSyncDocAsync(client, synctable); } // If there is an already existing syncdoc for this path, // return it; otherwise, return undefined. This is useful // for getting a reference to a syncdoc, e.g., for prettier. -export function get_syncdoc(path: string): SyncDoc | undefined { +export function getSyncDoc(path: string): SyncDoc | undefined { return syncDocs.get(path); } export function getSyncDocFromSyncTable(synctable: SyncTable) { - const { opts } = get_type_and_opts(synctable); - return get_syncdoc(opts.path); + const { opts } = getTypeAndOpts(synctable); + return getSyncDoc(opts.path); } -async function init_syncdoc_async( +async function initSyncDocAsync( client: Client, synctable: SyncTable, ): Promise { function log(...args): void { - logger.debug("init_syncdoc_async: ", ...args); + logger.debug("initSyncDocAsync: ", ...args); } log("waiting until synctable is ready"); - await wait_until_synctable_ready(synctable); + await waitUntilSyncTableReady(synctable); log("synctable ready. Now getting type and opts"); - const { type, opts } = get_type_and_opts(synctable); - const project_id = (opts.project_id = client.client_id()); + const { type, opts } = getTypeAndOpts(synctable); + const project_id = client.client_id(); + if (project_id == null) { + throw Error("client_id must be defined"); + } + opts.project_id = project_id; // log("type = ", type); // log("opts = ", JSON.stringify(opts)); opts.client = client; @@ -219,36 +229,36 @@ async function init_syncdoc_async( } } -async function wait_until_synctable_ready(synctable: SyncTable): Promise { +async function waitUntilSyncTableReady(synctable: SyncTable): Promise { if (synctable.get_state() == "disconnected") { - logger.debug("wait_until_synctable_ready: wait for synctable be connected"); + logger.debug("waitUntilSyncTableReady: wait for synctable be connected"); await once(synctable, "connected"); } const t = synctable.get_one(); if (t != null) { - logger.debug("wait_until_synctable_ready: currently", t.toJS()); + logger.debug("waitUntilSyncTableReady: currently", t.toJS()); } logger.debug( - "wait_until_synctable_ready: wait for document info to get loaded into synctable...", + "waitUntilSyncTableReady: wait for document info to get loaded into synctable...", ); // Next wait until there's a document in the synctable, since that will // have the path, patch type, etc. in it. That is set by the frontend. function is_ready(): boolean { const t = synctable.get_one(); if (t == null) { - logger.debug("wait_until_synctable_ready: is_ready: table is null still"); + logger.debug("waitUntilSyncTableReady: is_ready: table is null still"); return false; } else { - logger.debug("wait_until_synctable_ready: is_ready", JSON.stringify(t)); + logger.debug("waitUntilSyncTableReady: is_ready", JSON.stringify(t)); return t.has("path"); } } await synctable.wait(is_ready, 0); - logger.debug("wait_until_synctable_ready: document info is now in synctable"); + logger.debug("waitUntilSyncTableReady: document info is now in synctable"); } -function get_type_and_opts(synctable: SyncTable): { type: string; opts: any } { +function getTypeAndOpts(synctable: SyncTable): { type: string; opts: any } { const s = synctable.get_one(); if (s == null) { throw Error("synctable must not be empty"); @@ -281,18 +291,18 @@ function get_type_and_opts(synctable: SyncTable): { type: string; opts: any } { return { type, opts }; } -export async function syncdoc_call(path: string, mesg: any): Promise { - logger.debug("syncdoc_call", path, mesg); +export async function callSyncDoc(path: string, mesg: any): Promise { + logger.debug("callSyncDoc", path, mesg); const doc = syncDocs.get(path); if (doc == null) { - logger.debug("syncdoc_call -- not open: ", path); + logger.debug("callSyncDoc -- not open: ", path); return "not open"; } switch (mesg.cmd) { case "close": - logger.debug("syncdoc_call -- now closing: ", path); + logger.debug("callSyncDoc -- now closing: ", path); await syncDocs.close(path); - logger.debug("syncdoc_call -- closed: ", path); + logger.debug("callSyncDoc -- closed: ", path); return "successfully closed"; default: throw Error(`unknown command ${mesg.cmd}`); @@ -301,9 +311,7 @@ export async function syncdoc_call(path: string, mesg: any): Promise { // This is used when deleting a file/directory // filename may be a directory or actual filename -export async function close_all_syncdocs_in_tree( - filename: string, -): Promise { - logger.debug("close_all_syncdocs_in_tree", filename); +export async function closeAllSyncDocsInTree(filename: string): Promise { + logger.debug("closeAllSyncDocsInTree", filename); return await syncDocs.closeAll(filename); } diff --git a/src/packages/project/sync/usage-info.ts b/src/packages/sync-server/server/usage-info.ts similarity index 96% rename from src/packages/project/sync/usage-info.ts rename to src/packages/sync-server/server/usage-info.ts index ad3cac0412..223aaea08e 100644 --- a/src/packages/project/sync/usage-info.ts +++ b/src/packages/sync-server/server/usage-info.ts @@ -9,8 +9,11 @@ import { SyncTable, SyncTableState } from "@cocalc/sync/table"; import { once } from "@cocalc/util/async-utils"; import { close, merge } from "@cocalc/util/misc"; -import { UsageInfoServer } from "../usage-info"; -import type { ImmutableUsageInfo, UsageInfo } from "@cocalc/util/types/project-usage-info"; +import { UsageInfoServer } from "@cocalc/sync-server/monitor/usage"; +import type { + ImmutableUsageInfo, + UsageInfo, +} from "@cocalc/util/types/project-usage-info"; import { getLogger } from "@cocalc/backend/logger"; const L = getLogger("sync:usage-info"); @@ -85,7 +88,7 @@ class UsageInfoTable { async set(obj: { path: string; usage?: UsageInfo }): Promise { this.get_table().set( merge({ project_id: this.project_id }, obj), - "shallow" + "shallow", ); await this.get_table().save(); } @@ -167,7 +170,7 @@ class UsageInfoTable { let usage_info_table: UsageInfoTable | undefined = undefined; export function register_usage_info_table( table: SyncTable, - project_id: string + project_id: string, ): void { L.debug("register_usage_info_table"); if (usage_info_table != null) { diff --git a/src/packages/sync-server/tsconfig.json b/src/packages/sync-server/tsconfig.json new file mode 100644 index 0000000000..48a120589e --- /dev/null +++ b/src/packages/sync-server/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "rootDir": "./", + "outDir": "dist" + }, + "exclude": ["node_modules", "dist", "test"], + "references": [ + { + "path": "../backend", + "path": "../comm", + "path": "../jupyter", + "path": "../sync", + "path": "../terminal", + "path": "../util" + } + ] +} diff --git a/src/packages/project/x11/server.ts b/src/packages/sync-server/x11.ts similarity index 95% rename from src/packages/project/x11/server.ts rename to src/packages/sync-server/x11.ts index a7ac85f4a8..457cc7329d 100644 --- a/src/packages/project/x11/server.ts +++ b/src/packages/sync-server/x11.ts @@ -4,17 +4,17 @@ */ /* -X11 server channel. +X11 server channel. This is implemented under the hood using the standard +xpra server. Frontends use a custom xpra client we wrote. TODO: - [ ] other user activity - - [ ] when stopping project, kill xpra's + - [ ] when stopping sync server, kill xpra sessions */ import { spawn, SpawnOptions } from "node:child_process"; import { callback } from "awaiting"; import { clone } from "lodash"; - import abspath from "@cocalc/backend/misc/abspath"; import { path_split } from "@cocalc/util/misc"; diff --git a/src/packages/sync/client/types.ts b/src/packages/sync/client/types.ts index af6fa87b9f..12e15a89f7 100644 --- a/src/packages/sync/client/types.ts +++ b/src/packages/sync/client/types.ts @@ -17,6 +17,7 @@ export interface Client extends EventEmitter { touch_project: (project_id: string) => void; set_connected?: Function; is_deleted: (path: string, project_id: string) => true | false | undefined; + client_id: () => string | undefined; } export interface ClientFs extends Client { @@ -35,7 +36,6 @@ export interface ClientFs extends Client { debounce?: number; }) => any; server_time: () => Date; - client_id: () => string | undefined; } export interface Channel { diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 9d4605aa99..c853c4ea92 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1073,7 +1073,7 @@ export class SyncDoc extends EventEmitter { // WARNING: that 'closed' is emitted at the beginning of the // close function (before anything async) for the project is - // assumed in src/packages/project/sync/sync-doc.ts, because + // assumed in @cocalc/sync-server/server/sync-doc.ts, because // that ensures that the moment close is called we lock trying // try create the syncdoc again until closing is finished. // (This set_state call emits "closed"): @@ -1718,7 +1718,7 @@ export class SyncDoc extends EventEmitter { // this only potentially happens for tables in the project, // e.g., jupyter and compute servers: - // see packages/project/sync/server.ts + // see @cocalc/sync-server/server/server.ts this.patches_table.on("message", (...args) => { dbg("received message", args); this.emit("message", ...args); diff --git a/src/packages/sync/table/README.md b/src/packages/sync/table/README.md index bc77975f52..eede522dae 100644 --- a/src/packages/sync/table/README.md +++ b/src/packages/sync/table/README.md @@ -1,4 +1,186 @@ -# SYNCHRONIZED TABLE -- +# Synchronized Tables aka SyncTables + +A synchronized table is basically a small in\-memory key:value store that is replicated across many web browsers and a cocalc project, and usually persisted in the database. The keys are strings and the values are JSON-able objects. Some of our synctables are defined by a PostgreSQL table and rules that grant a user permission to read or write to a defined subset of the table. There are also ephemeral synctables, which are not persisted to the database, e.g., the location of user cursors in a document. + +A web browser can directly manage a synctable via queries \(through a hub\) to the database, e.g., the account settings and list of user projects are dealt with this way. + +For file editing, the synctable is managed by the project, and all web browsers connect via a websocket to the project. The project ensures that the syntable state is eventually identical for all browsers to its own state, and is also responsible for persisting data longterm to the database. It does NOT setup a changefeed for file editing with the database, since all file editing goes through the project, hence it is the only one that changes the database table of file editing state. + +```mermaid +graph TD; + A[Web Browsers...] -- WebSocket --> B[The Project Home Base]; + D -- TCP Socket --> B; + A -- Queries --> D[Hubs...]; + D -- SQL & LISTEN/NOTIFY --> C[(The PostgreSQL Database)]; + G[Compute Servers...] -- WebSocket --> B; + A -- WebSocket --> G; + + subgraph SyncTable Architecture + E((Ephemeral SyncTables)) + F((Persistent SyncTables)) + G + A + B + end + + style E fill:#f9f,stroke:#333,stroke-width:2px + style F fill:#ff9,stroke:#333,stroke-width:2px + style B fill:#afa,stroke:#333,stroke-width:2px + style G fill:#f88,stroke:#333,stroke-width:2px + style C fill:#ccf,stroke:#333,stroke-width:2px + style D fill:#ffb,stroke:#333,stroke-width:2px +``` + +The direction of the arrow indicates which side _initiates_ the connection; in most cases data flows in both directions, +possibly with a realtime push model (due to persistent connections). + +## SyncTables: Motivation and Broader Context + +Realtime synchronized text editing and editing complicated data structures like those in Jupyter notebooks is **not** done directly using synctables! Instead, synctables are a low level primitive on which we build those more complicated synchronized documents. + +### Global State + +We do use synctables to manage various aspects of user state that directly reflects the contents of the database, including: + +- all (or recent!) projects a user is a collaborator on, +- the log of activity in an open project, +- account settings for a user, +- files that were recently active in some project you collaborate on + +These are all managed via a direct connection between the web browser and a backend hub, which connects to the database and uses PostgreSQL LISTEN/NOTIFY to get realtime updates whenever the database changes. It is not necessary to that if one web browser makes a change, e.g., to account settings, that everybody else sees that as quickly as possible; also, the volume of such traffic is relatively low. + +### File Editing State + +In contrast, when actively editing a file, the higher level SyncDoc data structures create a synctables that is being populated with new patches every second or two, and these must be made visible to +all other clients as quickly as possible. Moreover, when evaluating code in a Jupyter notebook, a synctable changes to reflect that you want to evaluate code, and is then updated again with a new patch with the result of that code evaluation -- this must all happen quickly, and can't involve the database as part of the data flow. Instead, the browsers and home base communicate directly via a websocket, and the patches are persisted to the database (in big chunks) in the background. + +## Protocol Used to Synchronize SyncTables + +Synctables exist in three places: + +- Web Browsers +- The Project Home Base +- Compute Servers + +They are _eventually consistent_, in the sense that if everybody stops making changes and all network connections are working, then eventually all copies of the same table \(i.e., defined by the same query\) will be the same. + +There is no fundamental reason regarding the algorithm for not syncing between browsers directly. What is the algorithm though? + +### Database Backed Synctables + +For synctables that directly reflect the PostgreSQL database, the client writes changes to the hub until they succeed, and the hub listens using LISTEN/NOTIFY for changes to the relevant table in the database, and if they are relevant to the user, sends an update message. Doing this efficiently in general is probably impossible, but we have a pretty explicit list of queries and tables, and special data structures to make this efficient. + +The browser client writes changes to the hub until success. + +```mermaid +graph TD; + A[Web Browsers...] <--Websocket--> B[Hub]; + B <--SELECT/INSERT/UPDATE/LISTEN/NOTIFY--> C[(Database)] + D[Project Home Base] <--Websocket--> B; + + subgraph Clients of Database Backed SyncTables + A + D + E[Compute Servers...] <--WebSocket-->D + end +``` + +### Project Backed Synctable Protocol + +As mentioned above, there is one master \(the home base\) and a bunch of clients. We do not currently do any browser\-to\-browser synchronization of synctables, so when collaboratively editing documents, all data goes from the browser back to the project home base, and is then broadcast back to the other browsers. + +```mermaid +graph TD; + A[Web Browsers...] <---> B[Project Home Base]; + C[Compute Servers...] <---> B; + B -.Non-Ephemeral Data.-> H[Hub] --> D[(Database)]; + B -.Ephemeral Data.-> N[ /dev/null] + + subgraph Clients of Project-Backed SyncTables + A + C + end + +``` + +Consider a synctable $T_p$ in the project home base $p$ and the corresponding +synctable $T_c$ in a web browser or compute server client $c$. These are key\-value stores, and either synctable may be modified at any time. There is a persistent websocket connection between the project and browser, but we assume it can go down at any time, +but eventually comes back. Assume that all relevant data is stored in memory in +both the project and web browser, since longterm persistent is not relevant to this protocol. + +Our goal is to periodically do a sync operation involving $T_p$ and $T_c$, which doesn't require blocking writes on either side, but after which the state of both will be the same if there are no changes to either side. Moreover, when there are conflicts, we want "last change wins", where time is defined as follows. We assume that $p$ and $c$ have synchronized their clocks within a second or so, which is fine, given our requirements \(ping times should be far less than 1 second\), and easy to do via a ping. If between sync operations the same key has different values set, then we resolve the conflict by selecting the one that was changed most recently, with any tie being broken by the project winning. + +**NOTE:** This is not a distributed computing paper, it's a simple real life protocol. Time exists. + +First we assume that the network connection is robust and explain how the protocol works. Then we explain what we do to fix things when the network connection goes down and the client reconnects. + +#### Robust Network + +Assume that the network is working perfectly. Each client keeps track of the following: + +- **value:** the key:value mapping \(we use immutable.js to implement this\) +- **changes:** a map from keys that this client has changed but not yet saved to the timestamp when they made the change. Initially this is empty. +- **versions:** a map from all keys to when they were last changed \(or time when table was initialized\) + +When a participant _makes a change_ to their copy of the synctable, they do the following: + +- update **value**\[key\] \(which could include deleting the value\) + +- set **changes**\[key\] = time of change. \(In the code we add 1ms enough times to the realtime so that changes\[key\] are locally distinct, but I don't see why that is needed.\) + +- The project assigns **versions**\[key\] and immediately broadcasts to all connected clients the new **value**\[key\] and its version. The browser sets **versions**\[key\] to 0. + +When a browser saves their changes to the project, they send the following to the project for every key where **changes**\[key\] is set: + +- **value**\[key\] \-\- the value. This also determines the key since it the primary key. +- **time = changes**\[key\] \-\- when browser made the change to value\[key\] + +The project sends a message with the above data, and assuming the send succeeds, it clears the **changes** variable, assuming that the data was sent. + +**NOTE:** Our implementation does verify that the send happens \(write to the websocket succeeds\), but does not wait for an explicit acknowledgement back confirming that the data was received and processed by the project. I think the idea is that every reason for the project to not receive the data would **also** result in the websocket connection breaking and everything resetting anyways. + +The project then receives a message with the above data and does the following \(see `apply_changes_from_browser_client`\): + +- if the **changes**\[key\] is $\geq$ **versions**\[key\], make the change, recording the new value, versions and changes values. + +We do the above for all keys, then broadcast a message to all connected clients with the new values and times. The clients then receive those broadcast message and for each key, do the following: + +- if the **changes**\[key\] is $\geq$ **versions**\[key\], make the change, recording the new value and version. + +**NOTE:** We have not implemented deleting from a synctable yet, and there's code that awkwardly gets around this, e.g., by using a sentinel value. For collab editing we don't need delete, since all we are saving is patches, which we never delete \(unless deleting everything\). I don't see any reason why implementing delete would be hard though. + +**NOTE:** Maybe the comparison should be $>$ instead of $\geq$ because in the case of a tie, the project is going to send out the version again, and then the other client with the same timestamp will receive and change their result. It works because the other client does change their result, but it's extra work and a bit nerve racking, and it would just be more efficient to break the tie in the other direction. + +#### Network Connection Breaks + +Next we consider the various situation where the network connection breaks, the project restarts entirely, etc.. During that time, we assume that every table may be getting changed \(e.g., users are still typing away\), so we can't just reset everything. + +After something breaks \(network, project, etc.\), a client connects to the project and the following happens: + +- the project sends the entire table, which includes key/value pairs, and also their versions. For an ephemeral table, e.g., cursors, ipywidgets, etc., this is empty. For a table backed by the database, e.g., directory listings, this would be the latest version of the relevant data from the database and the versions are all reset to the time the table was initialized again in the project. +- the client then goes through this table and for each key where the version is bigger $\geq$ to what they have, the value and version is updated \(as above\). +- for each key that the client knows about and for which they did not change it when offline \(so it's not already marked to send out\), we set changes\[key\] to now, so that key will get sent to the server. E.g., if we have an ephemeral table and the project side restarts, this makes it so everything in all clients gets sent to the project on the next save, and who wins is somewhat random. + - **NOTE**: For an ephemeral table, if one client changes a key when the project is down and nobody else does, then that one client would have the oldest timestamp for that change, and all the other clients, with an older version of that data, would overwrite the newer version. _That's not optimal and we should probably change this._ Ephemeral tables are typically for things like cursors \(or ipywidgets\) when editing a document, so the impact of this might be a glitch for 1 second while reconnecting, and there's probably other much weirder stuff going on at the same time. + +**NOTE:** If the table is entirely reset to the database and the browser has made changes while waiting to connect, those would be lost, because the versions that the project assigns in this case are when the table is initialized again in the project. There's probably applications where this is bad. However, in cocalc it isn't a problem: for collaborative editing we never change entries in the table, instead just make new ones \(which can't clash\); for listings, those are only created by the project. + +### Compute Servers + +Our new goal is that compute servers can also act like the project. In fact a large number of web browsers connect to a compute server, it manages sync with all of them, then periodically sends a big single update to the project. This would allow for best security, scalability, etc., where a single project could have hundreds of notebooks run at once \(say\), across several compute servers, with realtime sync, and many clients. + +```mermaid +graph TD; + A[Web Browsers...] <---> B[Project Home Base]; + C[Compute Servers...] <---> B; + B -.Non-Ephemeral Data.-> H[Hub] --> D[(Database)]; + B -.Ephemeral Data.-> N[ /dev/null] + A <---> C + + subgraph Clients of Project-Backed SyncTables + A + C + end +``` ## Defined by an object query @@ -8,21 +190,26 @@ ## Methods -- constructor(query): query = the name of a table (or a more complicated object) +- **constructor\(query\):** query = the name of a table \(or a more complicated object\) -- set(map): Set the given keys of map to their values; one key must be - the primary key for the table. NOTE: Computed primary keys will - get automatically filled in; these are keys in schema.coffee, - where the set query looks like this say: - (obj, db) -> db.sha1(obj.project_id, obj.path) -- get(): Current value of the query, as an immutable.js Map from +- **set\(map\):** Set the given keys of map to their values; one key must be + the primary key for the table. NOTE: Computed primary keys will get automatically filled in; these are keys in `packages/util/db-schema` + where the set query looks like this: + `(obj, db) -> db.sha1(obj.project_id, obj.path)` + +- **get\(\):** Current value of the query, as an immutable.js Map from the primary key to the records, which are also immutable.js Maps. -- get(key): The record with given key, as an immutable Map. -- get(keys): Immutable Map from given keys to the corresponding records. -- get_one(): Returns one record as an immutable Map (useful if there - is only one record) -- close(): Frees up resources, stops syncing, don't use object further +- **get\(key\):** The record with given key, as an immutable Map. + +- **get\(keys\):** Immutable Map from given keys to the corresponding records. + +- **get\_one\(\):** Returns one record as an immutable Map \(useful if there + is only one record\) + +- **close\(\):** Frees up resources, stops syncing, don't use object further + +- **save\(\):** Sync all changes to the database or project home base ## Events @@ -43,18 +230,26 @@ A SyncTable is a finite state machine as follows: - -------------------<------------------ - \|/ | - [connecting] --> [connected] --> [disconnected] --> [reconnecting] +```mermaid +stateDiagram-v2 + [*] --> connecting + connecting --> connected + connected --> disconnected + disconnected --> reconnecting + reconnecting --> connecting + + connecting --> closed + connected --> closed + disconnected --> closed + reconnecting --> closed +``` -Also, there is a final state called 'closed', that the SyncTable moves to when +There is a final state called 'closed', that the SyncTable moves to when it will not be used further; this frees up all connections and used memory. The table can't be used after it is closed. The only way to get to the closed state is to explicitly call close() on the table; otherwise, the table will keep attempting to connect and work, until it works. - (anything) --> [closed] - - connecting -- connecting to the backend, and have never connected before. - connected -- successfully connected to the backend, initialized, and receiving updates. @@ -106,3 +301,4 @@ main thing this function deals with below. 3. We use a stable version, since otherwise things will randomly break if the key is an object. + diff --git a/src/packages/sync/table/synctable-no-changefeed.ts b/src/packages/sync/table/synctable-no-changefeed.ts index 93b57c1194..209375db26 100644 --- a/src/packages/sync/table/synctable-no-changefeed.ts +++ b/src/packages/sync/table/synctable-no-changefeed.ts @@ -125,4 +125,6 @@ class ClientNoChangefeed extends EventEmitter { // not implemented yet in general return undefined; }; + + client_id = () => this.client.client_id(); } diff --git a/src/packages/sync/table/synctable-no-database.ts b/src/packages/sync/table/synctable-no-database.ts index d86e5f0989..454e20dcda 100644 --- a/src/packages/sync/table/synctable-no-database.ts +++ b/src/packages/sync/table/synctable-no-database.ts @@ -124,4 +124,6 @@ class ClientNoDatabase extends EventEmitter { // not implemented yet in general return undefined; }; + + client_id = () => this.client.client_id(); } diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index 42b5ac47e2..b6baa1bffc 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -930,7 +930,7 @@ export class SyncTable extends EventEmitter { need to be saved. If writing to the database results in an error (but not due to no network), - then an error state is set (which client can consult), an even is emitted, + then an error state is set (which client can consult), an event is emitted, and we do not try to write to the database again until that error state is cleared. One way it can be cleared is by changing the table. */ @@ -965,7 +965,7 @@ export class SyncTable extends EventEmitter { const changes = copy(this.changes); //console.log("_save: send ", changes); for (const key in this.changes) { - if (this.versions[key] === 0) { + if (this.versions[key] == 0) { proposed_keys[key] = true; } const x = this.value.get(key); @@ -1362,8 +1362,13 @@ export class SyncTable extends EventEmitter { const received_keys = this.apply_changes_to_browser_client(changes); if (before != null) { before.forEach((_, key) => { - if (key == null || received_keys[key]) return; // received as part of init - if (this.changes[key] && this.versions[key] == 0) return; // not event sent yet + if (key == null || received_keys[key]) { + return; // received as part of init + } + if (this.changes[key] && this.versions[key] == 0) { + return; + // not event sent yet + } // This key was known and confirmed sent before init, but // didn't get sent back this time. So it was lost somehow, // e.g., due to not getting saved to the database and the project diff --git a/src/packages/sync/table/test/client-test.ts b/src/packages/sync/table/test/client-test.ts index 08268e260c..fec8a7a9bf 100644 --- a/src/packages/sync/table/test/client-test.ts +++ b/src/packages/sync/table/test/client-test.ts @@ -71,4 +71,7 @@ export class ClientTest extends EventEmitter { // not implemented yet in general return undefined; }; + + // a random uuid... + client_id = () => "3fa218e5-7196-4020-8b30-e2127847cc4f"; } diff --git a/src/packages/util/db-schema/project-status.ts b/src/packages/util/db-schema/project-status.ts index d1127d74ea..9d891a25da 100644 --- a/src/packages/util/db-schema/project-status.ts +++ b/src/packages/util/db-schema/project-status.ts @@ -8,7 +8,9 @@ This table contains the current overall status about a running project. This is the sister-table to "project-info". In contrast, this table provides much less frequently changed pieces of status information. For example, project version, certain "alerts", disk usage, etc. -Its intended usage is to subscribe to it once you open a project and notify the user if certain alerts go off. +Its intended use is to subscribe to it once you open a project and notify the user if certain alerts go off. + +It is an ephemeral table stored in memory only in the project and clients. */ import { Table } from "./types"; diff --git a/src/packages/util/http-caching.ts b/src/packages/util/http-caching.ts new file mode 100644 index 0000000000..4b6a9e3cac --- /dev/null +++ b/src/packages/util/http-caching.ts @@ -0,0 +1,33 @@ +/* +Helper functions for caching static files when using express +*/ + +import ms from "ms"; + +// Used for longterm caching of files. This should be in units of seconds. +const MAX_AGE = Math.round(ms("10 days") / 1000); +const SHORT_AGE = Math.round(ms("10 seconds") / 1000); + +export function cacheShortTerm(res) { + res.setHeader( + "Cache-Control", + `public, max-age=${SHORT_AGE}, must-revalidate`, + ); + res.setHeader( + "Expires", + new Date(Date.now().valueOf() + SHORT_AGE).toUTCString(), + ); +} + +// Various files such as the webpack static content should be cached long-term, +// and we use this function to set appropriate headers at various points below. +export function cacheLongTerm(res) { + res.setHeader( + "Cache-Control", + `public, max-age=${MAX_AGE}, must-revalidate'`, + ); + res.setHeader( + "Expires", + new Date(Date.now().valueOf() + MAX_AGE).toUTCString(), + ); +} diff --git a/src/packages/util/package.json b/src/packages/util/package.json index bb4f13a1e1..386f7a87d1 100644 --- a/src/packages/util/package.json +++ b/src/packages/util/package.json @@ -12,7 +12,8 @@ "./sync/table": "./dist/sync/table/index.js", "./sync/editor/db": "./dist/sync/editor/db/index.js", "./licenses/purchase/*": "./dist/licenses/purchase/*.js", - "./redux/*": "./dist/redux/*.js" + "./redux/*": "./dist/redux/*.js", + "./http-caching": "./dist/http-caching.js" }, "scripts": { "preinstall": "npx only-allow pnpm", @@ -21,24 +22,15 @@ "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "mathjax", - "markdown", - "cocalc" - ], + "keywords": ["utilities", "mathjax", "markdown", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@ant-design/colors": "^6.0.0", "@cocalc/util": "workspace:*", "@types/debug": "^4.1.12", + "@types/ms": "^0.7.31", "async": "^1.5.2", "awaiting": "^3.0.0", "dayjs": "^1.11.11", @@ -51,6 +43,7 @@ "jsonic": "^1.0.1", "lodash": "^4.17.21", "lru-cache": "^7.18.3", + "ms": "2.1.2", "prop-types": "^15.7.2", "react-intl": "^6.7.0", "redux": "^4.2.1", diff --git a/src/packages/util/types/execute-code.ts b/src/packages/util/types/execute-code.ts index d6c3085250..1b59b95b0a 100644 --- a/src/packages/util/types/execute-code.ts +++ b/src/packages/util/types/execute-code.ts @@ -48,6 +48,8 @@ export interface ExecuteCodeOptions { aggregate?: string | number; // if given, aggregates multiple calls with same sequence number into one -- see @cocalc/util/aggregate; typically make this a timestamp for compiling code (e.g., latex). verbose?: boolean; // default true -- impacts amount of logging async_call?: boolean; // default false -- if true, return right after the process started (to get the PID) or when it fails. + // if ccNewFile is true, cc-new-file [args...] makes blank files if they don't exist (no template). ONLY supported for async version of executeCode (not callback) + ccNewFile?: boolean; } export interface ExecuteCodeOptionsAsyncGet { diff --git a/src/workspaces.py b/src/workspaces.py index a07aa4eca3..92cd25dcd6 100755 --- a/src/workspaces.py +++ b/src/workspaces.py @@ -113,6 +113,7 @@ def all_packages() -> List[str]: 'packages/api-client', 'packages/jupyter', 'packages/comm', + 'packages/sync-server', 'packages/assets', 'packages/frontend', # static depends on frontend; frontend depends on assets 'packages/project', # project depends on frontend for nbconvert (but NEVER vice versa again), which also depends on assets