compute server: http interface

src/compute/compute/dev/README.md

@@ -47,3 +47,16 @@ For debugging set the DEBUG env variable to different things according to the de
 DEBUG_CONSOLE=yes DEBUG=* ./2-syncfs.sh
 ```
 
+### Console
+
+Once you have everything setup you can get an interactive console by doing this instead of [3\-compute.sh](http://3-compute.sh): 
+
+```sh
+~/cocalc/src/compute/compute/dev$ . env.sh
+~/cocalc/src/compute/compute/dev$ node
+Welcome to Node.js v18.17.1.
+Type ".help" for more information.
+> manager = require('./start-compute.js').manager
+
+```
+

src/compute/compute/dev/start-compute.js

@@ -1,11 +1,23 @@
 #!/usr/bin/env node
 
+/*
+To get an interactive console with access to the manager:
+
+~/cocalc/src/compute/compute/dev$ node
+Welcome to Node.js v18.17.1.
+Type ".help" for more information.
+> manager = require('./start-compute.js').manager
+
+*/
+
 process.env.API_SERVER = process.env.API_SERVER ?? "https://cocalc.com";
 console.log("API_SERVER=", process.env.API_SERVER);
 
 const { manager } = require("../dist/lib");
 
 const PROJECT_HOME = process.env.PROJECT_HOME ?? "/tmp/home";
+const PORT = process.env.PORT ?? 5004;
+const HOST = process.env.HOST ?? "0.0.0.0";
 
 async function main() {
   const exitHandler = async () => {
@@ -28,6 +40,8 @@ async function main() {
       process.env.UNIONFS_UPPER && process.env.UNIONFS_LOWER
         ? "fuse.unionfs-fuse"
         : "fuse",
+    host: HOST,
+    port: PORT,
   });
   exports.manager = M;
   await M.init();

src/compute/compute/lib/http-server/http-next-api.ts

@@ -0,0 +1,52 @@
+/*
+HTTP Next API
+
+- this is whatever we need from cocalc/src/packages/next/pages/api/v2 specifically
+  for working with one project.
+
+*/
+
+import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data";
+import { Router } from "express";
+import { getLogger } from "../logger";
+import type { Manager } from "../manager";
+
+const logger = getLogger("http-next-api");
+
+export default function initHttpNextApi({ manager }: { manager }): Router {
+  logger.info("initHttpNextApi");
+  const router = Router();
+
+  for (const path in HANDLERS) {
+    router.post("/" + path, handler(manager, HANDLERS[path]));
+    router.get("/" + path, handler(manager, HANDLERS[path]));
+  }
+
+  router.post("*", (req, res) => {
+    res.json({ error: `api endpoint '${req.path}' is not implemented` });
+  });
+  router.get("*", (req, res) => {
+    res.json({ error: `api endpoint '${req.path}' is not implemented` });
+  });
+
+  return router;
+}
+
+function handler(
+  manager,
+  f: (x: { req; res; manager: Manager }) => Promise<object>,
+) {
+  return async (req, res) => {
+    try {
+      res.json({ success: true, ...(await f({ req, res, manager })) });
+    } catch (err) {
+      res.json({ error: `${err}` });
+    }
+  };
+}
+
+const HANDLERS = {
+  "jupyter/kernels": async () => {
+    return { kernels: await get_kernel_data() };
+  },
+};

src/compute/compute/lib/http-server/hub-websocket.ts

@@ -0,0 +1,98 @@
+/*
+ *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
+ *  License: MS-RSL – see LICENSE.md for details
+ */
+
+/*
+Create websocket similar to connection to normal hub.
+Handle stuff that doesn't directly involve the project or compute server,
+e.g., user identity.
+*/
+
+import { Router } from "express";
+import { Server } from "http";
+import Primus from "primus";
+import { getLogger } from "../logger";
+import { from_json_socket, to_json_socket } from "@cocalc/util/misc";
+import * as message from "@cocalc/util/message";
+
+const logger = getLogger("hub-websocket");
+
+export default function initHubWebsocket({
+  server,
+  manager,
+}: {
+  server: Server;
+  manager;
+}): Router {
+  const opts = {
+    pathname: "/hub",
+    transformer: "websockets",
+  } as const;
+  logger.debug(`Initializing primus websocket server at "${opts.pathname}"...`);
+  const primus = new Primus(server, opts);
+  initApi({ primus, manager });
+
+  const router = Router();
+  const library: string = primus.library();
+
+  // it isn't actually minified, but this is what the static code expects.
+  router.get("/primus.min.js", (_, res) => {
+    logger.debug("serving up /primus.min.js to a specific client");
+    res.send(library);
+  });
+  logger.debug(
+    `waiting for browser client to request primus.min.js (length=${library.length})...`,
+  );
+
+  return router;
+}
+
+function initApi({ primus, manager }): void {
+  primus.on("connection", (spark) => {
+    logger.debug(`HUB: new connection from ${spark.address.ip} -- ${spark.id}`);
+
+    const sendResponse = (mesg) => {
+      const data = to_json_socket(mesg);
+      spark.write(data);
+    };
+
+    spark.on("data", async (data) => {
+      const mesg = from_json_socket(data);
+      logger.debug("HUB:", "request", mesg, "REQUEST");
+      const t0 = Date.now();
+      try {
+        const resp0 = await handleApiCall(mesg, spark, manager, primus);
+        const resp = {
+          ...resp0,
+          id: mesg.id,
+        };
+        sendResponse(resp);
+        logger.debug(
+          "HUB",
+          "response",
+          resp,
+          `FINISHED: time=${Date.now() - t0}ms`,
+        );
+      } catch (err) {
+        // console.trace(); logger.debug("primus-api error stacktrack", err.stack, err);
+        logger.debug("HUB:", "failed response to", mesg, "FAILED", `${err}`);
+        sendResponse({ id: mesg.id, error: err.toString() });
+      }
+    });
+  });
+
+  primus.on("disconnection", (spark) => {
+    logger.debug(`HUB: end connection from ${spark.address.ip} -- ${spark.id}`);
+  });
+}
+
+async function handleApiCall(mesg, spark, manager, primus): Promise<object> {
+  // @ts-ignore
+  const _foo = { mesg, spark, manager, primus };
+  switch (mesg.event) {
+    case "ping":
+      return message.pong({ now: new Date() });
+  }
+  throw Error("not implemented");
+}

src/compute/compute/lib/http-server/index.ts

@@ -0,0 +1,88 @@
+/*
+ *  This file is part of CoCalc: Copyright © 2024 Sagemath, Inc.
+ *  License: MS-RSL – see LICENSE.md for details
+ */
+
+import compression from "compression";
+import express from "express";
+import { createServer } from "http";
+import { getLogger } from "../logger";
+import type { Manager } from "../manager";
+import { path as STATIC_PATH } from "@cocalc/static";
+import { join } from "path";
+import { cacheShortTerm, cacheLongTerm } from "@cocalc/util/http-caching";
+import initWebsocket from "./websocket";
+import initHubWebsocket from "./hub-websocket";
+import initHttpNextApi from "./http-next-api";
+import initRaw from "./raw-server";
+
+const logger = getLogger("http-server");
+
+const ENTRY_POINT = "compute.html";
+
+export function initHttpServer({
+  port = 5004,
+  host = "localhost",
+  manager,
+}: {
+  port?: number;
+  host?: string;
+  manager: Manager;
+}) {
+  logger.info("starting http-server...");
+
+  const app = express();
+  const server = createServer(app);
+
+  // this is expected by the frontend code for where to find the project.
+  const projectBase = `/${manager.project_id}/raw/`;
+  logger.info({ projectBase });
+
+  app.use(projectBase, initWebsocket({ server, projectBase, manager }));
+
+  app.use("/", initHubWebsocket({ server, manager }));
+
+  // CRITICAL: compression must be after websocket above!
+  app.use(compression());
+
+  app.get("/", (_req, res) => {
+    const files = manager.getOpenFiles();
+    res.send(
+      `<h1>Compute Server</h1>  <a href="${join("/static", ENTRY_POINT)}">CoCalc App</a> <br/><br/> Open Files: ${files.join(", ")}`,
+    );
+  });
+
+  app.use(
+    `/static/${ENTRY_POINT}`,
+    express.static(`/${STATIC_PATH}/${ENTRY_POINT}`, {
+      setHeaders: cacheShortTerm,
+    }),
+  );
+  app.use(
+    "/static",
+    express.static(STATIC_PATH, { setHeaders: cacheLongTerm }),
+  );
+
+  app.get("/customize", (_req, res) => {
+    res.json({
+      configuration: {
+        compute_server: { project_id: manager.project_id },
+      },
+      registration: false,
+    });
+  });
+
+  app.use("/api/v2", initHttpNextApi({ manager }));
+
+  const rawUrl = `/${manager.project_id}/raw`;
+  logger.debug("raw server at ", { rawUrl });
+  app.use(rawUrl, initRaw({ home: manager.home }));
+
+  app.get("*", (_req, res) => {
+    res.redirect(join("/static", ENTRY_POINT));
+  });
+
+  server.listen(port, host, () => {
+    logger.info(`Server listening http://${host}:${port}`);
+  });
+}

src/compute/compute/lib/http-server/raw-server.ts

@@ -0,0 +1,30 @@
+/*
+Serve static files from the home directory.  
+
+NOTE: There is a very similar server in /src/packages/project/servers/browser/static.ts
+See comments there.
+*/
+
+import { static as staticServer } from "express";
+import index from "serve-index";
+import { getLogger } from "../logger";
+import { Router } from "express";
+
+const log = getLogger("http-server:static");
+
+export default function initStatic({ home }: { home: string }): Router {
+  const router = Router();
+  router.use("/", (req, res, next) => {
+    if (req.query.download != null) {
+      res.setHeader("Content-Type", "application/octet-stream");
+    }
+    res.setHeader("Cache-Control", "private, must-revalidate");
+    next();
+  });
+
+  log.info(`serving up HOME="${home}"`);
+
+  router.use("/", index(home, { hidden: true, icons: true }));
+  router.use("/", staticServer(home, { dotfiles: "allow" }));
+  return router;
+}

src/compute/compute/lib/http-server/synctable-channel.ts

@@ -0,0 +1,19 @@
+import { getLogger } from "../logger";
+import { synctable_channel } from "@cocalc/sync-server/server/server";
+
+const log = getLogger("synctable-channel");
+
+export default async function synctableChannel({
+  manager,
+  query,
+  options,
+  primus,
+}: {
+  manager;
+  query;
+  options;
+  primus;
+}) {
+  log.debug("synctableChannel ", query, options);
+  return await synctable_channel(manager.client, primus, log, query, options);
+}