Merge remote-tracking branch 'origin/master' into i18n-messages

src/compute/compute/dev/env.sh

@@ -6,10 +6,10 @@ export API_SERVER=`cat conf/api_server`
 export PROJECT_ID=`cat conf/project_id`
 export COMPUTE_SERVER_ID=`cat conf/compute_server_id`
 export HOSTNAME=`cat conf/hostname`
-export UNIONFS_UPPER=/tmp/upper
-export UNIONFS_LOWER=/tmp/lower
-export PROJECT_HOME=/tmp/home
-export READ_TRACKING_FILE=/tmp/reads
+export UNIONFS_UPPER=/tmp/upper5002
+export UNIONFS_LOWER=/tmp/lower5002
+export PROJECT_HOME=/tmp/home5002
+export READ_TRACKING_FILE=/tmp/reads5002
 export METADATA_FILE=$UNIONFS_LOWER/.compute-servers/$COMPUTE_SERVER_ID/meta/meta.lz4
 export EXCLUDE_FROM_SYNC=`cat conf/exclude_from_sync`
 

src/compute/compute/package.json

@@ -8,8 +8,8 @@
   "scripts": {
     "preinstall": "npx only-allow pnpm",
     "clean": "rm -rf dist node_modules",
-    "make": "pnpm install && pnpm run build",
-    "build": "../../packages/node_modules/.bin/tsc --build",
+    "make": "pnpm run build",
+    "build": "../../packages/node_modules/.bin/tsc",
     "tsc": "../../packages/node_modules/.bin/tsc --watch --pretty --preserveWatchOutput"
   },
   "bin": {

src/packages/database/postgres/blobs.ts

@@ -42,16 +42,19 @@ export async function unarchivePatches({
   }
   if (blob == null) {
     if (db.adminAlert != null) {
-      dbg("NONFATAL ERROR -- blob is GONE!");
-      // Instead of giving up, we basically give up on the syncstring history, and also
-      // send a message to admins to look into it.  This is better than completely blocking
-      // access to the file to the user, especially since they have the file on disk along
-      // with filesystem snapshots.  Also this *should* never happen.  I'm writing this because
-      // I switched .compute-servers.syncdb between ephemeral and not, which seems to have
-      // broken some of these, and I think we also hit this once or twice before.
-      await db.adminAlert({
-        subject: `missing TimeTravel history for path='${rows[0].path}'`,
-        body: `The blob with TimeTravel history for editing path='${rows[0].path}' is missing.
+      // having .compute-server.syncdb missing doesn't matter at all, since we don't care
+      // about that history
+      if (rows[0].path != ".compute-server.syncdb") {
+        dbg("NONFATAL ERROR -- blob is GONE!");
+        // Instead of giving up, we basically give up on the syncstring history, and also
+        // send a message to admins to look into it.  This is better than completely blocking
+        // access to the file to the user, especially since they have the file on disk along
+        // with filesystem snapshots.  Also this *should* never happen.  I'm writing this because
+        // I switched .compute-servers.syncdb between ephemeral and not, which seems to have
+        // broken some of these, and I think we also hit this once or twice before.
+        await db.adminAlert({
+          subject: `missing TimeTravel history for path='${rows[0].path}'`,
+          body: `The blob with TimeTravel history for editing path='${rows[0].path}' is missing.
 Instead of breaking things for the user, things might work, but with the history reset.  That said,
 an admin should look into this.
 
@@ -63,7 +66,8 @@ an admin should look into this.
 
 - error='${error}'
 `,
-      });
+        });
+      }
     } else {
       // can't even alert admins
       dbg("FATAL ERROR -- blob is gone (unable to alert admins)");

src/packages/frontend/client/api.ts

@@ -41,6 +41,7 @@ async function callApi(
   args?: object,
   numRetriesOnFail?: number,
 ) {
+  // console.log("callApi", { endpoint, args });
   const url = join(appBasePath, "api", endpoint);
   const resp = await fetch(url, {
     method: "POST",
@@ -73,5 +74,13 @@ async function callApi(
   if (typeof json == "object" && json.error) {
     throw Error(json.error);
   }
+  if (typeof json == "object" && json.errors) {
+    // This is what happens when the api request fails due to schema validation issues.
+    // I.e., this is soemthing we only see in dev mode since the schema stuff is disabled in production.
+    throw Error(
+      `API Schema Error: ${json.message} ${JSON.stringify(json.errors)}`,
+    );
+  }
+  // console.log("got ", json);
   return json;
 }

src/packages/frontend/client/client.ts

@@ -76,8 +76,8 @@ export interface WebappClient extends EventEmitter {
   synctable_project: Function;
   project_websocket: Function;
   prettier: Function;
-  exec: Function; // TODO: rewrite project_actions.ts to not use this at all.
-  touch_project: (project_id: string) => void;
+  exec: Function;
+  touch_project: (project_id: string, compute_server_id?: number) => void;
   ipywidgetsGetBuffer: (
     project_id: string,
     path: string,
@@ -99,7 +99,6 @@ export interface WebappClient extends EventEmitter {
   is_deleted: (filename: string, project_id: string) => boolean;
   set_deleted: Function;
   mark_file: (opts: any) => Promise<void>;
-
   set_connected?: Function;
   version: Function;
 }
@@ -157,8 +156,8 @@ class Client extends EventEmitter implements WebappClient {
   synctable_project: Function;
   project_websocket: Function;
   prettier: Function;
-  exec: Function; // TODO: rewrite project_actions.ts to not use this at all.
-  touch_project: (project_id: string) => void;
+  exec: Function;
+  touch_project: (project_id: string, compute_server_id?: number) => void;
   ipywidgetsGetBuffer: (
     project_id: string,
     path: string,
@@ -242,7 +241,7 @@ class Client extends EventEmitter implements WebappClient {
     this.idle_reset = this.idle_client.idle_reset.bind(this.idle_client);
 
     this.exec = this.project_client.exec.bind(this.project_client);
-    this.touch_project = this.project_client.touch.bind(this.project_client);
+    this.touch_project = this.project_client.touch_project.bind(this.project_client);
     this.ipywidgetsGetBuffer = this.project_client.ipywidgetsGetBuffer.bind(
       this.project_client,
     );

src/packages/frontend/client/project.ts

@@ -55,6 +55,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
 import { DirectoryListingEntry } from "@cocalc/util/types";
 import httpApi from "./api";
 import { WebappClient } from "./client";
+import { throttle } from "lodash";
 
 export class ProjectClient {
   private client: WebappClient;
@@ -414,7 +415,24 @@ export class ProjectClient {
     await this.call(message.remove_all_upgrades({ projects }));
   }
 
-  public async touch(project_id: string): Promise<void> {
+  touch_project = async (
+    // project_id where activity occured
+    project_id: string,
+    // optional global id of a compute server (in the given project), in which case we also mark
+    // that compute server as active, which keeps it running in case it has idle timeout configured.
+    compute_server_id?: number,
+  ): Promise<void> => {
+    if (compute_server_id) {
+      // this is throttled, etc. and is independent of everything below.
+      touchComputeServer({
+        project_id,
+        compute_server_id,
+        client: this.client,
+      });
+      // that said, we do still touch the project, since if a user is actively
+      // using a compute server, the project should also be considered active.
+    }
+
     const state = redux.getStore("projects")?.get_state(project_id);
     if (!(state == null && redux.getStore("account")?.get("is_admin"))) {
       // not trying to view project as admin so do some checks
@@ -448,7 +466,7 @@ export class ProjectClient {
       // the project (updating the db), but it still *does*
       // ensure there is a TCP connection to the project.
     }
-  }
+  };
 
   // Print file to pdf
   // The printed version of the file will be created in the same directory
@@ -622,3 +640,32 @@ export class ProjectClient {
     return await computeServers(project_id)?.getServerIdForPath(path);
   };
 }
+
+// (NOTE: this won't throw an exception)
+const touchComputeServer = throttle(
+  async ({ project_id, compute_server_id, client }) => {
+    if (!compute_server_id) {
+      // nothing to do
+      return;
+    }
+    try {
+      await client.async_query({
+        query: {
+          compute_servers: {
+            project_id,
+            id: compute_server_id,
+            last_edited_user: client.server_time(),
+          },
+        },
+      });
+    } catch (err) {
+      // just a warning -- if we can't connect then touching isn't something we should be doing anyways.
+      console.log(
+        "WARNING: failed to touch compute server -- ",
+        { compute_server_id },
+        err,
+      );
+    }
+  },
+  30000,
+);

src/packages/frontend/compute/action.tsx

@@ -472,7 +472,7 @@ function OnPremGuide({ setShow, configuration, id, title, action }) {
           </div>
         )}
         {!apiKey && !error && <Spin />}
-        {error && <ShowError error={error} setError={setError} />}
+        <ShowError error={error} setError={setError} />
       </div>
       {action == "stop" && (
         <div>

src/packages/frontend/compute/api.ts

@@ -1,10 +1,10 @@
 import api from "@cocalc/frontend/client/api";
-("");
 import type {
   Action,
+  Cloud,
   ComputeServerTemplate,
+  ComputeServerUserInfo,
   Configuration,
-  Cloud,
   Images,
   GoogleCloudImages,
 } from "@cocalc/util/db-schema/compute-servers";
@@ -21,11 +21,12 @@ export async function createServer(opts: {
   project_id: string;
   title?: string;
   color?: string;
-  idle_timeout?: number;
   autorestart?: boolean;
   cloud?: Cloud;
   configuration?: Configuration;
   notes?: string;
+  course_project_id?: string;
+  course_server_id?: number;
 }): Promise<number> {
   return await api("compute/create-server", opts);
 }
@@ -37,7 +38,25 @@ export async function computeServerAction(opts: {
   await api("compute/compute-server-action", opts);
 }
 
-export async function getServers(opts: { id?: number; project_id: string }) {
+// Get servers across potentially different projects by their global unique id.
+// Use the fields parameter to restrict to a much smaller subset of information
+// about each server (e.g., just the state field).  Caller must be a collaborator
+// on each project containing the servers.
+// If you give an id of a server that doesn't exist, it'll just be excluded in the result.
+// Similarly, if you give a field that doesn't exist, it is excluded.
+// The order of the returned servers and count probably will NOT match that in
+// ids, so you should include 'id' in fields.
+export async function getServersById(opts: {
+  ids: number[];
+  fields?: string[];
+}): Promise<Partial<ComputeServerUserInfo>[]> {
+  return await api("compute/get-servers-by-id", opts);
+}
+
+export async function getServers(opts: {
+  id?: number;
+  project_id: string;
+}): Promise<ComputeServerUserInfo[]> {
   return await api("compute/get-servers", opts);
 }
 
@@ -50,7 +69,12 @@ export async function getSerialPortOutput(id: number) {
 }
 
 export async function deleteServer(id: number) {
-  await api("compute/delete-server", { id });
+  return await api("compute/delete-server", { id });
+}
+
+export async function isDnsAvailable(dns: string) {
+  const { isAvailable } = await api("compute/is-dns-available", { dns });
+  return isAvailable;
 }
 
 export async function undeleteServer(id: number) {
@@ -69,7 +93,7 @@ export async function setServerTitle(opts: { id: number; title: string }) {
 
 export async function setServerConfiguration(opts: {
   id: number;
-  configuration;
+  configuration: Partial<Configuration>;
 }) {
   return await api("compute/set-server-configuration", opts);
 }
@@ -186,7 +210,7 @@ export async function deleteApiKey(opts: { id }): Promise<string> {
 }
 
 // Get the project log entries directly for just one compute server
-export async function getLog(opts: { id }) {
+export async function getLog(opts: { id; type: "activity" | "files" }) {
   return await api("compute/get-log", opts);
 }