diff --git a/src/packages/frontend/compute/log-entry.tsx b/src/packages/frontend/compute/log-entry.tsx
index 89ff472b96..3b4f563e72 100644
--- a/src/packages/frontend/compute/log-entry.tsx
+++ b/src/packages/frontend/compute/log-entry.tsx
@@ -2,8 +2,11 @@ import { useTypedRedux } from "@cocalc/frontend/app-framework";
import { Icon, isIconName } from "@cocalc/frontend/components";
import ComputeServerTag from "@cocalc/frontend/compute/server-tag";
import type { ComputeServerEvent } from "@cocalc/util/compute/log";
-import { STATE_INFO } from "@cocalc/util/db-schema/compute-servers";
-import { capitalize, plural } from "@cocalc/util/misc";
+import {
+ STATE_INFO,
+ spendLimitPeriod,
+} from "@cocalc/util/db-schema/compute-servers";
+import { capitalize, currency, plural } from "@cocalc/util/misc";
export default function LogEntry({
project_id,
@@ -72,6 +75,15 @@ export default function LogEntry({
{event.idle_timeout} {plural(event.idle_timeout, "minute")}) {tag}
>
);
+ case "spend-limit":
+ return (
+ <>
+ {cs} - Spend Limit Shutdown (total spend during the last{" "}
+ {spendLimitPeriod(event.spendLimit?.hours)} hit{" "}
+ {currency(event.total)} which exceeded limit of{" "}
+ {currency(event.spendLimit?.dollars)}) {tag}
+ >
+ );
default:
return (
<>
diff --git a/src/packages/frontend/compute/spend-limit.tsx b/src/packages/frontend/compute/spend-limit.tsx
index ea3b309e81..b5e9581d62 100644
--- a/src/packages/frontend/compute/spend-limit.tsx
+++ b/src/packages/frontend/compute/spend-limit.tsx
@@ -11,6 +11,7 @@ import { setServerConfiguration } from "./api";
import {
type SpendLimit as ISpendLimit,
SPEND_LIMIT_DEFAULTS,
+ spendLimitPeriod
} from "@cocalc/util/db-schema/compute-servers";
import { AutomaticShutdownCard } from "./automatic-shutdown";
@@ -104,7 +105,7 @@ export function SpendLimit({
- Maximum amount to spend per {period(spendLimit.hours)}:{" "}
+ Maximum amount to spend per {spendLimitPeriod(spendLimit.hours)}:{" "}
'{spendLimit}' AS spend_limit
+FROM compute_servers
+WHERE state = 'running'
+ AND (configuration#>>'{spendLimit,enabled}')::boolean = true
+`,
+ );
+ logger.debug(`got ${rows.length} servers with an enabled spend limit:`, rows);
+ const f = async (row) => {
+ logger.debug("checking if spend limit is hit", row);
+ const { dollars, hours } = validatedSpendLimit(row.spend_limit ?? {})!;
+ const { purchases } = await getPurchases({
+ compute_server_id: row.id,
+ account_id: row.account_id,
+ group: true,
+ cutoff: dayjs().subtract(hours, "hour").toDate(),
+ });
+ let total = 0;
+ for (const { cost, cost_so_far } of purchases) {
+ total += cost ?? cost_so_far ?? 0;
+ }
+ try {
+ await pool.query("UPDATE compute_servers SET spend=$1 where id=$2", [
+ total,
+ row.id,
+ ]);
+ } catch (err) {
+ logger.debug(`WARNING -- unable to update spend field -- ${err}`);
+ }
+ if (total < dollars) {
+ logger.debug("spend is under the limit -- nothing to do", row);
+ return;
+ }
+ try {
+ await createProjectLogEntry({ ...row, total });
+ const { account_id, id } = row;
+ await stop({ account_id, id });
+ } catch (err) {
+ logger.debug(
+ `WARNING -- failed to stop ${row.id} in response to idle timeout -- ${err}`,
+ );
+ }
+ };
+ await map(rows, 20, f);
+}
+
+async function createProjectLogEntry({
+ id,
+ account_id,
+ project_id,
+ spend_limit,
+ total,
+}: {
+ id: number;
+ account_id: string;
+ project_id: string;
+ spend_limit: SpendLimit;
+ total: number;
+}) {
+ logger.debug("log entry that we spend limit terminated compute server", {
+ id,
+ });
+ const pool = getPool();
+ await pool.query(
+ "INSERT INTO project_log(id, project_id, account_id, time, event) VALUES($1,$2,$3,NOW(),$4)",
+ [
+ uuid(),
+ project_id,
+ account_id,
+ {
+ event: "compute-server",
+ action: "spend-limit",
+ spendLimit: spend_limit,
+ total,
+ server_id: id,
+ } as ComputeServerEventLogEntry,
+ ],
+ );
+}
diff --git a/src/packages/server/compute/set-server-configuration.ts b/src/packages/server/compute/set-server-configuration.ts
index 28fcae3b1e..bc58b476b7 100644
--- a/src/packages/server/compute/set-server-configuration.ts
+++ b/src/packages/server/compute/set-server-configuration.ts
@@ -25,6 +25,7 @@ import updatePurchase from "./update-purchase";
import { isDnsAvailable } from "./dns";
import { setConfiguration } from "./util";
import { validatedSpendLimit } from "@cocalc/util/db-schema/compute-servers";
+import { isEqual } from "lodash";
export default async function setServerConfiguration({
account_id,
@@ -65,6 +66,12 @@ export default async function setServerConfiguration({
...configuration,
spendLimit: validatedSpendLimit(configuration.spendLimit),
};
+ if (!isEqual(currentConfiguration.spendLimit, configuration.spendLimit)) {
+ // changing spendLimit invalidates "spend during the given period".
+ await pool.query("UPDATE compute_servers SET spend=NULL where id=$1", [
+ id,
+ ]);
+ }
}
await validateConfigurationChange({
diff --git a/src/packages/server/purchases/get-purchases.ts b/src/packages/server/purchases/get-purchases.ts
index a3f3c6731f..8fa493b88c 100644
--- a/src/packages/server/purchases/get-purchases.ts
+++ b/src/packages/server/purchases/get-purchases.ts
@@ -20,7 +20,8 @@ import { getOwner } from "@cocalc/server/compute/owner";
interface Options {
account_id: string;
- cutoff?: Date; // returns purchases back to this date (limit/offset NOT ignored)
+ // returns purchases back to this date (limit/offset NOT ignored); never excludes unfinished purchases (i.e., with cost not set)
+ cutoff?: Date;
thisMonth?: boolean;
limit?: number;
offset?: number;
@@ -121,7 +122,7 @@ export default async function getPurchases({
}
if (cutoff) {
params.push(cutoff);
- conditions.push(`p.time >= $${params.length}`);
+ conditions.push(`(p.time >= $${params.length} OR p.cost IS NULL)`);
}
if (no_statement) {
conditions.push("p.day_statement_id IS NULL");
diff --git a/src/packages/util/compute/log.ts b/src/packages/util/compute/log.ts
index 16e922e7cc..e21f1fc8b7 100644
--- a/src/packages/util/compute/log.ts
+++ b/src/packages/util/compute/log.ts
@@ -1,6 +1,7 @@
import type {
State,
AutomaticShutdown,
+ SpendLimit,
} from "@cocalc/util/db-schema/compute-servers";
interface Event {
@@ -28,6 +29,12 @@ export interface IdleTimeoutEntry {
idle_timeout: number;
}
+export interface SpendLimitEntry {
+ action: "spend-limit";
+ spendLimit: SpendLimit;
+ total: number;
+}
+
interface Error {
action: "error";
error: string;
@@ -39,6 +46,7 @@ export type ComputeServerEvent = (
| Error
| AutomaticShutdownEntry
| IdleTimeoutEntry
+ | SpendLimitEntry
) &
Event;
@@ -47,4 +55,5 @@ export type ComputeServerEventLogEntry =
| StateChange
| AutomaticShutdownEntry
| IdleTimeoutEntry
+ | SpendLimitEntry
| Error;
diff --git a/src/packages/util/db-schema/compute-servers.ts b/src/packages/util/db-schema/compute-servers.ts
index 7bb0f71c63..2c5c6d80ad 100644
--- a/src/packages/util/db-schema/compute-servers.ts
+++ b/src/packages/util/db-schema/compute-servers.ts
@@ -421,14 +421,30 @@ export function validatedSpendLimit(spendLimit?: any): SpendLimit | undefined {
dollars = 1;
}
if (!isFinite(hours)) {
- throw Error("hours must be finite");
+ throw Error(`hours (=${hours}) must be finite`);
}
if (!isFinite(dollars)) {
- throw Error("dollars must be finite");
+ throw Error(`dollars (=${dollars}) must be finite`);
}
return { enabled, hours, dollars };
}
+export function spendLimitPeriod(hours) {
+ if (hours == 24) {
+ return "day";
+ }
+ if (hours == 24 * 7) {
+ return "week";
+ }
+ if (hours == 30.5 * 24 * 7) {
+ return "month";
+ }
+ if (hours == 12 * 30.5 * 24 * 7) {
+ return "year";
+ }
+ return `${hours} hours`;
+}
+
interface BaseConfiguration {
// image: name of the image to use, e.g. 'python' or 'pytorch'.
// images are managed in src/packages/server/compute/images.ts
@@ -727,6 +743,7 @@ export interface ComputeServerUserInfo {
update_purchase?: boolean;
last_purchase_update?: Date;
template?: ComputeServerTemplate;
+ spend?: number;
}
export interface ComputeServer extends ComputeServerUserInfo {
@@ -778,6 +795,7 @@ Table({
project_specific_id: null,
course_project_id: null,
course_server_id: null,
+ spend: null,
},
},
set: {
@@ -962,6 +980,10 @@ Table({
type: "integer",
desc: "If this compute server is a clone of an instructor server in a course, this is the id of that instructor server.",
},
+ spend: {
+ type: "number",
+ desc: "If configuration.spendLimit is enabled, then the spend during the current period gets recorded here every few minutes. This is useful to efficiently provide a UI element showing the current spend status. It is cleared whenever configuration.spendLimit is changed, to avoid confusion.",
+ },
},
});
diff --git a/src/packages/util/db-schema/purchases.ts b/src/packages/util/db-schema/purchases.ts
index 2d742299e8..0b0715aea0 100644
--- a/src/packages/util/db-schema/purchases.ts
+++ b/src/packages/util/db-schema/purchases.ts
@@ -257,7 +257,7 @@ Table({
},
pending: {
type: "boolean",
- desc: "If true, then this transaction is considered pending, which means that for a few days it doesn't count against the user's quotas for the purposes of deciding whether or not a purchase is allowed. This is needed so we can charge a user for their subscriptions, then collect the money from them, without all of the running pay-as-you-go project upgrades suddenly breaking (etc.).",
+ desc: "**DEPRECATED** -- not used anywhere; do NOT use! If true, then this transaction is considered pending, which means that for a few days it doesn't count against the user's quotas for the purposes of deciding whether or not a purchase is allowed. This is needed so we can charge a user for their subscriptions, then collect the money from them, without all of the running pay-as-you-go project upgrades suddenly breaking (etc.).",
},
cost_per_hour: {
title: "Cost Per Hour",