diff --git a/src/packages/frontend/compute/log-entry.tsx b/src/packages/frontend/compute/log-entry.tsx index 89ff472b96..3b4f563e72 100644 --- a/src/packages/frontend/compute/log-entry.tsx +++ b/src/packages/frontend/compute/log-entry.tsx @@ -2,8 +2,11 @@ import { useTypedRedux } from "@cocalc/frontend/app-framework"; import { Icon, isIconName } from "@cocalc/frontend/components"; import ComputeServerTag from "@cocalc/frontend/compute/server-tag"; import type { ComputeServerEvent } from "@cocalc/util/compute/log"; -import { STATE_INFO } from "@cocalc/util/db-schema/compute-servers"; -import { capitalize, plural } from "@cocalc/util/misc"; +import { + STATE_INFO, + spendLimitPeriod, +} from "@cocalc/util/db-schema/compute-servers"; +import { capitalize, currency, plural } from "@cocalc/util/misc"; export default function LogEntry({ project_id, @@ -72,6 +75,15 @@ export default function LogEntry({ {event.idle_timeout} {plural(event.idle_timeout, "minute")}) {tag} ); + case "spend-limit": + return ( + <> + {cs} - Spend Limit Shutdown (total spend during the last{" "} + {spendLimitPeriod(event.spendLimit?.hours)} hit{" "} + {currency(event.total)} which exceeded limit of{" "} + {currency(event.spendLimit?.dollars)}) {tag} + + ); default: return ( <> diff --git a/src/packages/frontend/compute/spend-limit.tsx b/src/packages/frontend/compute/spend-limit.tsx index ea3b309e81..b5e9581d62 100644 --- a/src/packages/frontend/compute/spend-limit.tsx +++ b/src/packages/frontend/compute/spend-limit.tsx @@ -11,6 +11,7 @@ import { setServerConfiguration } from "./api"; import { type SpendLimit as ISpendLimit, SPEND_LIMIT_DEFAULTS, + spendLimitPeriod } from "@cocalc/util/db-schema/compute-servers"; import { AutomaticShutdownCard } from "./automatic-shutdown"; @@ -104,7 +105,7 @@ export function SpendLimit({
- Maximum amount to spend per {period(spendLimit.hours)}:{" "} + Maximum amount to spend per {spendLimitPeriod(spendLimit.hours)}:{" "}
'{spendLimit}' AS spend_limit +FROM compute_servers +WHERE state = 'running' + AND (configuration#>>'{spendLimit,enabled}')::boolean = true +`, + ); + logger.debug(`got ${rows.length} servers with an enabled spend limit:`, rows); + const f = async (row) => { + logger.debug("checking if spend limit is hit", row); + const { dollars, hours } = validatedSpendLimit(row.spend_limit ?? {})!; + const { purchases } = await getPurchases({ + compute_server_id: row.id, + account_id: row.account_id, + group: true, + cutoff: dayjs().subtract(hours, "hour").toDate(), + }); + let total = 0; + for (const { cost, cost_so_far } of purchases) { + total += cost ?? cost_so_far ?? 0; + } + try { + await pool.query("UPDATE compute_servers SET spend=$1 where id=$2", [ + total, + row.id, + ]); + } catch (err) { + logger.debug(`WARNING -- unable to update spend field -- ${err}`); + } + if (total < dollars) { + logger.debug("spend is under the limit -- nothing to do", row); + return; + } + try { + await createProjectLogEntry({ ...row, total }); + const { account_id, id } = row; + await stop({ account_id, id }); + } catch (err) { + logger.debug( + `WARNING -- failed to stop ${row.id} in response to idle timeout -- ${err}`, + ); + } + }; + await map(rows, 20, f); +} + +async function createProjectLogEntry({ + id, + account_id, + project_id, + spend_limit, + total, +}: { + id: number; + account_id: string; + project_id: string; + spend_limit: SpendLimit; + total: number; +}) { + logger.debug("log entry that we spend limit terminated compute server", { + id, + }); + const pool = getPool(); + await pool.query( + "INSERT INTO project_log(id, project_id, account_id, time, event) VALUES($1,$2,$3,NOW(),$4)", + [ + uuid(), + project_id, + account_id, + { + event: "compute-server", + action: "spend-limit", + spendLimit: spend_limit, + total, + server_id: id, + } as ComputeServerEventLogEntry, + ], + ); +} diff --git a/src/packages/server/compute/set-server-configuration.ts b/src/packages/server/compute/set-server-configuration.ts index 28fcae3b1e..bc58b476b7 100644 --- a/src/packages/server/compute/set-server-configuration.ts +++ b/src/packages/server/compute/set-server-configuration.ts @@ -25,6 +25,7 @@ import updatePurchase from "./update-purchase"; import { isDnsAvailable } from "./dns"; import { setConfiguration } from "./util"; import { validatedSpendLimit } from "@cocalc/util/db-schema/compute-servers"; +import { isEqual } from "lodash"; export default async function setServerConfiguration({ account_id, @@ -65,6 +66,12 @@ export default async function setServerConfiguration({ ...configuration, spendLimit: validatedSpendLimit(configuration.spendLimit), }; + if (!isEqual(currentConfiguration.spendLimit, configuration.spendLimit)) { + // changing spendLimit invalidates "spend during the given period". + await pool.query("UPDATE compute_servers SET spend=NULL where id=$1", [ + id, + ]); + } } await validateConfigurationChange({ diff --git a/src/packages/server/purchases/get-purchases.ts b/src/packages/server/purchases/get-purchases.ts index a3f3c6731f..8fa493b88c 100644 --- a/src/packages/server/purchases/get-purchases.ts +++ b/src/packages/server/purchases/get-purchases.ts @@ -20,7 +20,8 @@ import { getOwner } from "@cocalc/server/compute/owner"; interface Options { account_id: string; - cutoff?: Date; // returns purchases back to this date (limit/offset NOT ignored) + // returns purchases back to this date (limit/offset NOT ignored); never excludes unfinished purchases (i.e., with cost not set) + cutoff?: Date; thisMonth?: boolean; limit?: number; offset?: number; @@ -121,7 +122,7 @@ export default async function getPurchases({ } if (cutoff) { params.push(cutoff); - conditions.push(`p.time >= $${params.length}`); + conditions.push(`(p.time >= $${params.length} OR p.cost IS NULL)`); } if (no_statement) { conditions.push("p.day_statement_id IS NULL"); diff --git a/src/packages/util/compute/log.ts b/src/packages/util/compute/log.ts index 16e922e7cc..e21f1fc8b7 100644 --- a/src/packages/util/compute/log.ts +++ b/src/packages/util/compute/log.ts @@ -1,6 +1,7 @@ import type { State, AutomaticShutdown, + SpendLimit, } from "@cocalc/util/db-schema/compute-servers"; interface Event { @@ -28,6 +29,12 @@ export interface IdleTimeoutEntry { idle_timeout: number; } +export interface SpendLimitEntry { + action: "spend-limit"; + spendLimit: SpendLimit; + total: number; +} + interface Error { action: "error"; error: string; @@ -39,6 +46,7 @@ export type ComputeServerEvent = ( | Error | AutomaticShutdownEntry | IdleTimeoutEntry + | SpendLimitEntry ) & Event; @@ -47,4 +55,5 @@ export type ComputeServerEventLogEntry = | StateChange | AutomaticShutdownEntry | IdleTimeoutEntry + | SpendLimitEntry | Error; diff --git a/src/packages/util/db-schema/compute-servers.ts b/src/packages/util/db-schema/compute-servers.ts index 7bb0f71c63..2c5c6d80ad 100644 --- a/src/packages/util/db-schema/compute-servers.ts +++ b/src/packages/util/db-schema/compute-servers.ts @@ -421,14 +421,30 @@ export function validatedSpendLimit(spendLimit?: any): SpendLimit | undefined { dollars = 1; } if (!isFinite(hours)) { - throw Error("hours must be finite"); + throw Error(`hours (=${hours}) must be finite`); } if (!isFinite(dollars)) { - throw Error("dollars must be finite"); + throw Error(`dollars (=${dollars}) must be finite`); } return { enabled, hours, dollars }; } +export function spendLimitPeriod(hours) { + if (hours == 24) { + return "day"; + } + if (hours == 24 * 7) { + return "week"; + } + if (hours == 30.5 * 24 * 7) { + return "month"; + } + if (hours == 12 * 30.5 * 24 * 7) { + return "year"; + } + return `${hours} hours`; +} + interface BaseConfiguration { // image: name of the image to use, e.g. 'python' or 'pytorch'. // images are managed in src/packages/server/compute/images.ts @@ -727,6 +743,7 @@ export interface ComputeServerUserInfo { update_purchase?: boolean; last_purchase_update?: Date; template?: ComputeServerTemplate; + spend?: number; } export interface ComputeServer extends ComputeServerUserInfo { @@ -778,6 +795,7 @@ Table({ project_specific_id: null, course_project_id: null, course_server_id: null, + spend: null, }, }, set: { @@ -962,6 +980,10 @@ Table({ type: "integer", desc: "If this compute server is a clone of an instructor server in a course, this is the id of that instructor server.", }, + spend: { + type: "number", + desc: "If configuration.spendLimit is enabled, then the spend during the current period gets recorded here every few minutes. This is useful to efficiently provide a UI element showing the current spend status. It is cleared whenever configuration.spendLimit is changed, to avoid confusion.", + }, }, }); diff --git a/src/packages/util/db-schema/purchases.ts b/src/packages/util/db-schema/purchases.ts index 2d742299e8..0b0715aea0 100644 --- a/src/packages/util/db-schema/purchases.ts +++ b/src/packages/util/db-schema/purchases.ts @@ -257,7 +257,7 @@ Table({ }, pending: { type: "boolean", - desc: "If true, then this transaction is considered pending, which means that for a few days it doesn't count against the user's quotas for the purposes of deciding whether or not a purchase is allowed. This is needed so we can charge a user for their subscriptions, then collect the money from them, without all of the running pay-as-you-go project upgrades suddenly breaking (etc.).", + desc: "**DEPRECATED** -- not used anywhere; do NOT use! If true, then this transaction is considered pending, which means that for a few days it doesn't count against the user's quotas for the purposes of deciding whether or not a purchase is allowed. This is needed so we can charge a user for their subscriptions, then collect the money from them, without all of the running pay-as-you-go project upgrades suddenly breaking (etc.).", }, cost_per_hour: { title: "Cost Per Hour",