From c3e2409469ed61b4cdca8851620af489f2d2c234 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Fri, 5 Jul 2024 17:22:27 +0200 Subject: [PATCH 01/20] hub: cleanup data of deleted projects --- .../database/postgres-server-queries.coffee | 6 +- .../database/postgres/delete-projects.ts | 87 ++++++++++++++++++- src/packages/database/postgres/types.ts | 5 ++ src/packages/hub/run/delete-projects.js | 1 + src/packages/util/compute-states.ts | 13 ++- 5 files changed, 107 insertions(+), 5 deletions(-) diff --git a/src/packages/database/postgres-server-queries.coffee b/src/packages/database/postgres-server-queries.coffee index f2ac65cf35..ce739226c8 100644 --- a/src/packages/database/postgres-server-queries.coffee +++ b/src/packages/database/postgres-server-queries.coffee @@ -51,7 +51,7 @@ read = require('read') {site_license_manager_set} = require('./postgres/site-license/manager') {matching_site_licenses, manager_site_licenses} = require('./postgres/site-license/search') {project_datastore_set, project_datastore_get, project_datastore_del} = require('./postgres/project-queries') -{permanently_unlink_all_deleted_projects_of_user, unlink_old_deleted_projects} = require('./postgres/delete-projects') +{permanently_unlink_all_deleted_projects_of_user, unlink_old_deleted_projects, cleanup_old_projects_data} = require('./postgres/delete-projects') {get_all_public_paths, unlist_all_public_paths} = require('./postgres/public-paths') {get_personal_user} = require('./postgres/personal') {set_passport_settings, get_passport_settings, get_all_passport_settings, get_all_passport_settings_cached, create_passport, passport_exists, update_account_and_passport, _passport_key} = require('./postgres/passport') @@ -2590,6 +2590,10 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext unlink_old_deleted_projects: () => return await unlink_old_deleted_projects(@) + # async function + cleanup_old_projects_data: () => + return await cleanup_old_projects_data(@) + # async function unlist_all_public_paths: (account_id, is_owner) => return await unlist_all_public_paths(@, account_id, is_owner) diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index d07da9c396..8e88c8bf66 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -7,8 +7,15 @@ Code related to permanently deleting projects. */ +import getLogger from "@cocalc/backend/logger"; +import getPool from "@cocalc/database/pool"; import { callback2 } from "@cocalc/util/async-utils"; import { PostgreSQL } from "./types"; +import { minutes_ago } from "@cocalc/util/misc"; +import { getServerSettings } from "@cocalc/database/settings"; +import { KUCALC_ON_PREMISES } from "@cocalc/util/db-schema/site-defaults"; + +const log = getLogger("db:delete-projects"); /* Permanently delete from the database all project records, where the @@ -20,7 +27,7 @@ later then purges these projects from disk as well as the database. */ export async function permanently_unlink_all_deleted_projects_of_user( db: PostgreSQL, - account_id_or_email_address: string + account_id_or_email_address: string, ): Promise { // Get the account_id if necessary. const account_id = await get_account_id(db, account_id_or_email_address); @@ -36,7 +43,7 @@ export async function permanently_unlink_all_deleted_projects_of_user( async function get_account_id( db: PostgreSQL, - account_id_or_email_address: string + account_id_or_email_address: string, ): Promise { if (account_id_or_email_address.indexOf("@") == -1) { return account_id_or_email_address; @@ -57,7 +64,7 @@ Another task has to run to actually get rid of the data, etc. */ export async function unlink_old_deleted_projects( db: PostgreSQL, - age_d = 30 + age_d = 30, ): Promise { await callback2(db._query, { query: "UPDATE projects", @@ -69,3 +76,77 @@ export async function unlink_old_deleted_projects( ], }); } + +const Q_CLEANUP_SYNCSTRINGS = ` +SELECT p.project_id, s.string_id +FROM projects as p + INNER JOIN syncstrings as s + ON p.project_id = s.project_id +WHERE p.deleted = true + AND p.state ->> 'state' != 'deleted' +`; + +/* + This is more thorough than the above. It issues actual delete operations on data of projects marked as deleted. + When done, it sets the state.state to "deleted". + + The operations involves deleting all syncstrings of that project (and associated with that, patches), + and only for on-prem setups, it also deletes all the data stored in the project on disk. + + This function is called every couple of hours. Hence ensure it does not run longer than the given max_run_m time (minutes) +*/ +export async function cleanup_old_projects_data( + db: PostgreSQL, + delay_ms = 50, + max_run_m = 60, +) { + const settings = await getServerSettings(); + const on_prem = settings.kucalc === KUCALC_ON_PREMISES; + + log.debug("cleanup_old_projects_data", { delay_ms, max_run_m, on_prem }); + const start_ts = new Date(); + + const pool = getPool(); + const { rows } = await pool.query(Q_CLEANUP_SYNCSTRINGS); + + let num = 0; + let pid = ""; + + for (const row of rows) { + const { project_id, string_id } = row; + if (start_ts < minutes_ago(max_run_m)) { + log.debug( + `cleanup_old_projects_data: too much time elapsed, breaking after ${num} syncstrings`, + ); + break; + } + + log.debug( + `cleanup_old_projects_data: deleting syncstring ${project_id}/${string_id}`, + ); + num += 1; + await callback2(db.delete_syncstring, { string_id }); + + // wait for the given amount of delay_ms millio seconds + await new Promise((done) => setTimeout(done, delay_ms)); + + if (pid != project_id) { + pid = project_id; + if (on_prem) { + log.debug( + `cleanup_old_projects_data: deleting project data in ${project_id}`, + ); + // TODO: this only works on-prem, and requires the project files to be mounted + + log.debug(`deleting all shared files in project ${project_id}`); + // TODO: do it directly like above, and also get rid of all those shares in the database + } + + // now, that we're done with that project, mark it as state.state ->> 'deleted' + await callback2(db.set_project_state, { + project_id, + state: "deleted", + }); + } + } +} diff --git a/src/packages/database/postgres/types.ts b/src/packages/database/postgres/types.ts index 38b4c0f14f..24b45caf9d 100644 --- a/src/packages/database/postgres/types.ts +++ b/src/packages/database/postgres/types.ts @@ -7,6 +7,7 @@ import { EventEmitter } from "events"; import { Client } from "pg"; import { PassportStrategyDB } from "@cocalc/database/settings/auth-sso-types"; +import { ProjectState } from "@cocalc/util/db-schema/projects"; import { CB, CBDB, @@ -305,6 +306,8 @@ export interface PostgreSQL extends EventEmitter { cb: CB; }); + delete_syncstring(opts: { string_id: string; cb: CB }); + projects_that_need_to_be_started(): Promise; is_connected(): boolean; @@ -316,4 +319,6 @@ export interface PostgreSQL extends EventEmitter { email_address: string; }>; }): Promise; + + set_project_state(opts: { project_id: string; state: ProjectState["state"] }); } diff --git a/src/packages/hub/run/delete-projects.js b/src/packages/hub/run/delete-projects.js index 0b0d9e06e8..dbef215ed9 100755 --- a/src/packages/hub/run/delete-projects.js +++ b/src/packages/hub/run/delete-projects.js @@ -16,6 +16,7 @@ async function update() { console.log("unlinking old deleted projects..."); try { await db.unlink_old_deleted_projects(); + await db.cleanup_old_projects_data(); } catch (err) { if (err !== null) { throw Error(`failed to unlink projects -- ${err}`); diff --git a/src/packages/util/compute-states.ts b/src/packages/util/compute-states.ts index 022c11179f..d57f56c8d5 100644 --- a/src/packages/util/compute-states.ts +++ b/src/packages/util/compute-states.ts @@ -28,7 +28,8 @@ export type State = | "running" | "starting" | "stopping" - | "unarchiving"; + | "unarchiving" + | "deleted"; // @hsy: completely unclear what this is for. type Operation = @@ -218,4 +219,14 @@ export const COMPUTE_STATES: ComputeState = { "migrate_live", ], }, + + // projects are deleted in hub -> postgres.delete-projects and this is a one-way operation + deleted: { + desc: "Project is deleted", + icon: "trash", + display: "Deleted", + stable: true, + to: {}, + commands: [], + }, } as const; From b546a4d268e5560daea29e4c6b5bafe947041626 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Wed, 10 Jul 2024 12:56:50 +0200 Subject: [PATCH 02/20] npm: updating pg + @types/pg --- src/packages/database/package.json | 4 +- src/packages/next/package.json | 3 +- src/packages/pnpm-lock.yaml | 171 +++++++++++++++++------------ 3 files changed, 102 insertions(+), 76 deletions(-) diff --git a/src/packages/database/package.json b/src/packages/database/package.json index 7d45595763..6862fe52dd 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -23,7 +23,7 @@ "@cocalc/util": "workspace:*", "@qdrant/js-client-rest": "^1.6.0", "@types/lodash": "^4.14.202", - "@types/pg": "^8.6.1", + "@types/pg": "^8.11.6", "@types/uuid": "^8.3.1", "async": "^1.5.2", "awaiting": "^3.0.0", @@ -34,7 +34,7 @@ "lodash": "^4.17.21", "lru-cache": "^7.14.1", "node-fetch": "2.6.7", - "pg": "^8.7.1", + "pg": "^8.12.0", "random-key": "^0.3.2", "read": "^1.0.7", "sql-string-escape": "^1.1.6", diff --git a/src/packages/next/package.json b/src/packages/next/package.json index 97f9060418..73a2e1590e 100644 --- a/src/packages/next/package.json +++ b/src/packages/next/package.json @@ -63,6 +63,7 @@ "@cocalc/util": "workspace:*", "@openapitools/openapi-generator-cli": "^2.13.4", "@types/express": "^4.17.13", + "@types/pg": "8.11.6", "@types/react": "^18.0.26", "@types/react-dom": "^18.0.10", "@vscode/vscode-languagedetection": "^1.0.22", @@ -82,7 +83,7 @@ "next-remove-imports": "^1.0.11", "next-rest-framework": "6.0.0-beta.4", "password-hash": "^1.2.2", - "pg": "^8.7.1", + "pg": "^8.12.0", "react": "^18.2.0", "react-dom": "^18.2.0", "react-google-recaptcha": "^2.1.0", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index fa5c6a09d9..c79c989743 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -179,8 +179,8 @@ importers: specifier: ^4.14.202 version: 4.14.202 '@types/pg': - specifier: ^8.6.1 - version: 8.6.6 + specifier: ^8.11.6 + version: 8.11.6 '@types/uuid': specifier: ^8.3.1 version: 8.3.4 @@ -212,8 +212,8 @@ importers: specifier: 2.6.7 version: 2.6.7(encoding@0.1.13) pg: - specifier: ^8.7.1 - version: 8.8.0 + specifier: ^8.12.0 + version: 8.12.0 random-key: specifier: ^0.3.2 version: 0.3.2 @@ -839,7 +839,7 @@ importers: version: 2.1.2 next: specifier: 14.1.1 - version: 14.1.1(@babel/core@7.21.8)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(sass@1.77.6) + version: 14.1.1(@babel/core@7.24.5)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(sass@1.77.6) nyc: specifier: ^15.1.0 version: 15.1.0 @@ -1064,6 +1064,9 @@ importers: '@types/express': specifier: ^4.17.13 version: 4.17.15 + '@types/pg': + specifier: 8.11.6 + version: 8.11.6 '@types/react': specifier: ^18.0.26 version: 18.0.26 @@ -1122,8 +1125,8 @@ importers: specifier: ^1.2.2 version: 1.2.2 pg: - specifier: ^8.7.1 - version: 8.8.0 + specifier: ^8.12.0 + version: 8.12.0 react: specifier: ^18.2.0 version: 18.2.0 @@ -1380,7 +1383,7 @@ importers: version: 0.2.2(encoding@0.1.13)(langchain@0.2.3(axios@1.6.8)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(fast-xml-parser@4.4.0)(handlebars@4.7.8)(ignore@5.3.1)(openai@4.52.1(encoding@0.1.13))(ws@8.18.0))(openai@4.52.1(encoding@0.1.13)) '@langchain/community': specifier: ^0.2.14 - version: 0.2.14(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@google-cloud/storage@7.11.1(encoding@0.1.13))(@qdrant/js-client-rest@1.10.0(typescript@5.5.3))(axios@1.6.8)(better-sqlite3@8.3.0)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(fast-xml-parser@4.4.0)(google-auth-library@9.4.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ignore@5.3.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.52.1(encoding@0.1.13))(pg@8.12.0)(ws@8.18.0) + version: 0.2.14(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@google-cloud/storage@7.11.1(encoding@0.1.13))(@qdrant/js-client-rest@1.10.0(typescript@5.5.3))(axios@1.6.8)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(fast-xml-parser@4.4.0)(google-auth-library@9.4.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ignore@5.3.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.52.1(encoding@0.1.13))(pg@8.12.0)(ws@8.18.0) '@langchain/core': specifier: ^0.2.10 version: 0.2.10(langchain@0.2.3(axios@1.6.8)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(fast-xml-parser@4.4.0)(handlebars@4.7.8)(ignore@5.3.1)(openai@4.52.1(encoding@0.1.13))(ws@8.18.0))(openai@4.52.1(encoding@0.1.13)) @@ -2790,6 +2793,7 @@ packages: '@humanwhocodes/config-array@0.11.14': resolution: {integrity: sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==} engines: {node: '>=10.10.0'} + deprecated: Use @eslint/config-array instead '@humanwhocodes/module-importer@1.0.1': resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} @@ -2797,6 +2801,7 @@ packages: '@humanwhocodes/object-schema@2.0.2': resolution: {integrity: sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw==} + deprecated: Use @eslint/object-schema instead '@icons/material@0.2.4': resolution: {integrity: sha512-QPcGmICAPbGLGb6F/yNf/KzKqvFx8z5qx3D1yFqVAjoFmXK35EgyW+cJ57Te3CNsmzblwtzakLGFqHPqrfb4Tw==} @@ -4197,8 +4202,8 @@ packages: '@types/passport@1.0.11': resolution: {integrity: sha512-pz1cx9ptZvozyGKKKIPLcVDVHwae4hrH5d6g5J+DkMRRjR3cVETb4jMabhXAUbg3Ov7T22nFHEgaK2jj+5CBpw==} - '@types/pg@8.6.6': - resolution: {integrity: sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw==} + '@types/pg@8.11.6': + resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==} '@types/pica@5.1.3': resolution: {integrity: sha512-13SEyETRE5psd9bE0AmN+0M1tannde2fwHfLVaVIljkbL9V0OfFvKwCicyeDvVYLkmjQWEydbAlsDsmjrdyTOg==} @@ -4512,6 +4517,7 @@ packages: abab@2.0.6: resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==} + deprecated: Use your platform's native atob() and btoa() methods instead abbrev@1.1.1: resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} @@ -4992,6 +4998,7 @@ packages: bootstrap-colorpicker@2.5.3: resolution: {integrity: sha512-xdllX8LSMvKULs3b8JrgRXTvyvjkSMHHHVuHjjN5FNMqr6kRe5NPiMHFmeAFjlgDF73MspikudLuEwR28LbzLw==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. bootstrap@3.4.1: resolution: {integrity: sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA==} @@ -5050,10 +5057,6 @@ packages: buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} - buffer-writer@2.0.0: - resolution: {integrity: sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==} - engines: {node: '>=4'} - buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} @@ -6726,12 +6729,15 @@ packages: glob@6.0.4: resolution: {integrity: sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==} + deprecated: Glob versions prior to v9 are no longer supported glob@7.2.0: resolution: {integrity: sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==} + deprecated: Glob versions prior to v9 are no longer supported glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + deprecated: Glob versions prior to v9 are no longer supported global@4.4.0: resolution: {integrity: sha512-wv/LAoHdRE3BeTGz53FAamhGlPLhlssK45usmGFThIi4XqnBmjKQ16u+RNbP7WvigRZDxUsM0J3gcQ5yicaL0w==} @@ -7166,6 +7172,7 @@ packages: inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. inherits@2.0.3: resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} @@ -8100,6 +8107,7 @@ packages: ldapjs@2.3.3: resolution: {integrity: sha512-75QiiLJV/PQqtpH+HGls44dXweviFwQ6SiIK27EqzKQ5jU/7UFrl2E5nLdQ3IYRBzJ/AVFJI66u0MZ0uofKYwg==} engines: {node: '>=10.13.0'} + deprecated: This package has been decomissioned. See https://github.com/ldapjs/node-ldapjs/blob/8ffd0bc9c149088a10ec4c1ec6a18450f76ad05d/README.md lean-client-js-core@1.5.0: resolution: {integrity: sha512-1uQ+ldW85sMW1zvwab424qGM1bDwJ2d51qcUt5bxWPIp9Zef7pcqRhYytAvxj1G1+4+gZhUzbyHS7wfh7bRrcg==} @@ -8813,6 +8821,9 @@ packages: resolution: {integrity: sha512-FVVTkD1vENCsAcwNs9k6jea2uHC/X0+JcjG8YA60FN5CMaJmG95wT9jek/xX9nornqGRrBkKtzuAu2wuHpKqvw==} engines: {node: '>= 0.4'} + obuf@1.1.2: + resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + octicons@3.5.0: resolution: {integrity: sha512-jIjd+/oT46YgOK2SZbicD8vIGkinUwpx7HRm6okT3dU5fZQO/sEbMOKSfLxLhJIuI2KRlylN9nYfKiuM4uf+gA==} @@ -8913,9 +8924,6 @@ packages: resolution: {integrity: sha512-whdkPIooSu/bASggZ96BWVvZTRMOFxnyUG5PnTSGKoJE2gd5mbVNmR2Nj20QFzxYYgAXpoqC+AiXzl+UMRh7zQ==} engines: {node: '>=8'} - packet-reader@1.0.0: - resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} - pako@2.1.0: resolution: {integrity: sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==} @@ -8984,6 +8992,7 @@ packages: passport-azure-ad@4.3.5: resolution: {integrity: sha512-LBpXEght7hCMuMNFK4oegdN0uPBa3lpDMy71zQoB0zPg1RrGwdzpjwTiN1WzN0hY77fLyjz9tBr3TGAxnSgtEg==} engines: {node: '>= 8.0.0'} + deprecated: This package is deprecated and no longer supported. For more please visit https://github.com/AzureAD/passport-azure-ad?tab=readme-ov-file#node-js-validation-replacement-for-passportjs passport-facebook@3.0.0: resolution: {integrity: sha512-K/qNzuFsFISYAyC1Nma4qgY/12V3RSLFdFVsPKXiKZt434wOvthFW1p7zKa1iQihQMRhaWorVE1o3Vi1o+ZgeQ==} @@ -9109,9 +9118,6 @@ packages: pg-cloudflare@1.1.1: resolution: {integrity: sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==} - pg-connection-string@2.5.0: - resolution: {integrity: sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==} - pg-connection-string@2.6.4: resolution: {integrity: sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==} @@ -9119,19 +9125,15 @@ packages: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} engines: {node: '>=4.0.0'} - pg-pool@3.5.2: - resolution: {integrity: sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==} - peerDependencies: - pg: '>=8.0' + pg-numeric@1.0.2: + resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} + engines: {node: '>=4'} pg-pool@3.6.2: resolution: {integrity: sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==} peerDependencies: pg: '>=8.0' - pg-protocol@1.5.0: - resolution: {integrity: sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ==} - pg-protocol@1.6.1: resolution: {integrity: sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==} @@ -9139,6 +9141,10 @@ packages: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} engines: {node: '>=4'} + pg-types@4.0.2: + resolution: {integrity: sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==} + engines: {node: '>=10'} + pg@8.12.0: resolution: {integrity: sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==} engines: {node: '>= 8.0.0'} @@ -9148,15 +9154,6 @@ packages: pg-native: optional: true - pg@8.8.0: - resolution: {integrity: sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw==} - engines: {node: '>= 8.0.0'} - peerDependencies: - pg-native: '>=3.0.1' - peerDependenciesMeta: - pg-native: - optional: true - pgpass@1.0.5: resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} @@ -9267,18 +9264,37 @@ packages: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} engines: {node: '>=4'} + postgres-array@3.0.2: + resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==} + engines: {node: '>=12'} + postgres-bytea@1.0.0: resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} engines: {node: '>=0.10.0'} + postgres-bytea@3.0.0: + resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} + engines: {node: '>= 6'} + postgres-date@1.0.7: resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} engines: {node: '>=0.10.0'} + postgres-date@2.1.0: + resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==} + engines: {node: '>=12'} + postgres-interval@1.2.0: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} + postgres-interval@3.0.0: + resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} + engines: {node: '>=12'} + + postgres-range@1.1.4: + resolution: {integrity: sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==} + potpack@1.0.2: resolution: {integrity: sha512-choctRBIV9EMT9WGAZHn3V7t0Z2pMQyl0EZE6pFc/6ml3ssw7Dlf/oAOvFwjm1HVsqfQN8GfeFyJ+d8tRzqueQ==} @@ -9349,6 +9365,7 @@ packages: project-name-generator@2.1.9: resolution: {integrity: sha512-QmLHqz2C4VHmAyDEAFlVfnuWAHr4vwZhK2bbm4IrwuHNzNKOdG9b4U+NmQbsm1uOoV4kGWv7+FVLsu7Bb/ieYQ==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. hasBin: true prom-client@13.2.0: @@ -10143,18 +10160,22 @@ packages: rimraf@2.4.5: resolution: {integrity: sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true rimraf@2.6.3: resolution: {integrity: sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true rimraf@2.7.1: resolution: {integrity: sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true rimraf@3.0.2: resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true rimraf@5.0.5: @@ -10421,6 +10442,7 @@ packages: sinon@4.5.0: resolution: {integrity: sha512-trdx+mB0VBBgoYucy6a9L7/jfQOmvGeaKZT4OOJ+lPAtI8623xyGr8wLiE4eojzBS8G9yXbhx42GHUOVLr4X2w==} + deprecated: 16.1.1 sirv@1.0.19: resolution: {integrity: sha512-JuLThK3TnZG1TAKDwNIqNq6QA2afLOCcm+iE8D1Kj3GA40pSPsxQjjJl0J8X3tsR7T+CP1GavpzLwYkgVLWrZQ==} @@ -11623,21 +11645,25 @@ packages: xterm-addon-fit@0.6.0: resolution: {integrity: sha512-9/7A+1KEjkFam0yxTaHfuk9LEvvTSBi0PZmEkzJqgafXPEXL9pCMAVV7rB09sX6ATRDXAdBpQhZkhKj7CGvYeg==} + deprecated: This package is now deprecated. Move to @xterm/addon-fit instead. peerDependencies: xterm: ^5.0.0 xterm-addon-web-links@0.7.0: resolution: {integrity: sha512-6PqoqzzPwaeSq22skzbvyboDvSnYk5teUYEoKBwMYvhbkwOQkemZccjWHT5FnNA8o1aInTc4PRYAl4jjPucCKA==} + deprecated: This package is now deprecated. Move to @xterm/addon-web-links instead. peerDependencies: xterm: ^5.0.0 xterm-addon-webgl@0.13.0: resolution: {integrity: sha512-xL4qBQWUHjFR620/8VHCtrTMVQsnZaAtd1IxFoiKPhC63wKp6b+73a45s97lb34yeo57PoqZhE9Jq5pB++ksPQ==} + deprecated: This package is now deprecated. Move to @xterm/addon-webgl instead. peerDependencies: xterm: ^5.0.0 xterm@5.0.0: resolution: {integrity: sha512-tmVsKzZovAYNDIaUinfz+VDclraQpPUnAME+JawosgWRMphInDded/PuY0xmU5dOhyeYZsI0nz5yd8dPYsdLTA==} + deprecated: This package is now deprecated. Move to @xterm/xterm instead. xxhashjs@0.2.2: resolution: {integrity: sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==} @@ -13175,7 +13201,7 @@ snapshots: - langchain - openai - '@langchain/community@0.2.14(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@google-cloud/storage@7.11.1(encoding@0.1.13))(@qdrant/js-client-rest@1.10.0(typescript@5.5.3))(axios@1.6.8)(better-sqlite3@8.3.0)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(fast-xml-parser@4.4.0)(google-auth-library@9.4.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ignore@5.3.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.52.1(encoding@0.1.13))(pg@8.12.0)(ws@8.18.0)': + '@langchain/community@0.2.14(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@google-cloud/storage@7.11.1(encoding@0.1.13))(@qdrant/js-client-rest@1.10.0(typescript@5.5.3))(axios@1.6.8)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(fast-xml-parser@4.4.0)(google-auth-library@9.4.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ignore@5.3.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.52.1(encoding@0.1.13))(pg@8.12.0)(ws@8.18.0)': dependencies: '@langchain/core': 0.2.10(langchain@0.2.3(axios@1.6.8)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(fast-xml-parser@4.4.0)(handlebars@4.7.8)(ignore@5.3.1)(openai@4.52.1(encoding@0.1.13))(ws@8.18.0))(openai@4.52.1(encoding@0.1.13)) '@langchain/openai': 0.1.0(encoding@0.1.13)(langchain@0.2.3(axios@1.6.8)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(fast-xml-parser@4.4.0)(handlebars@4.7.8)(ignore@5.3.1)(openai@4.52.1(encoding@0.1.13))(ws@8.18.0)) @@ -13192,7 +13218,6 @@ snapshots: '@google-ai/generativelanguage': 2.6.0(encoding@0.1.13) '@google-cloud/storage': 7.11.1(encoding@0.1.13) '@qdrant/js-client-rest': 1.10.0(typescript@5.5.3) - better-sqlite3: 8.3.0 cheerio: 1.0.0-rc.12 google-auth-library: 9.4.1(encoding@0.1.13) googleapis: 137.1.0(encoding@0.1.13) @@ -14262,11 +14287,11 @@ snapshots: dependencies: '@types/express': 4.17.15 - '@types/pg@8.6.6': + '@types/pg@8.11.6': dependencies: '@types/node': 18.19.31 - pg-protocol: 1.5.0 - pg-types: 2.2.0 + pg-protocol: 1.6.1 + pg-types: 4.0.2 '@types/pica@5.1.3': {} @@ -15311,8 +15336,6 @@ snapshots: buffer-from@1.1.2: {} - buffer-writer@2.0.0: {} - buffer@5.7.1: dependencies: base64-js: 1.5.1 @@ -19601,7 +19624,7 @@ snapshots: - '@babel/core' - babel-plugin-macros - next@14.1.1(@babel/core@7.21.8)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(sass@1.77.6): + next@14.1.1(@babel/core@7.24.5)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(sass@1.77.6): dependencies: '@next/env': 14.1.1 '@swc/helpers': 0.5.2 @@ -19611,7 +19634,7 @@ snapshots: postcss: 8.4.31 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - styled-jsx: 5.1.1(@babel/core@7.21.8)(react@18.2.0) + styled-jsx: 5.1.1(@babel/core@7.24.5)(react@18.2.0) optionalDependencies: '@next/swc-darwin-arm64': 14.1.1 '@next/swc-darwin-x64': 14.1.1 @@ -19888,6 +19911,8 @@ snapshots: define-properties: 1.1.4 es-abstract: 1.20.5 + obuf@1.1.2: {} + octicons@3.5.0: {} on-finished@2.4.1: @@ -20001,8 +20026,6 @@ snapshots: lodash.flattendeep: 4.4.0 release-zalgo: 1.0.0 - packet-reader@1.0.0: {} - pako@2.1.0: {} param-case@3.0.4: @@ -20218,26 +20241,17 @@ snapshots: pg-cloudflare@1.1.1: optional: true - pg-connection-string@2.5.0: {} - - pg-connection-string@2.6.4: - optional: true + pg-connection-string@2.6.4: {} pg-int8@1.0.1: {} - pg-pool@3.5.2(pg@8.8.0): - dependencies: - pg: 8.8.0 + pg-numeric@1.0.2: {} pg-pool@3.6.2(pg@8.12.0): dependencies: pg: 8.12.0 - optional: true - - pg-protocol@1.5.0: {} - pg-protocol@1.6.1: - optional: true + pg-protocol@1.6.1: {} pg-types@2.2.0: dependencies: @@ -20247,6 +20261,16 @@ snapshots: postgres-date: 1.0.7 postgres-interval: 1.2.0 + pg-types@4.0.2: + dependencies: + pg-int8: 1.0.1 + pg-numeric: 1.0.2 + postgres-array: 3.0.2 + postgres-bytea: 3.0.0 + postgres-date: 2.1.0 + postgres-interval: 3.0.0 + postgres-range: 1.1.4 + pg@8.12.0: dependencies: pg-connection-string: 2.6.4 @@ -20256,17 +20280,6 @@ snapshots: pgpass: 1.0.5 optionalDependencies: pg-cloudflare: 1.1.1 - optional: true - - pg@8.8.0: - dependencies: - buffer-writer: 2.0.0 - packet-reader: 1.0.0 - pg-connection-string: 2.5.0 - pg-pool: 3.5.2(pg@8.8.0) - pg-protocol: 1.5.0 - pg-types: 2.2.0 - pgpass: 1.0.5 pgpass@1.0.5: dependencies: @@ -20418,14 +20431,26 @@ snapshots: postgres-array@2.0.0: {} + postgres-array@3.0.2: {} + postgres-bytea@1.0.0: {} + postgres-bytea@3.0.0: + dependencies: + obuf: 1.1.2 + postgres-date@1.0.7: {} + postgres-date@2.1.0: {} + postgres-interval@1.2.0: dependencies: xtend: 4.0.2 + postgres-interval@3.0.0: {} + + postgres-range@1.1.4: {} + potpack@1.0.2: {} potpack@2.0.0: {} @@ -20577,7 +20602,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 18.16.14 + '@types/node': 18.19.31 long: 5.2.3 protocol-buffers-schema@3.6.0: {} @@ -22133,12 +22158,12 @@ snapshots: optionalDependencies: '@babel/core': 7.21.4 - styled-jsx@5.1.1(@babel/core@7.21.8)(react@18.2.0): + styled-jsx@5.1.1(@babel/core@7.24.5)(react@18.2.0): dependencies: client-only: 0.0.1 react: 18.2.0 optionalDependencies: - '@babel/core': 7.21.8 + '@babel/core': 7.24.5 stylis@4.3.0: {} From 3bcabebc7cf5e212cd1ddeaa63349662d0c87f34 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Wed, 10 Jul 2024 13:26:41 +0200 Subject: [PATCH 03/20] database/cleanup: WIP bulk delete --- .../database/postgres/bulk-delete.test.ts | 49 +++++++++++++ src/packages/database/postgres/bulk-delete.ts | 73 +++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 src/packages/database/postgres/bulk-delete.test.ts create mode 100644 src/packages/database/postgres/bulk-delete.ts diff --git a/src/packages/database/postgres/bulk-delete.test.ts b/src/packages/database/postgres/bulk-delete.test.ts new file mode 100644 index 0000000000..eb523d7df4 --- /dev/null +++ b/src/packages/database/postgres/bulk-delete.test.ts @@ -0,0 +1,49 @@ +/* + * This file is part of CoCalc: Copyright © 2024 Sagemath, Inc. + * License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details + */ + +import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; +import { uuid } from "@cocalc/util/misc"; +import { bulk_delete } from "./bulk-delete"; + +beforeAll(async () => { + await initEphemeralDatabase(); +}, 15000); + +afterAll(async () => { + await getPool().end(); +}); + +describe("bulk delete", () => { + test("deleting projects", async () => { + const p = getPool(); + const project_id = uuid(); + const N = 2000; + for (let i = 0; i < N; i++) { + await p.query( + "INSERT INTO project_log (id, project_id, time) VALUES($1::UUID, $2::UUID, $3::TIMESTAMP)", + [uuid(), project_id, new Date()], + ); + } + + const num1 = await p.query( + "SELECT COUNT(*)::INT as num FROM project_log WHERE project_id = $1", + [project_id], + ); + expect(num1.rows[0].num).toEqual(N); + + await bulk_delete({ + table: "project_log", + field: "project_id", + value: project_id, + limit: 100, + }); + + const num2 = await p.query( + "SELECT COUNT(*)::INT as num FROM project_log WHERE project_id = $1", + [project_id], + ); + expect(num2.rows[0].num).toEqual(0); + }); +}); diff --git a/src/packages/database/postgres/bulk-delete.ts b/src/packages/database/postgres/bulk-delete.ts new file mode 100644 index 0000000000..28677e366b --- /dev/null +++ b/src/packages/database/postgres/bulk-delete.ts @@ -0,0 +1,73 @@ +import { escapeIdentifier } from "pg"; + +import getPool from "@cocalc/database/pool"; +import { SCHEMA } from "@cocalc/util/schema"; + +interface Opts { + table: string; + field: "project_id" | "account_id"; // for now, we only support a few + value: string; // a UUID + limit?: number; +} + +type Ret = Promise<{ + rowsDeleted: number; + durationS: number; +}>; + +function deleteQuery(table: string, field: string) { + const T = escapeIdentifier(table); + const F = escapeIdentifier(field); + + return ` +DELETE FROM ${T} +WHERE ${F} IN ( + SELECT ${F} FROM ${T} WHERE ${F} = $1 LIMIT $2 +) +RETURNING 1 +`; +} + +export async function bulk_delete(opts: Opts): Ret { + const { table, field, value } = opts; + let { limit = 1000 } = opts; + // assert table name is a key in SCHEMA + if (!(table in SCHEMA)) { + throw new Error(`table ${table} does not exist`); + } + + const q = deleteQuery(table, field); + console.log(q); + console.log(opts); + + const pool = getPool(); + + const start_ts = Date.now(); + let rowsDeleted = 0; + + while (true) { + const t0 = Date.now(); + const ret = await pool.query(q, [value, limit]); + const td = Date.now() - t0; + rowsDeleted += ret.rowCount ?? 0; + + // adjust the limit + const next = Math.round( + td > 0.1 ? limit / 2 : td < 0.05 ? limit * 2 : limit, + ); + limit = Math.max(1, Math.min(10000, next)); + + // wait for a bit, but not more than 1 second ~ this aims for a max utilization of 10% + const wait_ms = Math.min(1000, td * 10); + await new Promise((done) => setTimeout(done, wait_ms)); + + console.log( + `loop: deleted ${ret.rowCount} | wait=${wait_ms} | limit=${limit}`, + ); + + if (ret.rowCount === 0) break; + } + + const durationS = (Date.now() - start_ts) / 1000; + return { durationS, rowsDeleted }; +} From af28b7f5a8911003ace9cd32c6c9ec7452b67655 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Wed, 10 Jul 2024 15:18:29 +0200 Subject: [PATCH 04/20] fix fallout of b546a4d268e5560 --- src/packages/next/package.json | 2 +- src/packages/pnpm-lock.yaml | 2 +- src/packages/server/shopping/cart/add.ts | 18 +++++++++--------- src/packages/server/shopping/cart/checked.ts | 4 ++-- src/packages/server/shopping/cart/delete.ts | 6 +++--- src/packages/server/shopping/cart/edit.ts | 2 +- .../server/shopping/cart/recent-purchases.ts | 2 +- src/packages/server/shopping/cart/remove.ts | 7 +++---- 8 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/packages/next/package.json b/src/packages/next/package.json index 73a2e1590e..5761b4b24a 100644 --- a/src/packages/next/package.json +++ b/src/packages/next/package.json @@ -63,7 +63,7 @@ "@cocalc/util": "workspace:*", "@openapitools/openapi-generator-cli": "^2.13.4", "@types/express": "^4.17.13", - "@types/pg": "8.11.6", + "@types/pg": "^8.11.6", "@types/react": "^18.0.26", "@types/react-dom": "^18.0.10", "@vscode/vscode-languagedetection": "^1.0.22", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index c79c989743..edc56f1c97 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1065,7 +1065,7 @@ importers: specifier: ^4.17.13 version: 4.17.15 '@types/pg': - specifier: 8.11.6 + specifier: ^8.11.6 version: 8.11.6 '@types/react': specifier: ^18.0.26 diff --git a/src/packages/server/shopping/cart/add.ts b/src/packages/server/shopping/cart/add.ts index ffb223f87f..a58b7614ea 100644 --- a/src/packages/server/shopping/cart/add.ts +++ b/src/packages/server/shopping/cart/add.ts @@ -13,19 +13,19 @@ checking periodically, then blacklisting...? This isn't something of any value to a spammer so it's very unlikely to be exploited maliciously. */ -import { isValidUUID } from "@cocalc/util/misc"; import getPool from "@cocalc/database/pool"; import { - ProductType, ProductDescription, + ProductType, } from "@cocalc/util/db-schema/shopping-cart-items"; +import { isValidUUID } from "@cocalc/util/misc"; import { getItem } from "./get"; export default async function addToCart( account_id: string, product: ProductType, description?: ProductDescription, - project_id?: string + project_id?: string, ): Promise { if (!isValidUUID(account_id)) { throw Error("account_id is invalid"); @@ -37,16 +37,16 @@ export default async function addToCart( const { rowCount } = await pool.query( `INSERT INTO shopping_cart_items (account_id, added, product, description, checked, project_id) VALUES($1, NOW(), $2, $3, true, $4)`, - [account_id, product, description, project_id] + [account_id, product, description, project_id], ); - return rowCount; + return rowCount ?? 0; } // Puts an item back in the cart that was removed. // - Mutates item that was actually removed and not purchased. export async function putBackInCart( account_id: string, - id: number + id: number, ): Promise { if (!isValidUUID(account_id)) { throw Error("account_id is invalid"); @@ -54,15 +54,15 @@ export async function putBackInCart( const pool = getPool(); const { rowCount } = await pool.query( "UPDATE shopping_cart_items SET removed=NULL, checked=TRUE WHERE account_id=$1 AND id=$2 AND removed IS NOT NULL AND purchased IS NULL", - [account_id, id] + [account_id, id], ); - return rowCount; + return rowCount ?? 0; } // Makes copy of item that was purchased and puts it in the cart. export async function buyItAgain( account_id: string, - id: number + id: number, ): Promise { if (!isValidUUID(account_id)) { throw Error("account_id is invalid"); diff --git a/src/packages/server/shopping/cart/checked.ts b/src/packages/server/shopping/cart/checked.ts index d76c42d0fc..c5140eab9d 100644 --- a/src/packages/server/shopping/cart/checked.ts +++ b/src/packages/server/shopping/cart/checked.ts @@ -12,7 +12,7 @@ import getPool from "@cocalc/database/pool"; export default async function setCheck( account_id: string, checked: boolean, - id?: number + id?: number, ): Promise { if (!isValidUUID(account_id)) { throw Error("account_id is invalid"); @@ -28,5 +28,5 @@ export default async function setCheck( } const { rowCount } = await pool.query(query, params); - return rowCount; + return rowCount ?? 0; } diff --git a/src/packages/server/shopping/cart/delete.ts b/src/packages/server/shopping/cart/delete.ts index 16cb4b9529..502b33c108 100644 --- a/src/packages/server/shopping/cart/delete.ts +++ b/src/packages/server/shopping/cart/delete.ts @@ -16,7 +16,7 @@ import getPool from "@cocalc/database/pool"; // Returns number of items "deleted". export default async function deleteItem( account_id: string, - id: number + id: number, ): Promise { if (!isValidUUID(account_id)) { throw Error("account_id is invalid"); @@ -24,7 +24,7 @@ export default async function deleteItem( const pool = getPool(); const { rowCount } = await pool.query( "DELETE FROM shopping_cart_items WHERE account_id=$1 AND id=$2 AND purchased IS NULL", - [account_id, id] + [account_id, id], ); - return rowCount; + return rowCount ?? 0; } diff --git a/src/packages/server/shopping/cart/edit.ts b/src/packages/server/shopping/cart/edit.ts index 46e79b903a..bb4c22dfb7 100644 --- a/src/packages/server/shopping/cart/edit.ts +++ b/src/packages/server/shopping/cart/edit.ts @@ -47,5 +47,5 @@ export default async function editCart({ query += " AND purchased IS NULL"; const { rowCount } = await pool.query(query, params); - return rowCount; + return rowCount ?? 0; } diff --git a/src/packages/server/shopping/cart/recent-purchases.ts b/src/packages/server/shopping/cart/recent-purchases.ts index f89994d65d..d0b3a7705f 100644 --- a/src/packages/server/shopping/cart/recent-purchases.ts +++ b/src/packages/server/shopping/cart/recent-purchases.ts @@ -29,7 +29,7 @@ export default async function getRecentPurchases({ const pool = getPool(); const { rows } = await pool.query( `SELECT * FROM shopping_cart_items WHERE account_id=$1 AND purchased IS NOT NULL AND (purchased#>>'{time}')::timestamptz >= NOW() - $2::interval AND purchased#>>'{voucher_id}' IS NULL`, - [account_id, recent ?? "1 week"] + [account_id, recent ?? "1 week"], ); rows.sort((a, b) => -cmp(a.purchased?.time, b.purchased?.time)); return rows; diff --git a/src/packages/server/shopping/cart/remove.ts b/src/packages/server/shopping/cart/remove.ts index f441b48a53..d36c89f5f2 100644 --- a/src/packages/server/shopping/cart/remove.ts +++ b/src/packages/server/shopping/cart/remove.ts @@ -15,7 +15,7 @@ import getPool from "@cocalc/database/pool"; // You can't remove an item more than once from a cart. export default async function removeFromCart( account_id: string, - id: number + id: number, ): Promise { if (!isValidUUID(account_id)) { throw Error("account_id is invalid"); @@ -23,8 +23,7 @@ export default async function removeFromCart( const pool = getPool(); const { rowCount } = await pool.query( "UPDATE shopping_cart_items SET removed=NOW() WHERE account_id=$1 AND id=$2 AND removed IS NULL AND purchased IS NULL", - [account_id, id] + [account_id, id], ); - return rowCount; + return rowCount ?? 0; } - From 4126e2c50f7de1f2f606982ae2c07e0531b524b2 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Wed, 10 Jul 2024 16:14:13 +0200 Subject: [PATCH 05/20] database/bulk-delete: delete many rows without overwhelming the DB --- .../database/postgres/bulk-delete.test.ts | 28 ++++++++- src/packages/database/postgres/bulk-delete.ts | 62 +++++++++++-------- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/src/packages/database/postgres/bulk-delete.test.ts b/src/packages/database/postgres/bulk-delete.test.ts index eb523d7df4..423c10b009 100644 --- a/src/packages/database/postgres/bulk-delete.test.ts +++ b/src/packages/database/postgres/bulk-delete.test.ts @@ -20,6 +20,14 @@ describe("bulk delete", () => { const p = getPool(); const project_id = uuid(); const N = 2000; + + // extra entry, which has to remain + const other = uuid(); + await p.query( + "INSERT INTO project_log (id, project_id, time) VALUES($1::UUID, $2::UUID, $3::TIMESTAMP)", + [other, uuid(), new Date()], + ); + for (let i = 0; i < N; i++) { await p.query( "INSERT INTO project_log (id, project_id, time) VALUES($1::UUID, $2::UUID, $3::TIMESTAMP)", @@ -33,17 +41,31 @@ describe("bulk delete", () => { ); expect(num1.rows[0].num).toEqual(N); - await bulk_delete({ + const res = await bulk_delete({ table: "project_log", field: "project_id", value: project_id, - limit: 100, + limit: 128, }); + // if this ever fails, the "ret.rowCount" value is inaccurate. + // This must be replaced by "RETURNING 1" in the the query and a "SELECT COUNT(*) ..." and so. + // (and not only here, but everywhere in the code base) + expect(res.rowsDeleted).toEqual(N); + expect(res.durationS).toBeGreaterThan(0.01); + expect(res.totalPgTimeS).toBeGreaterThan(0.001); + expect(res.totalWaitS).toBeGreaterThan(0.001); + expect((res.totalPgTimeS * 10) / res.totalWaitS).toBeGreaterThan(0.5); + const num2 = await p.query( "SELECT COUNT(*)::INT as num FROM project_log WHERE project_id = $1", [project_id], ); expect(num2.rows[0].num).toEqual(0); - }); + + const otherRes = await p.query("SELECT * FROM project_log WHERE id = $1", [ + other, + ]); + expect(otherRes.rows[0].id).toEqual(other); + }, 10000); }); diff --git a/src/packages/database/postgres/bulk-delete.ts b/src/packages/database/postgres/bulk-delete.ts index 28677e366b..b6b38490a4 100644 --- a/src/packages/database/postgres/bulk-delete.ts +++ b/src/packages/database/postgres/bulk-delete.ts @@ -1,73 +1,81 @@ +// see packages/database/pool/pool.ts for where this name is also hard coded: +process.env.PGDATABASE = "smc_ephemeral_testing_database"; + import { escapeIdentifier } from "pg"; import getPool from "@cocalc/database/pool"; import { SCHEMA } from "@cocalc/util/schema"; interface Opts { - table: string; + table: string; // e.g. project_log, etc. field: "project_id" | "account_id"; // for now, we only support a few + id?: string; // default "id", the ID field in the table, which identifies each row uniquely value: string; // a UUID - limit?: number; + limit?: number; // default 1024 + maxUtilPct?: number; // 0-100, percent } type Ret = Promise<{ rowsDeleted: number; durationS: number; + totalWaitS: number; + totalPgTimeS: number; }>; -function deleteQuery(table: string, field: string) { +function deleteQuery(table: string, field: string, id: string) { const T = escapeIdentifier(table); const F = escapeIdentifier(field); + const ID = escapeIdentifier(id); return ` DELETE FROM ${T} -WHERE ${F} IN ( - SELECT ${F} FROM ${T} WHERE ${F} = $1 LIMIT $2 -) -RETURNING 1 -`; +WHERE ${ID} IN ( + SELECT ${ID} FROM ${T} WHERE ${F} = $1 LIMIT $2 +)`; } export async function bulk_delete(opts: Opts): Ret { - const { table, field, value } = opts; - let { limit = 1000 } = opts; + const { table, field, value, id = "id", maxUtilPct = 10 } = opts; + let { limit = 1024 } = opts; // assert table name is a key in SCHEMA if (!(table in SCHEMA)) { throw new Error(`table ${table} does not exist`); } - const q = deleteQuery(table, field); - console.log(q); - console.log(opts); + if (maxUtilPct < 1 || maxUtilPct > 99) { + throw new Error(`maxUtilPct must be between 1 and 99`); + } + const q = deleteQuery(table, field, id); const pool = getPool(); - const start_ts = Date.now(); - let rowsDeleted = 0; + let rowsDeleted = 0; + let totalWaitS = 0; + let totalPgTimeS = 0; while (true) { const t0 = Date.now(); const ret = await pool.query(q, [value, limit]); - const td = Date.now() - t0; + const dt = (Date.now() - t0) / 1000; rowsDeleted += ret.rowCount ?? 0; + totalPgTimeS += dt; - // adjust the limit - const next = Math.round( - td > 0.1 ? limit / 2 : td < 0.05 ? limit * 2 : limit, - ); - limit = Math.max(1, Math.min(10000, next)); + // adjust the limit: we aim to keep the operation between 0.1 and 0.2 secs + const next = dt > 0.2 ? limit / 2 : dt < 0.1 ? limit * 2 : limit; + limit = Math.max(1, Math.min(32768, Math.round(next))); // wait for a bit, but not more than 1 second ~ this aims for a max utilization of 10% - const wait_ms = Math.min(1000, td * 10); - await new Promise((done) => setTimeout(done, wait_ms)); + const waitS = Math.min(1, dt * ((100 - maxUtilPct) / maxUtilPct)); + await new Promise((done) => setTimeout(done, 1000 * waitS)); + totalWaitS += waitS; - console.log( - `loop: deleted ${ret.rowCount} | wait=${wait_ms} | limit=${limit}`, - ); + // console.log( + // `deleted ${ret.rowCount} | dt=${dt} | wait=${waitS} | limit=${limit}`, + // ); if (ret.rowCount === 0) break; } const durationS = (Date.now() - start_ts) / 1000; - return { durationS, rowsDeleted }; + return { durationS, rowsDeleted, totalWaitS, totalPgTimeS }; } From 59131f72ec9b229f5543187bfd7cfe3399a8588e Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Wed, 10 Jul 2024 16:54:49 +0200 Subject: [PATCH 06/20] database/delete-projects: expand scope --- .../database/postgres-server-queries.coffee | 4 +- .../database/postgres/bulk-delete.test.ts | 5 +- src/packages/database/postgres/bulk-delete.ts | 11 +- .../database/postgres/delete-projects.ts | 114 ++++++++++++++---- src/packages/hub/run/delete-projects.js | 3 +- 5 files changed, 106 insertions(+), 31 deletions(-) diff --git a/src/packages/database/postgres-server-queries.coffee b/src/packages/database/postgres-server-queries.coffee index ce739226c8..aae46a7964 100644 --- a/src/packages/database/postgres-server-queries.coffee +++ b/src/packages/database/postgres-server-queries.coffee @@ -2591,8 +2591,8 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext return await unlink_old_deleted_projects(@) # async function - cleanup_old_projects_data: () => - return await cleanup_old_projects_data(@) + cleanup_old_projects_data: (max_run_m) => + return await cleanup_old_projects_data(@, max_run_m) # async function unlist_all_public_paths: (account_id, is_owner) => diff --git a/src/packages/database/postgres/bulk-delete.test.ts b/src/packages/database/postgres/bulk-delete.test.ts index 423c10b009..8f156c0c9c 100644 --- a/src/packages/database/postgres/bulk-delete.test.ts +++ b/src/packages/database/postgres/bulk-delete.test.ts @@ -3,12 +3,15 @@ * License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details */ +// see packages/database/pool/pool.ts for where this name is also hard coded: +process.env.PGDATABASE = "smc_ephemeral_testing_database"; + import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import { bulk_delete } from "./bulk-delete"; beforeAll(async () => { - await initEphemeralDatabase(); + await initEphemeralDatabase({ reset: true }); }, 15000); afterAll(async () => { diff --git a/src/packages/database/postgres/bulk-delete.ts b/src/packages/database/postgres/bulk-delete.ts index b6b38490a4..098f73cafd 100644 --- a/src/packages/database/postgres/bulk-delete.ts +++ b/src/packages/database/postgres/bulk-delete.ts @@ -1,14 +1,17 @@ -// see packages/database/pool/pool.ts for where this name is also hard coded: -process.env.PGDATABASE = "smc_ephemeral_testing_database"; - import { escapeIdentifier } from "pg"; import getPool from "@cocalc/database/pool"; import { SCHEMA } from "@cocalc/util/schema"; +type Field = + | "project_id" + | "account_id" + | "target_project_id" + | "source_project_id"; + interface Opts { table: string; // e.g. project_log, etc. - field: "project_id" | "account_id"; // for now, we only support a few + field: Field; // for now, we only support a few id?: string; // default "id", the ID field in the table, which identifies each row uniquely value: string; // a UUID limit?: number; // default 1024 diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index 8e88c8bf66..293af2fe72 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -9,11 +9,12 @@ Code related to permanently deleting projects. import getLogger from "@cocalc/backend/logger"; import getPool from "@cocalc/database/pool"; -import { callback2 } from "@cocalc/util/async-utils"; -import { PostgreSQL } from "./types"; -import { minutes_ago } from "@cocalc/util/misc"; import { getServerSettings } from "@cocalc/database/settings"; +import { callback2 } from "@cocalc/util/async-utils"; import { KUCALC_ON_PREMISES } from "@cocalc/util/db-schema/site-defaults"; +import { minutes_ago } from "@cocalc/util/misc"; +import { bulk_delete } from "./bulk-delete"; +import { PostgreSQL } from "./types"; const log = getLogger("db:delete-projects"); @@ -59,8 +60,9 @@ async function get_account_id( } /* -This deletes all projects older than the given number of days, from the perspective of a user. -Another task has to run to actually get rid of the data, etc. +This removes all users from all projects older than the given number of days and marked as deleted. +In particular, users are no longer able to access that project. +The "cleanup_old_projects_data" function has to run to actually get rid of the data, etc. */ export async function unlink_old_deleted_projects( db: PostgreSQL, @@ -70,7 +72,7 @@ export async function unlink_old_deleted_projects( query: "UPDATE projects", set: { users: null }, where: [ - "deleted = true", + "deleted = true", "users IS NOT NULL", `last_edited <= NOW() - '${age_d} days'::INTERVAL`, ], @@ -83,27 +85,32 @@ FROM projects as p INNER JOIN syncstrings as s ON p.project_id = s.project_id WHERE p.deleted = true + AND users IS NULL AND p.state ->> 'state' != 'deleted' +ORDER BY + p.project_id, s.string_id `; /* - This is more thorough than the above. It issues actual delete operations on data of projects marked as deleted. + This more thorough delete procedure comes after the above. + It issues actual delete operations on data of projects marked as deleted. When done, it sets the state.state to "deleted". The operations involves deleting all syncstrings of that project (and associated with that, patches), - and only for on-prem setups, it also deletes all the data stored in the project on disk. + and only for on-prem setups, it also deletes all the data stored in the project on disk and various tables. - This function is called every couple of hours. Hence ensure it does not run longer than the given max_run_m time (minutes) + This function is called every couple of hours. Hence it checks to not run longer than the given max_run_m time (minutes). */ export async function cleanup_old_projects_data( db: PostgreSQL, - delay_ms = 50, max_run_m = 60, ) { const settings = await getServerSettings(); const on_prem = settings.kucalc === KUCALC_ON_PREMISES; + const L0 = log.extend("cleanup_old_projects_data"); + const L = L0.debug; - log.debug("cleanup_old_projects_data", { delay_ms, max_run_m, on_prem }); + log.debug("cleanup_old_projects_data", { max_run_m, on_prem }); const start_ts = new Date(); const pool = getPool(); @@ -115,34 +122,95 @@ export async function cleanup_old_projects_data( for (const row of rows) { const { project_id, string_id } = row; if (start_ts < minutes_ago(max_run_m)) { - log.debug( - `cleanup_old_projects_data: too much time elapsed, breaking after ${num} syncstrings`, - ); + L(`too much time elapsed, breaking after ${num} syncstrings`); break; } - log.debug( - `cleanup_old_projects_data: deleting syncstring ${project_id}/${string_id}`, - ); + L(`deleting syncstring ${project_id}/${string_id}`); num += 1; await callback2(db.delete_syncstring, { string_id }); - // wait for the given amount of delay_ms millio seconds - await new Promise((done) => setTimeout(done, delay_ms)); + // wait a bit after deleting syncstrings, e.g. to let the standby db catch up + await new Promise((done) => setTimeout(done, 100)); + // Q_CLEANUP_SYNCSTRINGS orders by project_id, hence we trigger project specific actions when the id changes if (pid != project_id) { pid = project_id; + const L2 = L0.extend(project_id).debug; + if (on_prem) { - log.debug( - `cleanup_old_projects_data: deleting project data in ${project_id}`, - ); + L2(`cleanup_old_projects_data for project_id=${project_id}`); // TODO: this only works on-prem, and requires the project files to be mounted - log.debug(`deleting all shared files in project ${project_id}`); + L2(`deleting all shared files in project ${project_id}`); // TODO: do it directly like above, and also get rid of all those shares in the database + + const delPublicPaths = await bulk_delete({ + table: "public_paths", + field: "project_id", + value: project_id, + }); + L2(`deleted public_paths ${delPublicPaths.rowsDeleted} entries`); + + const delProjectLog = await bulk_delete({ + table: "project_log", + field: "project_id", + value: project_id, + }); + L2(`deleted project_log ${delProjectLog.rowsDeleted} entries`); + + const delFileUse = await bulk_delete({ + table: "file_use", + field: "project_id", + value: project_id, + }); + L2(`deleted file_use ${delFileUse.rowsDeleted} entries`); + + const delAccessLog = await bulk_delete({ + table: "file_access_log", + field: "project_id", + value: project_id, + }); + L2(`deleted file_access_log ${delAccessLog.rowsDeleted} entries`); + + const delJupyterApiLog = await bulk_delete({ + table: "jupyter_api_log", + field: "project_id", + value: project_id, + }); + L2(`deleted jupyter_api_log ${delJupyterApiLog.rowsDeleted} entries`); + + for (const field of [ + "target_project_id", + "source_project_id", + ] as const) { + const delCopyPaths = await bulk_delete({ + table: "copy_paths", + field, + value: project_id, + }); + L2(`deleted copy_paths/${field} ${delCopyPaths.rowsDeleted} entries`); + } + + const delListings = await bulk_delete({ + table: "listings", + field: "project_id", + id: "project_id", // TODO listings has a more complex ID, is this a problem? + value: project_id, + }); + L2(`deleted ${delListings.rowsDeleted} listings`); + + const delInviteTokens = await bulk_delete({ + table: "project_invite_tokens", + field: "project_id", + value: project_id, + id: "token", + }); + L2(`deleted ${delInviteTokens.rowsDeleted} entries`); } // now, that we're done with that project, mark it as state.state ->> 'deleted' + // in addition to the flag "deleted = true" await callback2(db.set_project_state, { project_id, state: "deleted", diff --git a/src/packages/hub/run/delete-projects.js b/src/packages/hub/run/delete-projects.js index dbef215ed9..1b0c8c833e 100755 --- a/src/packages/hub/run/delete-projects.js +++ b/src/packages/hub/run/delete-projects.js @@ -16,7 +16,8 @@ async function update() { console.log("unlinking old deleted projects..."); try { await db.unlink_old_deleted_projects(); - await db.cleanup_old_projects_data(); + const max_run_m = (INTERVAL_MS / 2) / (1000 * 60) + await db.cleanup_old_projects_data(max_run_m); } catch (err) { if (err !== null) { throw Error(`failed to unlink projects -- ${err}`); From 5d7c4aa98df0710f87f5866a389aa635e3dbc735 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Wed, 10 Jul 2024 18:00:24 +0200 Subject: [PATCH 07/20] database/test: attempting to actually fix testing the database in the database package --- src/packages/database/jest.config.js | 1 + src/packages/database/package.json | 2 +- src/packages/database/postgres/bulk-delete.test.ts | 5 +---- src/packages/database/postgres/site-license/hook.test.ts | 2 -- src/packages/database/test/setup.js | 7 +++++++ 5 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 src/packages/database/test/setup.js diff --git a/src/packages/database/jest.config.js b/src/packages/database/jest.config.js index 140b9467f2..4c9b378d0e 100644 --- a/src/packages/database/jest.config.js +++ b/src/packages/database/jest.config.js @@ -3,4 +3,5 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', testMatch: ['**/?(*.)+(spec|test).ts?(x)'], + setupFiles: ['./test/setup.js'], // Path to your setup file }; diff --git a/src/packages/database/package.json b/src/packages/database/package.json index 6862fe52dd..74c3799308 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -50,7 +50,7 @@ "build": "../node_modules/.bin/tsc --build && coffee -c -o dist/ ./", "clean": "rm -rf dist", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest", + "test": "TZ=UTC jest --forceExit --runInBand", "prepublishOnly": "pnpm test" }, "repository": { diff --git a/src/packages/database/postgres/bulk-delete.test.ts b/src/packages/database/postgres/bulk-delete.test.ts index 8f156c0c9c..9f1db7ddf6 100644 --- a/src/packages/database/postgres/bulk-delete.test.ts +++ b/src/packages/database/postgres/bulk-delete.test.ts @@ -3,15 +3,12 @@ * License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details */ -// see packages/database/pool/pool.ts for where this name is also hard coded: -process.env.PGDATABASE = "smc_ephemeral_testing_database"; - import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import { bulk_delete } from "./bulk-delete"; beforeAll(async () => { - await initEphemeralDatabase({ reset: true }); + await initEphemeralDatabase({}); }, 15000); afterAll(async () => { diff --git a/src/packages/database/postgres/site-license/hook.test.ts b/src/packages/database/postgres/site-license/hook.test.ts index a7c5f0c561..67e9359108 100644 --- a/src/packages/database/postgres/site-license/hook.test.ts +++ b/src/packages/database/postgres/site-license/hook.test.ts @@ -14,8 +14,6 @@ * The quota function uses a deep copy operation on all its arguments to avoid this. */ -// see packages/database/pool/pool.ts for where this name is also hard coded: -process.env.PGDATABASE = "smc_ephemeral_testing_database"; import { isEqual } from "lodash"; diff --git a/src/packages/database/test/setup.js b/src/packages/database/test/setup.js new file mode 100644 index 0000000000..ee2e6cce0d --- /dev/null +++ b/src/packages/database/test/setup.js @@ -0,0 +1,7 @@ +// test/setup.js + +// see packages/database/pool/pool.ts for where this name is also hard coded: +process.env.PGDATABASE = "smc_ephemeral_testing_database"; + +// checked for in some code to behave differently while running unit tests. +process.env.COCALC_TEST_MODE = true; From d8f481c0010a6d7887cd9501f92210f75bb3ea64 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Thu, 11 Jul 2024 11:15:44 +0200 Subject: [PATCH 08/20] database/delete-project: refactor/fixes --- src/packages/backend/metrics.ts | 4 +- .../database/postgres/bulk-delete.test.ts | 4 +- src/packages/database/postgres/bulk-delete.ts | 2 +- .../database/postgres/delete-projects.ts | 217 +++++++++++------- src/packages/hub/run/delete-projects.js | 10 +- 5 files changed, 143 insertions(+), 94 deletions(-) diff --git a/src/packages/backend/metrics.ts b/src/packages/backend/metrics.ts index 14551ae9cb..6500653db0 100644 --- a/src/packages/backend/metrics.ts +++ b/src/packages/backend/metrics.ts @@ -1,6 +1,6 @@ import { Counter, Gauge, Histogram } from "prom-client"; -type Aspect = "db" | "database" | "server" | "llm"; +type Aspect = "db" | "database" | "server" | "llm" | "database"; function withPrefix(aspect: Aspect, name: string): string { return `cocalc_${aspect}_${name}`; @@ -13,7 +13,7 @@ export function newCounter( name: string, help: string, labelNames: string[] = [], -) { +): Counter { name = withPrefix(aspect, name); const key = `counter-${name}`; if (cache[key] != null) { diff --git a/src/packages/database/postgres/bulk-delete.test.ts b/src/packages/database/postgres/bulk-delete.test.ts index 9f1db7ddf6..c6c2ba475c 100644 --- a/src/packages/database/postgres/bulk-delete.test.ts +++ b/src/packages/database/postgres/bulk-delete.test.ts @@ -5,7 +5,7 @@ import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; -import { bulk_delete } from "./bulk-delete"; +import { bulkDelete } from "./bulk-delete"; beforeAll(async () => { await initEphemeralDatabase({}); @@ -41,7 +41,7 @@ describe("bulk delete", () => { ); expect(num1.rows[0].num).toEqual(N); - const res = await bulk_delete({ + const res = await bulkDelete({ table: "project_log", field: "project_id", value: project_id, diff --git a/src/packages/database/postgres/bulk-delete.ts b/src/packages/database/postgres/bulk-delete.ts index 098f73cafd..0102c9c76a 100644 --- a/src/packages/database/postgres/bulk-delete.ts +++ b/src/packages/database/postgres/bulk-delete.ts @@ -37,7 +37,7 @@ WHERE ${ID} IN ( )`; } -export async function bulk_delete(opts: Opts): Ret { +export async function bulkDelete(opts: Opts): Ret { const { table, field, value, id = "id", maxUtilPct = 10 } = opts; let { limit = 1024 } = opts; // assert table name is a key in SCHEMA diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index 293af2fe72..bda4b2c7c2 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -8,16 +8,24 @@ Code related to permanently deleting projects. */ import getLogger from "@cocalc/backend/logger"; +import { newCounter } from "@cocalc/backend/metrics"; import getPool from "@cocalc/database/pool"; import { getServerSettings } from "@cocalc/database/settings"; import { callback2 } from "@cocalc/util/async-utils"; import { KUCALC_ON_PREMISES } from "@cocalc/util/db-schema/site-defaults"; import { minutes_ago } from "@cocalc/util/misc"; -import { bulk_delete } from "./bulk-delete"; +import { bulkDelete } from "./bulk-delete"; import { PostgreSQL } from "./types"; const log = getLogger("db:delete-projects"); +const delete_projects_prom = newCounter( + "database", + "delete_projects_total", + "Deleting projects and associated data operations counter.", + ["op"], +); + /* Permanently delete from the database all project records, where the project is explicitly deleted already (so the deleted field is true). @@ -25,6 +33,8 @@ Call this function to setup projects for permanent deletion. This blanks the user field so the user no longer can access the project, and we don't know that the user had anything to do with the project. A separate phase later then purges these projects from disk as well as the database. + +TODO:it's referenced from postgres-server-queries.coffee, but is it actually used anywhere? */ export async function permanently_unlink_all_deleted_projects_of_user( db: PostgreSQL, @@ -80,15 +90,24 @@ export async function unlink_old_deleted_projects( } const Q_CLEANUP_SYNCSTRINGS = ` -SELECT p.project_id, s.string_id -FROM projects as p - INNER JOIN syncstrings as s +SELECT s.string_id, p.project_id +FROM projects as p INNER JOIN syncstrings as s ON p.project_id = s.project_id WHERE p.deleted = true - AND users IS NULL - AND p.state ->> 'state' != 'deleted' + AND p.users IS NULL ORDER BY p.project_id, s.string_id +LIMIT 10000 +`; + +const Q_CLEANUP_PROJECTS = ` +SELECT project_id +FROM projects +WHERE deleted = true + AND users IS NULL + AND state ->> 'state' != 'deleted' +ORDER BY created ASC +LIMIT 1000 `; /* @@ -110,103 +129,53 @@ export async function cleanup_old_projects_data( const L0 = log.extend("cleanup_old_projects_data"); const L = L0.debug; - log.debug("cleanup_old_projects_data", { max_run_m, on_prem }); + L("args", { max_run_m, on_prem }); const start_ts = new Date(); const pool = getPool(); - const { rows } = await pool.query(Q_CLEANUP_SYNCSTRINGS); - let num = 0; - let pid = ""; + let numSyncStr = 0; + let numProj = 0; - for (const row of rows) { - const { project_id, string_id } = row; + while (true) { if (start_ts < minutes_ago(max_run_m)) { - L(`too much time elapsed, breaking after ${num} syncstrings`); - break; + L(`too much time elapsed, breaking after ${numSyncStr} syncstrings`); + return; } - L(`deleting syncstring ${project_id}/${string_id}`); - num += 1; - await callback2(db.delete_syncstring, { string_id }); - - // wait a bit after deleting syncstrings, e.g. to let the standby db catch up - await new Promise((done) => setTimeout(done, 100)); + const { rows: syncstrings } = await pool.query(Q_CLEANUP_SYNCSTRINGS); + L(`deleting ${syncstrings.length} syncstrings`); + for (const { project_id, string_id } of syncstrings) { + L(`deleting syncstring ${project_id}/${string_id}`); + numSyncStr += 1; + const t0 = Date.now(); + await callback2(db.delete_syncstring, { string_id }); + const elapsed_ms = Date.now() - t0; + delete_projects_prom.labels("syncstring").inc(); + // wait a bit after deleting syncstrings, e.g. to let the standby db catch up + // this ensures a max of "10%" utilization of the database – or wait 1 second + await new Promise((done) => + setTimeout(done, Math.min(1000, elapsed_ms * 9)), + ); + } - // Q_CLEANUP_SYNCSTRINGS orders by project_id, hence we trigger project specific actions when the id changes - if (pid != project_id) { - pid = project_id; + const { rows: projects } = await pool.query(Q_CLEANUP_PROJECTS); + L(`deleting the data of ${projects.length} projects`); + for (const { project_id } of projects) { const L2 = L0.extend(project_id).debug; + delete_projects_prom.labels("project").inc(); + numProj += 1; + let delRows = 0; if (on_prem) { - L2(`cleanup_old_projects_data for project_id=${project_id}`); + L2(`delete all project files`); // TODO: this only works on-prem, and requires the project files to be mounted - L2(`deleting all shared files in project ${project_id}`); + L2(`deleting all shared files`); // TODO: do it directly like above, and also get rid of all those shares in the database - const delPublicPaths = await bulk_delete({ - table: "public_paths", - field: "project_id", - value: project_id, - }); - L2(`deleted public_paths ${delPublicPaths.rowsDeleted} entries`); - - const delProjectLog = await bulk_delete({ - table: "project_log", - field: "project_id", - value: project_id, - }); - L2(`deleted project_log ${delProjectLog.rowsDeleted} entries`); - - const delFileUse = await bulk_delete({ - table: "file_use", - field: "project_id", - value: project_id, - }); - L2(`deleted file_use ${delFileUse.rowsDeleted} entries`); - - const delAccessLog = await bulk_delete({ - table: "file_access_log", - field: "project_id", - value: project_id, - }); - L2(`deleted file_access_log ${delAccessLog.rowsDeleted} entries`); - - const delJupyterApiLog = await bulk_delete({ - table: "jupyter_api_log", - field: "project_id", - value: project_id, - }); - L2(`deleted jupyter_api_log ${delJupyterApiLog.rowsDeleted} entries`); - - for (const field of [ - "target_project_id", - "source_project_id", - ] as const) { - const delCopyPaths = await bulk_delete({ - table: "copy_paths", - field, - value: project_id, - }); - L2(`deleted copy_paths/${field} ${delCopyPaths.rowsDeleted} entries`); - } - - const delListings = await bulk_delete({ - table: "listings", - field: "project_id", - id: "project_id", // TODO listings has a more complex ID, is this a problem? - value: project_id, - }); - L2(`deleted ${delListings.rowsDeleted} listings`); - - const delInviteTokens = await bulk_delete({ - table: "project_invite_tokens", - field: "project_id", - value: project_id, - id: "token", - }); - L2(`deleted ${delInviteTokens.rowsDeleted} entries`); + // for now, on-prem only as well. This gets rid of all sorts of data in tables specific to the given project. + delRows += await delete_associated_project_data(L2, project_id); } // now, that we're done with that project, mark it as state.state ->> 'deleted' @@ -215,6 +184,80 @@ export async function cleanup_old_projects_data( project_id, state: "deleted", }); + L2( + `finished deleting project data | deleted ${delRows} entries | setting state.state="deleted"`, + ); } + + if (projects.length === 0 && syncstrings.length === 0) { + L(`all data of deleted projects and associated syncstrings are deleted.`); + L( + `In total ${numSyncStr} syncstrings and ${numProj} projects were processed.`, + ); + return; + } + } +} + +async function delete_associated_project_data( + L2, + project_id: string, +): Promise { + let total = 0; + // collecting tables, where the primary key is the default (i.e. "id") and + // the field to check is always called "project_id" + const tables = [ + "public_paths", + "project_log", + "file_use", + "file_access_log", + "jupyter_api_log", + "openai_chatgpt_log", + ] as const; + + for (const table of tables) { + const { rowsDeleted } = await bulkDelete({ + table, + field: "project_id", + value: project_id, + }); + total += rowsDeleted; + L2(`deleted ${table} ${rowsDeleted} entries`); + } + + // these tables are different, i.e. another id, or the field to check the project_id value against is called differently + + for (const field of ["target_project_id", "source_project_id"] as const) { + const { rowsDeleted } = await bulkDelete({ + table: "copy_paths", + field, + value: project_id, + }); + total += rowsDeleted; + L2(`deleted copy_paths/${field} ${rowsDeleted} entries`); } + + { + const { rowsDeleted } = await bulkDelete({ + table: "listings", + field: "project_id", + id: "project_id", // TODO listings has a more complex ID, is this a problem? + value: project_id, + }); + total += rowsDeleted; + L2(`deleted ${rowsDeleted} listings`); + } + + { + const { rowsDeleted } = await bulkDelete({ + table: "project_invite_tokens", + field: "project_id", + value: project_id, + id: "token", + }); + total += rowsDeleted; + L2(`deleted ${rowsDeleted} entries`); + } + + return total; } diff --git a/src/packages/hub/run/delete-projects.js b/src/packages/hub/run/delete-projects.js index 1b0c8c833e..9d9372ad1f 100755 --- a/src/packages/hub/run/delete-projects.js +++ b/src/packages/hub/run/delete-projects.js @@ -1,9 +1,14 @@ #!/usr/bin/env node + /* Periodically delete projects. -TODO: For now, this just calls the unlink function. Later on it -should do more (actually delete data, etc.). +STATUS: +For now, this just calls the unlink function and deletes all assocated syncstrings and data. +In "onprem" mode, this also entries in various tables, which contain data specific to the deleted projects. + +TESTING: to run this in development and see logging, call it like that: +./src/packages/hub$ env DEBUG_CONSOLE=yes DEBUG=cocalc:debug:db:* pnpm cocalc-hub-delete-projects */ const postgres = require("@cocalc/database"); @@ -16,6 +21,7 @@ async function update() { console.log("unlinking old deleted projects..."); try { await db.unlink_old_deleted_projects(); + // limit the max runtime to half the interval time const max_run_m = (INTERVAL_MS / 2) / (1000 * 60) await db.cleanup_old_projects_data(max_run_m); } catch (err) { From 39d0065a4f93eb7b3938d4a972c93f584d10f463 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Thu, 11 Jul 2024 17:50:57 +0200 Subject: [PATCH 09/20] hub/delete-project: expand functionality and acutally delete files --- src/packages/backend/files/path-to-files.ts | 17 ++++++++++ .../database/postgres/delete-projects.ts | 33 +++++++++++++++++-- src/packages/next/lib/share/get-contents.ts | 12 ++++--- src/packages/next/lib/share/path-to-files.ts | 13 ++------ src/packages/next/lib/share/virtual-hosts.ts | 9 ++--- 5 files changed, 62 insertions(+), 22 deletions(-) create mode 100644 src/packages/backend/files/path-to-files.ts diff --git a/src/packages/backend/files/path-to-files.ts b/src/packages/backend/files/path-to-files.ts new file mode 100644 index 0000000000..fdb344b0f3 --- /dev/null +++ b/src/packages/backend/files/path-to-files.ts @@ -0,0 +1,17 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details + */ + +// This is used to find files on the share server (public_paths) in "next" +// and also in the hub, for deleting shared files of projects + +import { join } from "node:path"; + +import { projects } from "@cocalc/backend/data"; + +// Given a project_id/path, return the directory on the file system where +// that path should be located. +export function pathToFiles(project_id: string, path: string): string { + return join(projects.replace("[project_id]", project_id), path); +} diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index bda4b2c7c2..701e831839 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -7,6 +7,10 @@ Code related to permanently deleting projects. */ +import { promises as fs } from "node:fs"; +import { join } from "node:path"; + +import { pathToFiles } from "@cocalc/backend/files/path-to-files"; import getLogger from "@cocalc/backend/logger"; import { newCounter } from "@cocalc/backend/metrics"; import getPool from "@cocalc/database/pool"; @@ -105,7 +109,7 @@ SELECT project_id FROM projects WHERE deleted = true AND users IS NULL - AND state ->> 'state' != 'deleted' + AND coalesce(state ->> 'state', '') != 'deleted' ORDER BY created ASC LIMIT 1000 `; @@ -169,10 +173,12 @@ export async function cleanup_old_projects_data( if (on_prem) { L2(`delete all project files`); - // TODO: this only works on-prem, and requires the project files to be mounted + await deleteProjectFiles(L2, project_id); L2(`deleting all shared files`); - // TODO: do it directly like above, and also get rid of all those shares in the database + // this is something like /shared/projects/${project_id} + const shared_path = pathToFiles(project_id, ""); + await fs.rm(shared_path, { recursive: true, force: true }); // for now, on-prem only as well. This gets rid of all sorts of data in tables specific to the given project. delRows += await delete_associated_project_data(L2, project_id); @@ -261,3 +267,24 @@ async function delete_associated_project_data( return total; } + +async function deleteProjectFiles(L2, project_id: string) { + // TODO: this only works on-prem, and requires the project files to be mounted + const projects_root = process.env["MOUNTED_PROJECTS_ROOT"]; + if (!projects_root) return; + const project_dir = join(projects_root, project_id); + try { + await fs.access(project_dir, fs.constants.F_OK | fs.constants.R_OK); + const stats = await fs.lstat(project_dir); + if (stats.isDirectory()) { + L2(`deleting all files in ${project_dir}`); + await fs.rm(project_dir, { recursive: true, force: true }); + } else { + L2(`is not a directory: ${project_dir}`); + } + } catch (err) { + L2( + `not deleting project files: either it does not exist or is not accessible`, + ); + } +} diff --git a/src/packages/next/lib/share/get-contents.ts b/src/packages/next/lib/share/get-contents.ts index 02d60c23d8..9f19ca9964 100644 --- a/src/packages/next/lib/share/get-contents.ts +++ b/src/packages/next/lib/share/get-contents.ts @@ -3,14 +3,16 @@ * License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details */ -import pathToFiles from "./path-to-files"; import { promises as fs } from "fs"; -import { join } from "path"; import { sortBy } from "lodash"; +import { join } from "path"; + +import { pathToFiles } from "@cocalc/backend/files/path-to-files"; import { hasSpecialViewer } from "@cocalc/frontend/file-extensions"; import { getExtension } from "./util"; const MB: number = 1000000; + const LIMITS = { listing: 10000, // directory listing is truncated after this many files ipynb: 15 * MB, @@ -18,7 +20,7 @@ const LIMITS = { whiteboard: 5 * MB, slides: 5 * MB, other: 2 * MB, -}; +} as const; export interface FileInfo { name: string; @@ -40,7 +42,7 @@ export interface PathContents { export default async function getContents( project_id: string, - path: string + path: string, ): Promise { const fsPath = pathToFiles(project_id, path); const obj: PathContents = {}; @@ -72,7 +74,7 @@ export default async function getContents( } async function getDirectoryListing( - path: string + path: string, ): Promise<{ listing: FileInfo[]; truncated?: string }> { const listing: FileInfo[] = []; let truncated: string | undefined = undefined; diff --git a/src/packages/next/lib/share/path-to-files.ts b/src/packages/next/lib/share/path-to-files.ts index 1bb0a63449..ef741bd2a1 100644 --- a/src/packages/next/lib/share/path-to-files.ts +++ b/src/packages/next/lib/share/path-to-files.ts @@ -3,24 +3,17 @@ * License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details */ -import { join } from "path"; +import { pathToFiles } from "@cocalc/backend/files/path-to-files"; import getPool from "@cocalc/database/pool"; -import { projects } from "@cocalc/backend/data"; - -// Given a project_id/path, return the directory on the file system where -// that path should be located. -export default function pathToFiles(project_id: string, path: string): string { - return join(projects.replace("[project_id]", project_id), path); -} export async function pathFromID( - id: string + id: string, ): Promise<{ projectPath: string; fsPath: string }> { // 'infinite' since actually result can't change since id determines the path (it's a reverse sha1 hash computation) const pool = getPool("infinite"); const { rows } = await pool.query( "SELECT project_id, path FROM public_paths WHERE id=$1 AND disabled IS NOT TRUE", - [id] + [id], ); if (rows.length == 0) { throw Error(`no such public path: ${id}`); diff --git a/src/packages/next/lib/share/virtual-hosts.ts b/src/packages/next/lib/share/virtual-hosts.ts index 3c1aa69cbc..95be46785d 100644 --- a/src/packages/next/lib/share/virtual-hosts.ts +++ b/src/packages/next/lib/share/virtual-hosts.ts @@ -8,12 +8,13 @@ Support for virtual hosts. */ import type { Request, Response } from "express"; + +import basePath from "@cocalc/backend/base-path"; +import { pathToFiles } from "@cocalc/backend/files/path-to-files"; import { getLogger } from "@cocalc/backend/logger"; -import pathToFiles from "./path-to-files"; import isAuthenticated from "./authenticate"; import getVirtualHostInfo from "./get-vhost-info"; import { staticHandler } from "./handle-raw"; -import basePath from "@cocalc/backend/base-path"; const logger = getLogger("virtual-hosts"); @@ -23,7 +24,7 @@ export default function virtualHostsMiddleware() { return async function ( req: Request, res: Response, - next: Function + next: Function, ): Promise { // For debugging in cc-in-cc dev, just manually set host to something // else and comment this out. That's the only way, since dev is otherwise @@ -69,7 +70,7 @@ export default function virtualHostsMiddleware() { logger.debug( "not authenticated -- denying vhost='%s', path='%s'", vhost, - path + path, ); res.status(403).end(); return; From f4cabc5f85a0bcfcb8e5e2419a0e0243ac7000fb Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Fri, 12 Jul 2024 12:18:25 +0200 Subject: [PATCH 10/20] hub/delete-project: add explicit delete_project_data setting --- .../database/postgres/delete-projects.ts | 45 ++++++++++++------- src/packages/util/db-schema/site-defaults.ts | 7 +++ 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index 701e831839..8fd493a397 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -101,7 +101,7 @@ WHERE p.deleted = true AND p.users IS NULL ORDER BY p.project_id, s.string_id -LIMIT 10000 +LIMIT 1000 `; const Q_CLEANUP_PROJECTS = ` @@ -130,12 +130,18 @@ export async function cleanup_old_projects_data( ) { const settings = await getServerSettings(); const on_prem = settings.kucalc === KUCALC_ON_PREMISES; + const delete_data = settings.delete_project_data; const L0 = log.extend("cleanup_old_projects_data"); const L = L0.debug; - L("args", { max_run_m, on_prem }); - const start_ts = new Date(); + L("args", { max_run_m, on_prem, delete_data }); + if (!delete_data) { + L(`deleting project data is disabled ('delete_project_data' setting).`); + return; + } + + const start_ts = new Date(); const pool = getPool(); let numSyncStr = 0; @@ -171,27 +177,34 @@ export async function cleanup_old_projects_data( numProj += 1; let delRows = 0; + // Clean up data *on* a given project. For now, remove all site licenses. + await pool.query( + `UPDATE projects SET site_license = NULL WHERE project_id = $1`, + [project_id], + ); + if (on_prem) { L2(`delete all project files`); await deleteProjectFiles(L2, project_id); L2(`deleting all shared files`); - // this is something like /shared/projects/${project_id} - const shared_path = pathToFiles(project_id, ""); - await fs.rm(shared_path, { recursive: true, force: true }); - - // for now, on-prem only as well. This gets rid of all sorts of data in tables specific to the given project. - delRows += await delete_associated_project_data(L2, project_id); + try { + // this is something like /shared/projects/${project_id} + const shared_path = pathToFiles(project_id, ""); + await fs.rm(shared_path, { recursive: true, force: true }); + } catch (err) { + L2(`Unable to delete shared files: ${err}`); + } } + // This gets rid of all sorts of data in tables specific to the given project. + delRows += await delete_associated_project_data(L2, project_id); + // now, that we're done with that project, mark it as state.state ->> 'deleted' - // in addition to the flag "deleted = true" - await callback2(db.set_project_state, { - project_id, - state: "deleted", - }); + // in addition to the flag "deleted = true". This also updates the state.time timestamp. + await callback2(db.set_project_state, { project_id, state: "deleted" }); L2( - `finished deleting project data | deleted ${delRows} entries | setting state.state="deleted"`, + `finished deleting project data | deleted ${delRows} entries | state.state="deleted"`, ); } @@ -247,7 +260,7 @@ async function delete_associated_project_data( const { rowsDeleted } = await bulkDelete({ table: "listings", field: "project_id", - id: "project_id", // TODO listings has a more complex ID, is this a problem? + id: "project_id", // TODO listings has a more complex ID, which means this gets rid of everything in one go. should be fine, though. value: project_id, }); total += rowsDeleted; diff --git a/src/packages/util/db-schema/site-defaults.ts b/src/packages/util/db-schema/site-defaults.ts index 6b93f775b7..ddfe3889dd 100644 --- a/src/packages/util/db-schema/site-defaults.ts +++ b/src/packages/util/db-schema/site-defaults.ts @@ -80,6 +80,7 @@ export type SiteSettingsKeys = | "require_license_to_create_project" | "google_analytics" | "kucalc" + | "delete_project_data" | "dns" | "datastore" | "ssh_gateway" @@ -652,6 +653,12 @@ export const site_settings_conf: SiteSettings = { to_val: split_iframe_comm_hosts, to_display: num_dns_hosts, }, + delete_project_data : { + name :"Delete Project Data", + desc: "When a project has been marked as deleted, also actually delete associated data from the database and (for OnPrem) also its files.", + default: "no", + to_val: to_bool, + }, email_enabled: { name: "Email sending enabled", desc: "Controls visibility of UI elements and if any emails are sent. This is independent of any particular email configuration!", From 3ba695acd569e0e826758ec58b15745078aad04a Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Wed, 17 Jul 2024 15:15:38 +0200 Subject: [PATCH 11/20] database/bulk-delete: add env vars and improve test --- .../database/postgres/bulk-delete.test.ts | 21 +++++++------ src/packages/database/postgres/bulk-delete.ts | 30 ++++++++++++++----- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/packages/database/postgres/bulk-delete.test.ts b/src/packages/database/postgres/bulk-delete.test.ts index c6c2ba475c..e06f3b4a2f 100644 --- a/src/packages/database/postgres/bulk-delete.test.ts +++ b/src/packages/database/postgres/bulk-delete.test.ts @@ -19,7 +19,7 @@ describe("bulk delete", () => { test("deleting projects", async () => { const p = getPool(); const project_id = uuid(); - const N = 2000; + const N = 100000; // extra entry, which has to remain const other = uuid(); @@ -28,12 +28,12 @@ describe("bulk delete", () => { [other, uuid(), new Date()], ); - for (let i = 0; i < N; i++) { - await p.query( - "INSERT INTO project_log (id, project_id, time) VALUES($1::UUID, $2::UUID, $3::TIMESTAMP)", - [uuid(), project_id, new Date()], - ); - } + await p.query( + `INSERT INTO project_log (id, project_id, time) + SELECT gen_random_uuid(), $1::UUID, NOW() - interval '1 second' * g.n + FROM generate_series(1, $2) AS g(n)`, + [project_id, N], + ); const num1 = await p.query( "SELECT COUNT(*)::INT as num FROM project_log WHERE project_id = $1", @@ -45,16 +45,15 @@ describe("bulk delete", () => { table: "project_log", field: "project_id", value: project_id, - limit: 128, }); // if this ever fails, the "ret.rowCount" value is inaccurate. // This must be replaced by "RETURNING 1" in the the query and a "SELECT COUNT(*) ..." and so. // (and not only here, but everywhere in the code base) expect(res.rowsDeleted).toEqual(N); - expect(res.durationS).toBeGreaterThan(0.01); - expect(res.totalPgTimeS).toBeGreaterThan(0.001); - expect(res.totalWaitS).toBeGreaterThan(0.001); + expect(res.durationS).toBeGreaterThan(0.1); + expect(res.totalPgTimeS).toBeGreaterThan(0.1); + expect(res.totalWaitS).toBeGreaterThan(0.1); expect((res.totalPgTimeS * 10) / res.totalWaitS).toBeGreaterThan(0.5); const num2 = await p.query( diff --git a/src/packages/database/postgres/bulk-delete.ts b/src/packages/database/postgres/bulk-delete.ts index 0102c9c76a..04519e5b2f 100644 --- a/src/packages/database/postgres/bulk-delete.ts +++ b/src/packages/database/postgres/bulk-delete.ts @@ -1,14 +1,30 @@ import { escapeIdentifier } from "pg"; +import getLogger from "@cocalc/backend/logger"; +import { envToInt } from "@cocalc/backend/misc/env-to-number"; import getPool from "@cocalc/database/pool"; import { SCHEMA } from "@cocalc/util/schema"; +const log = getLogger("db:bulk-delete"); +const D = log.debug; + type Field = | "project_id" | "account_id" | "target_project_id" | "source_project_id"; +const MAX_UTIL_PCT = envToInt("COCALC_DB_BULK_DELETE_MAX_UTIL_PCT", 10); +// adjust the time limits: by default, we aim to keep the operation between 0.1 and 0.2 secs +const MAX_TIME_TARGET_MS = envToInt( + "COCALC_DB_BULK_DELETE_MAX_TIME_TARGET_MS", + 100, +); +const MAX_TARGET_S = MAX_TIME_TARGET_MS / 1000; +const MIN_TARGET_S = MAX_TARGET_S / 2; +const DEFAULT_LIMIT = envToInt("COCALC_DB_BULK_DELETE_DEFAULT_LIMIT", 16); +const MAX_LIMIT = envToInt("COCALC_DB_BULK_DELETE_MAX_LIMIT", 32768); + interface Opts { table: string; // e.g. project_log, etc. field: Field; // for now, we only support a few @@ -38,8 +54,8 @@ WHERE ${ID} IN ( } export async function bulkDelete(opts: Opts): Ret { - const { table, field, value, id = "id", maxUtilPct = 10 } = opts; - let { limit = 1024 } = opts; + const { table, field, value, id = "id", maxUtilPct = MAX_UTIL_PCT } = opts; + let { limit = DEFAULT_LIMIT } = opts; // assert table name is a key in SCHEMA if (!(table in SCHEMA)) { throw new Error(`table ${table} does not exist`); @@ -63,18 +79,16 @@ export async function bulkDelete(opts: Opts): Ret { rowsDeleted += ret.rowCount ?? 0; totalPgTimeS += dt; - // adjust the limit: we aim to keep the operation between 0.1 and 0.2 secs - const next = dt > 0.2 ? limit / 2 : dt < 0.1 ? limit * 2 : limit; - limit = Math.max(1, Math.min(32768, Math.round(next))); + const next = + dt > MAX_TARGET_S ? limit / 2 : dt < MIN_TARGET_S ? limit * 2 : limit; + limit = Math.max(1, Math.min(MAX_LIMIT, Math.round(next))); // wait for a bit, but not more than 1 second ~ this aims for a max utilization of 10% const waitS = Math.min(1, dt * ((100 - maxUtilPct) / maxUtilPct)); await new Promise((done) => setTimeout(done, 1000 * waitS)); totalWaitS += waitS; - // console.log( - // `deleted ${ret.rowCount} | dt=${dt} | wait=${waitS} | limit=${limit}`, - // ); + D(`deleted ${ret.rowCount} | dt=${dt} | wait=${waitS} | limit=${limit}`); if (ret.rowCount === 0) break; } From a0dea091318a4bd8c731b6caf520ae9e1096ffbe Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Wed, 17 Jul 2024 16:41:01 +0200 Subject: [PATCH 12/20] frontend/settings: modernize project visibility and delete controls --- .../database/postgres/delete-projects.ts | 2 +- .../project/settings/hide-delete-box.tsx | 158 +++++++++--------- .../frontend/project/warnings/deleted.tsx | 4 +- 3 files changed, 80 insertions(+), 84 deletions(-) diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index 8fd493a397..ca01ef34be 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -38,7 +38,7 @@ the user field so the user no longer can access the project, and we don't know that the user had anything to do with the project. A separate phase later then purges these projects from disk as well as the database. -TODO:it's referenced from postgres-server-queries.coffee, but is it actually used anywhere? +TODO: it's referenced from postgres-server-queries.coffee, but is it actually used anywhere? */ export async function permanently_unlink_all_deleted_projects_of_user( db: PostgreSQL, diff --git a/src/packages/frontend/project/settings/hide-delete-box.tsx b/src/packages/frontend/project/settings/hide-delete-box.tsx index eed41457dc..91f7e9f776 100644 --- a/src/packages/frontend/project/settings/hide-delete-box.tsx +++ b/src/packages/frontend/project/settings/hide-delete-box.tsx @@ -3,15 +3,20 @@ * License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details */ -import { Alert, Button, Space } from "antd"; -import { useState } from "react"; +import { Alert, Button, Popconfirm, Switch } from "antd"; -import { Col, Row, Well } from "@cocalc/frontend/antd-bootstrap"; -import { Icon, SettingBox } from "@cocalc/frontend/components"; +import { Col, Row } from "@cocalc/frontend/antd-bootstrap"; +import { + Icon, + Paragraph, + SettingBox, + Title, +} from "@cocalc/frontend/components"; import { HelpEmailLink } from "@cocalc/frontend/customize"; import { ProjectsActions } from "@cocalc/frontend/todo-types"; import { webapp_client } from "@cocalc/frontend/webapp-client"; import { COLORS } from "@cocalc/util/theme"; +import { useMemo } from "react"; import { DeletedProjectWarning } from "../warnings/deleted"; import { Project } from "./types"; @@ -24,12 +29,12 @@ interface Props { export function HideDeleteBox(props: Readonly) { const { project, actions, mode = "project" } = props; const isFlyout = mode === "flyout"; - - const [show_delete_conf, set_show_delete_conf] = useState(false); + const is_deleted = useMemo(() => { + return project.get("deleted"); + }, [project.get("deleted")]); function toggle_delete_project(): void { actions.toggle_delete_project(project.get("project_id")); - set_show_delete_conf(false); } function toggle_hide_project(): void { @@ -42,7 +47,7 @@ export function HideDeleteBox(props: Readonly) { } function delete_message(): JSX.Element { - if (project.get("deleted")) { + if (is_deleted) { return ; } else { return ( @@ -63,60 +68,65 @@ export function HideDeleteBox(props: Readonly) { if (user == undefined) { return Does not make sense for admin.; } - if (user.get("hide")) { + + return ( + + Hide this project, so it does not show up in your default project + listing. This only impacts you, not your collaborators, and you can + easily unhide it. + + ); + } + + function render_delete_undelete_button(): JSX.Element { + if (is_deleted) { return ( - - Unhide this project, so it shows up in your default project listing. - Right now it only appears when hidden is checked. - + ); } else { return ( - - Hide this project, so it does not show up in your default project - listing. This only impacts you, not your collaborators, and you can - easily unhide it. - + } + > + + ); } } - function render_delete_undelete_button(is_deleted, is_expanded): JSX.Element { - let disabled, onClick, text; - if (is_deleted) { - text = "Undelete Project"; - onClick = toggle_delete_project; - disabled = false; - } else { - text = "Delete Project..."; - onClick = () => set_show_delete_conf(true); - disabled = is_expanded; - } - - return ( - - ); - } - function render_expanded_delete_info(): JSX.Element { const has_upgrades = webapp_client.account_id == null ? false : user_has_applied_upgrades(webapp_client.account_id, project); return ( - + +
+ Are you sure you want to delete this project? +
{has_upgrades ? ( @@ -128,22 +138,7 @@ export function HideDeleteBox(props: Readonly) { } /> ) : undefined} - {!has_upgrades ? ( -
- Are you sure you want to delete this project? -
- ) : undefined} - - - - - + ); } @@ -151,32 +146,33 @@ export function HideDeleteBox(props: Readonly) { return ( <> - {hide_message()} - - + + + <Icon name={hidden ? "eye-slash" : "eye"} /> Project{" "} + {hidden ? "hidden" : "visible"} + <Switch + checked={hidden} + style={{ float: "right" }} + checkedChildren={"Hidden"} + unCheckedChildren={"Visible"} + onChange={toggle_hide_project} + /> + + {hide_message()}
- {delete_message()} - - {render_delete_undelete_button( - project.get("deleted"), - show_delete_conf, - )} + + + <Icon name="trash" /> Delete project{" "} + {render_delete_undelete_button()} + + + + {delete_message()} - {show_delete_conf && !project.get("deleted") ? ( - - {render_expanded_delete_info()} - - ) : undefined}
diff --git a/src/packages/frontend/project/warnings/deleted.tsx b/src/packages/frontend/project/warnings/deleted.tsx index e319cc7e1b..49b06b8ccb 100644 --- a/src/packages/frontend/project/warnings/deleted.tsx +++ b/src/packages/frontend/project/warnings/deleted.tsx @@ -3,8 +3,8 @@ * License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details */ -import { Alert } from "../../antd-bootstrap"; -import { Icon } from "../../components"; +import { Alert } from "@cocalc/frontend/antd-bootstrap"; +import { Icon } from "@cocalc/frontend/components"; // A warning to put on pages when the project is deleted export const DeletedProjectWarning: React.FC = () => { From 4826f843bf64c85adc2e7898a48dea9e64dde10d Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Wed, 17 Jul 2024 19:59:51 +0200 Subject: [PATCH 13/20] hub/delete-project: refactor, delete files, bugfixes --- src/packages/backend/misc.ts | 7 +++++ .../database/postgres/delete-projects.ts | 21 ++++++++++++-- .../server/projects/control/multi-user.ts | 23 +++++++-------- .../server/projects/control/single-user.ts | 2 +- src/packages/server/projects/control/util.ts | 29 +++++++++---------- src/packages/util/db-schema/site-defaults.ts | 7 +++-- 6 files changed, 54 insertions(+), 35 deletions(-) diff --git a/src/packages/backend/misc.ts b/src/packages/backend/misc.ts index 22f9c00f34..23198986a9 100644 --- a/src/packages/backend/misc.ts +++ b/src/packages/backend/misc.ts @@ -1,4 +1,6 @@ import { createHash } from "crypto"; + +import { projects } from "@cocalc/backend/data"; import { is_valid_uuid_string } from "@cocalc/util/misc"; /* @@ -69,3 +71,8 @@ export function envForSpawn() { } return env; } + +// return the absolute home directory of given @project_id project on disk +export function homePath(project_id: string): string { + return projects.replace("[project_id]", project_id); +} diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index ca01ef34be..d258a8cd6e 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -13,6 +13,7 @@ import { join } from "node:path"; import { pathToFiles } from "@cocalc/backend/files/path-to-files"; import getLogger from "@cocalc/backend/logger"; import { newCounter } from "@cocalc/backend/metrics"; +import { homePath } from "@cocalc/backend/misc"; import getPool from "@cocalc/database/pool"; import { getServerSettings } from "@cocalc/database/settings"; import { callback2 } from "@cocalc/util/async-utils"; @@ -82,7 +83,8 @@ export async function unlink_old_deleted_projects( db: PostgreSQL, age_d = 30, ): Promise { - await callback2(db._query, { + const L = log.extend("unlink_old_deleted_projects").debug; + const { rowCount } = await callback2(db._query, { query: "UPDATE projects", set: { users: null }, where: [ @@ -91,6 +93,7 @@ export async function unlink_old_deleted_projects( `last_edited <= NOW() - '${age_d} days'::INTERVAL`, ], }); + L("unlinked projects:", rowCount); } const Q_CLEANUP_SYNCSTRINGS = ` @@ -184,6 +187,13 @@ export async function cleanup_old_projects_data( ); if (on_prem) { + // we don't delete the central_log, it has its own expiration + // such an entry is good to have for reconstructing what really happened + db.log({ + event: "delete_project", + value: { deleting: "files", project_id }, + }); + L2(`delete all project files`); await deleteProjectFiles(L2, project_id); @@ -199,6 +209,10 @@ export async function cleanup_old_projects_data( // This gets rid of all sorts of data in tables specific to the given project. delRows += await delete_associated_project_data(L2, project_id); + db.log({ + event: "delete_project", + value: { deleting: "database", project_id }, + }); // now, that we're done with that project, mark it as state.state ->> 'deleted' // in addition to the flag "deleted = true". This also updates the state.time timestamp. @@ -283,7 +297,8 @@ async function delete_associated_project_data( async function deleteProjectFiles(L2, project_id: string) { // TODO: this only works on-prem, and requires the project files to be mounted - const projects_root = process.env["MOUNTED_PROJECTS_ROOT"]; + const projects_root = + process.env["MOUNTED_PROJECTS_ROOT"] ?? homePath(project_id); if (!projects_root) return; const project_dir = join(projects_root, project_id); try { @@ -297,7 +312,7 @@ async function deleteProjectFiles(L2, project_id: string) { } } catch (err) { L2( - `not deleting project files: either it does not exist or is not accessible`, + `not deleting project files: either path does not exist or is not accessible`, ); } } diff --git a/src/packages/server/projects/control/multi-user.ts b/src/packages/server/projects/control/multi-user.ts index bf39e6617b..7e3b983eb3 100644 --- a/src/packages/server/projects/control/multi-user.ts +++ b/src/packages/server/projects/control/multi-user.ts @@ -16,6 +16,15 @@ This code is very similar to single-user.ts, except with some small modifications due to having to create and delete Linux users. */ +import getLogger from "@cocalc/backend/logger"; +import { getUid, homePath } from "@cocalc/backend/misc"; +import { + BaseProject, + CopyOptions, + getProject, + ProjectState, + ProjectStatus, +} from "./base"; import { chown, copyPath, @@ -25,22 +34,12 @@ import { getEnvironment, getState, getStatus, - homePath, isProjectRunning, launchProjectDaemon, mkdir, setupDataPath, stopProjectProcesses, } from "./util"; -import { - BaseProject, - CopyOptions, - getProject, - ProjectStatus, - ProjectState, -} from "./base"; -import getLogger from "@cocalc/backend/logger"; -import { getUid } from "@cocalc/backend/misc"; const winston = getLogger("project-control:multi-user"); @@ -71,7 +70,7 @@ class Project extends BaseProject { const status = await getStatus(this.HOME); // TODO: don't include secret token in log message. winston.debug( - `got status of ${this.project_id} = ${JSON.stringify(status)}` + `got status of ${this.project_id} = ${JSON.stringify(status)}`, ); this.saveStatusToDatabase(status); return status; @@ -155,7 +154,7 @@ class Project extends BaseProject { await copyPath( opts, this.project_id, - opts.target_project_id ? getUid(opts.target_project_id) : undefined + opts.target_project_id ? getUid(opts.target_project_id) : undefined, ); return ""; } diff --git a/src/packages/server/projects/control/single-user.ts b/src/packages/server/projects/control/single-user.ts index 5e66721f59..26261d3bbb 100644 --- a/src/packages/server/projects/control/single-user.ts +++ b/src/packages/server/projects/control/single-user.ts @@ -19,6 +19,7 @@ This is useful for: import { kill } from "process"; import getLogger from "@cocalc/backend/logger"; +import { homePath } from "@cocalc/backend/misc"; import { BaseProject, CopyOptions, @@ -33,7 +34,6 @@ import { getProjectPID, getState, getStatus, - homePath, isProjectRunning, launchProjectDaemon, mkdir, diff --git a/src/packages/server/projects/control/util.ts b/src/packages/server/projects/control/util.ts index d3e9ced692..8566da4eb1 100644 --- a/src/packages/server/projects/control/util.ts +++ b/src/packages/server/projects/control/util.ts @@ -1,19 +1,20 @@ -import { promisify } from "util"; -import { dirname, join, resolve } from "path"; -import { exec as exec0, spawn } from "child_process"; import spawnAsync from "await-spawn"; -import * as fs from "fs"; -import { writeFile } from "fs/promises"; -import { projects, root, blobstore } from "@cocalc/backend/data"; -import { is_valid_uuid_string } from "@cocalc/util/misc"; -import { callback2 } from "@cocalc/util/async-utils"; -import getLogger from "@cocalc/backend/logger"; -import { CopyOptions, ProjectState, ProjectStatus } from "./base"; -import { getUid } from "@cocalc/backend/misc"; +import { exec as exec0, spawn } from "node:child_process"; +import * as fs from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { promisify } from "node:util"; + import base_path from "@cocalc/backend/base-path"; +import { blobstore, root } from "@cocalc/backend/data"; +import getLogger from "@cocalc/backend/logger"; +import { getUid, homePath } from "@cocalc/backend/misc"; import { db } from "@cocalc/database"; -import { getProject } from "."; +import { callback2 } from "@cocalc/util/async-utils"; +import { is_valid_uuid_string } from "@cocalc/util/misc"; import { pidFilename, pidUpdateIntervalMs } from "@cocalc/util/project-info"; +import { getProject } from "."; +import { CopyOptions, ProjectState, ProjectStatus } from "./base"; const logger = getLogger("project-control:util"); @@ -31,10 +32,6 @@ export function dataPath(HOME: string): string { return join(HOME, ".smc"); } -export function homePath(project_id: string): string { - return projects.replace("[project_id]", project_id); -} - export function getUsername(project_id: string): string { return project_id.split("-").join(""); } diff --git a/src/packages/util/db-schema/site-defaults.ts b/src/packages/util/db-schema/site-defaults.ts index ddfe3889dd..a97c1fef30 100644 --- a/src/packages/util/db-schema/site-defaults.ts +++ b/src/packages/util/db-schema/site-defaults.ts @@ -653,10 +653,11 @@ export const site_settings_conf: SiteSettings = { to_val: split_iframe_comm_hosts, to_display: num_dns_hosts, }, - delete_project_data : { - name :"Delete Project Data", - desc: "When a project has been marked as deleted, also actually delete associated data from the database and (for OnPrem) also its files.", + delete_project_data: { + name: "Delete Project Data", + desc: "When a project has been marked as deleted, also actually delete associated data from the database and (OnPrem only) also its files.", default: "no", + valid: only_booleans, to_val: to_bool, }, email_enabled: { From 55d2a118b0d2f9762f8057b747e9a9ccd9c76748 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Thu, 18 Jul 2024 15:29:40 +0200 Subject: [PATCH 14/20] backend/logger: tighter typing --- src/packages/backend/logger.ts | 54 ++++++++++--------- .../database/postgres/delete-projects.ts | 18 ++++--- src/packages/project/project-status/server.ts | 11 ++-- src/packages/project/usage-info/server.ts | 8 +-- src/packages/server/software-envs.ts | 3 +- src/packages/util/db-schema/site-defaults.ts | 2 +- 6 files changed, 52 insertions(+), 44 deletions(-) diff --git a/src/packages/backend/logger.ts b/src/packages/backend/logger.ts index bc4a9ded88..04454929d7 100644 --- a/src/packages/backend/logger.ts +++ b/src/packages/backend/logger.ts @@ -12,9 +12,11 @@ process.env.DEBUG_HIDE_DATE = "yes"; // since we supply it ourselves // otherwise, maybe stuff like this works: (debug as any).inspectOpts["hideDate"] = true; import debug, { Debugger } from "debug"; -import { mkdirSync, createWriteStream, statSync, ftruncate } from "fs"; -import { format } from "util"; -import { dirname, join } from "path"; + +import { createWriteStream, ftruncate, mkdirSync, statSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { format } from "node:util"; + import { logs } from "./data"; const MAX_FILE_SIZE_BYTES = 20 * 1024 * 1024; // 20MB @@ -25,12 +27,12 @@ let _trimLogFileSizePath = ""; export function trimLogFileSize() { // THIS JUST DOESN'T REALLY WORK! return; - + if (!_trimLogFileSizePath) return; let stats; try { stats = statSync(_trimLogFileSizePath); - } catch(_) { + } catch (_) { // this happens if the file doesn't exist, which is fine since "trimming" it would be a no-op return; } @@ -141,19 +143,7 @@ function initTransports() { initTransports(); -const DEBUGGERS = { - error: COCALC.extend("error"), - warn: COCALC.extend("warn"), - info: COCALC.extend("info"), - http: COCALC.extend("http"), - verbose: COCALC.extend("verbose"), - debug: COCALC.extend("debug"), - silly: COCALC.extend("silly"), -}; - -type Level = keyof typeof DEBUGGERS; - -const LEVELS: Level[] = [ +const LEVELS = [ "error", "warn", "info", @@ -161,7 +151,19 @@ const LEVELS: Level[] = [ "verbose", "debug", "silly", -]; +] as const; + +type Level = (typeof LEVELS)[number]; + +const DEBUGGERS: { [key in Level]: Debugger } = { + error: COCALC.extend("error"), + warn: COCALC.extend("warn"), + info: COCALC.extend("info"), + http: COCALC.extend("http"), + verbose: COCALC.extend("verbose"), + debug: COCALC.extend("debug"), + silly: COCALC.extend("silly"), +} as const; class Logger { private name: string; @@ -194,13 +196,13 @@ class Logger { } export interface WinstonLogger { - error: Function; - warn: Function; - info: Function; - http: Function; - verbose: Function; - debug: Function; - silly: Function; + error: Debugger; + warn: Debugger; + info: Debugger; + http: Debugger; + verbose: Debugger; + debug: Debugger; + silly: Debugger; extend: (name: string) => WinstonLogger; isEnabled: (level: Level) => boolean; } diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index d258a8cd6e..fbbdfc2da8 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -11,7 +11,7 @@ import { promises as fs } from "node:fs"; import { join } from "node:path"; import { pathToFiles } from "@cocalc/backend/files/path-to-files"; -import getLogger from "@cocalc/backend/logger"; +import getLogger, { WinstonLogger } from "@cocalc/backend/logger"; import { newCounter } from "@cocalc/backend/metrics"; import { homePath } from "@cocalc/backend/misc"; import getPool from "@cocalc/database/pool"; @@ -22,6 +22,8 @@ import { minutes_ago } from "@cocalc/util/misc"; import { bulkDelete } from "./bulk-delete"; import { PostgreSQL } from "./types"; +const { F_OK, R_OK, W_OK } = fs.constants; + const log = getLogger("db:delete-projects"); const delete_projects_prom = newCounter( @@ -197,10 +199,10 @@ export async function cleanup_old_projects_data( L2(`delete all project files`); await deleteProjectFiles(L2, project_id); - L2(`deleting all shared files`); try { // this is something like /shared/projects/${project_id} const shared_path = pathToFiles(project_id, ""); + L2(`deleting all shared files in ${shared_path}`); await fs.rm(shared_path, { recursive: true, force: true }); } catch (err) { L2(`Unable to delete shared files: ${err}`); @@ -233,7 +235,7 @@ export async function cleanup_old_projects_data( } async function delete_associated_project_data( - L2, + L2: WinstonLogger["debug"], project_id: string, ): Promise { let total = 0; @@ -295,14 +297,18 @@ async function delete_associated_project_data( return total; } -async function deleteProjectFiles(L2, project_id: string) { - // TODO: this only works on-prem, and requires the project files to be mounted +async function deleteProjectFiles( + L2: WinstonLogger["debug"], + project_id: string, +) { + // $MOUNTED_PROJECTS_ROOT is for OnPrem and homePath only works in dev/single-user const projects_root = process.env["MOUNTED_PROJECTS_ROOT"] ?? homePath(project_id); if (!projects_root) return; const project_dir = join(projects_root, project_id); + L2(`attempting to delete all files in ${project_dir}`); try { - await fs.access(project_dir, fs.constants.F_OK | fs.constants.R_OK); + await fs.access(project_dir, F_OK | R_OK | W_OK); const stats = await fs.lstat(project_dir); if (stats.isDirectory()) { L2(`deleting all files in ${project_dir}`); diff --git a/src/packages/project/project-status/server.ts b/src/packages/project/project-status/server.ts index cc5e798c85..893c73ff92 100644 --- a/src/packages/project/project-status/server.ts +++ b/src/packages/project/project-status/server.ts @@ -15,13 +15,10 @@ status updates. Hence in particular, information like cpu, memory and disk are smoothed out and throttled. */ -import { getLogger } from "@cocalc/project/logger"; -import { how_long_ago_m, round1 } from "@cocalc/util/misc"; -import { version as smcVersion } from "@cocalc/util/smc-version"; import { delay } from "awaiting"; import { EventEmitter } from "events"; import { isEqual } from "lodash"; -import { get_ProjectInfoServer, ProjectInfoServer } from "../project-info"; + import { ProjectInfo } from "@cocalc/comm/project-info/types"; import { ALERT_DISK_FREE, @@ -36,6 +33,10 @@ import { ProjectStatus, } from "@cocalc/comm/project-status/types"; import { cgroup_stats } from "@cocalc/comm/project-status/utils"; +import { getLogger } from "@cocalc/project/logger"; +import { how_long_ago_m, round1 } from "@cocalc/util/misc"; +import { version as smcVersion } from "@cocalc/util/smc-version"; +import { get_ProjectInfoServer, ProjectInfoServer } from "../project-info"; // TODO: only return the "next" value, if it is significantly different from "prev" //function threshold(prev?: number, next?: number): number | undefined { @@ -83,7 +84,7 @@ export class ProjectStatusServer extends EventEmitter { constructor(testing = false) { super(); this.testing = testing; - this.dbg = (...msg) => winston.debug(...msg); + this.dbg = (...msg) => winston.debug(msg[0], ...msg.slice(1)); this.project_info = get_ProjectInfoServer(); } diff --git a/src/packages/project/usage-info/server.ts b/src/packages/project/usage-info/server.ts index 1c8994db71..2c46af1c55 100644 --- a/src/packages/project/usage-info/server.ts +++ b/src/packages/project/usage-info/server.ts @@ -12,17 +12,17 @@ from the ProjectInfoServer (which collects data about everything) */ import { delay } from "awaiting"; +import { throttle } from "lodash"; import { EventEmitter } from "node:events"; -import { getLogger } from "../logger"; -import { ProjectInfoServer, get_ProjectInfoServer } from "../project-info"; import { Process, ProjectInfo } from "@cocalc/comm/project-info/types"; import type { UsageInfo } from "@cocalc/util/types/project-usage-info"; -import { throttle } from "lodash"; +import { getLogger } from "../logger"; +import { ProjectInfoServer, get_ProjectInfoServer } from "../project-info"; const L = getLogger("usage-info:server").debug; -const throttled_dbg = throttle((...args) => L(...args), 10000); +const throttled_dbg = throttle(L, 10000); function is_diff(prev: UsageInfo, next: UsageInfo, key: keyof UsageInfo) { // we assume a,b >= 0, hence we leave out Math.abs operations diff --git a/src/packages/server/software-envs.ts b/src/packages/server/software-envs.ts index a09f6b106d..4e17cbb8ab 100644 --- a/src/packages/server/software-envs.ts +++ b/src/packages/server/software-envs.ts @@ -65,8 +65,7 @@ async function readConfig(purpose: Purpose): Promise { // parse the content of softwareFn as json try { const software = JSON.parse((await readFile(softwareFn)).toString()); - const dbg = (...msg) => L(...msg); - const sanitized = sanitizeSoftwareEnv({ software, registry, purpose }, dbg); + const sanitized = sanitizeSoftwareEnv({ software, registry, purpose }, L); return sanitized; } catch (err) { W(`WARNING: ${softwareFn} is not a valid JSON file -- ${err}`); diff --git a/src/packages/util/db-schema/site-defaults.ts b/src/packages/util/db-schema/site-defaults.ts index a97c1fef30..5d1d52ea86 100644 --- a/src/packages/util/db-schema/site-defaults.ts +++ b/src/packages/util/db-schema/site-defaults.ts @@ -655,7 +655,7 @@ export const site_settings_conf: SiteSettings = { }, delete_project_data: { name: "Delete Project Data", - desc: "When a project has been marked as deleted, also actually delete associated data from the database and (OnPrem only) also its files.", + desc: "When a project has been marked as deleted, also actually delete associated data from the database and – for OnPrem and single-user dev mode only – also its files.", default: "no", valid: only_booleans, to_val: to_bool, From 33ddc7a29878092e8b2c26c65f4b67de6fa2bc3e Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Thu, 18 Jul 2024 15:48:24 +0200 Subject: [PATCH 15/20] hub/delete-projects: fix homePath, to make it work for the OnPrem situation --- src/packages/backend/misc.ts | 9 ++++++++- src/packages/database/postgres/delete-projects.ts | 10 ++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/packages/backend/misc.ts b/src/packages/backend/misc.ts index 23198986a9..779843e7c8 100644 --- a/src/packages/backend/misc.ts +++ b/src/packages/backend/misc.ts @@ -1,4 +1,5 @@ import { createHash } from "crypto"; +import { join } from "node:path"; import { projects } from "@cocalc/backend/data"; import { is_valid_uuid_string } from "@cocalc/util/misc"; @@ -74,5 +75,11 @@ export function envForSpawn() { // return the absolute home directory of given @project_id project on disk export function homePath(project_id: string): string { - return projects.replace("[project_id]", project_id); + // $MOUNTED_PROJECTS_ROOT is for OnPrem and that "projects" location is only for dev/single-user + const projects_root = process.env.MOUNTED_PROJECTS_ROOT; + if (projects_root) { + return join(projects_root, project_id); + } else { + return projects.replace("[project_id]", project_id); + } } diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index fbbdfc2da8..a71b5d6c4b 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -8,7 +8,6 @@ Code related to permanently deleting projects. */ import { promises as fs } from "node:fs"; -import { join } from "node:path"; import { pathToFiles } from "@cocalc/backend/files/path-to-files"; import getLogger, { WinstonLogger } from "@cocalc/backend/logger"; @@ -301,12 +300,7 @@ async function deleteProjectFiles( L2: WinstonLogger["debug"], project_id: string, ) { - // $MOUNTED_PROJECTS_ROOT is for OnPrem and homePath only works in dev/single-user - const projects_root = - process.env["MOUNTED_PROJECTS_ROOT"] ?? homePath(project_id); - if (!projects_root) return; - const project_dir = join(projects_root, project_id); - L2(`attempting to delete all files in ${project_dir}`); + const project_dir = homePath(project_id); try { await fs.access(project_dir, F_OK | R_OK | W_OK); const stats = await fs.lstat(project_dir); @@ -318,7 +312,7 @@ async function deleteProjectFiles( } } catch (err) { L2( - `not deleting project files: either path does not exist or is not accessible`, + `not deleting project files: either '${project_dir}' does not exist or is not accessible`, ); } } From 8d3877f28d90b0db9711a6031af0b8343a6c7e63 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Tue, 30 Jul 2024 13:58:14 +0200 Subject: [PATCH 16/20] hub/delete-projects: also blobs table --- src/packages/database/postgres/delete-projects.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index 1ca8b6f713..9620c337f2 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -247,6 +247,7 @@ async function delete_associated_project_data( "file_access_log", "jupyter_api_log", "openai_chatgpt_log", + "blobs", ] as const; for (const table of tables) { @@ -256,7 +257,7 @@ async function delete_associated_project_data( value: project_id, }); total += rowsDeleted; - L2(`deleted ${table} ${rowsDeleted} entries`); + L2(`deleted in ${table}: ${rowsDeleted} entries`); } // these tables are different, i.e. another id, or the field to check the project_id value against is called differently @@ -268,7 +269,7 @@ async function delete_associated_project_data( value: project_id, }); total += rowsDeleted; - L2(`deleted copy_paths/${field} ${rowsDeleted} entries`); + L2(`deleted copy_paths/${field}: ${rowsDeleted} entries`); } { @@ -279,7 +280,7 @@ async function delete_associated_project_data( value: project_id, }); total += rowsDeleted; - L2(`deleted ${rowsDeleted} listings`); + L2(`deleted in listings: ${rowsDeleted} entries`); } { @@ -290,7 +291,7 @@ async function delete_associated_project_data( id: "token", }); total += rowsDeleted; - L2(`deleted ${rowsDeleted} entries`); + L2(`deleted in project_invite_tokens: ${rowsDeleted} entries`); } return total; From 1d0665616e8f42b7b07a731aeeff2912c44da1ca Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Tue, 15 Oct 2024 10:55:14 +0200 Subject: [PATCH 17/20] util/compute-states: catch up new project state with translations --- src/packages/frontend/i18n/README.md | 10 ++++++---- src/packages/frontend/i18n/trans/ar_EG.json | 2 ++ src/packages/frontend/i18n/trans/de_DE.json | 2 ++ src/packages/frontend/i18n/trans/es_ES.json | 2 ++ src/packages/frontend/i18n/trans/fr_FR.json | 2 ++ src/packages/frontend/i18n/trans/he_IL.json | 2 ++ src/packages/frontend/i18n/trans/hi_IN.json | 2 ++ src/packages/frontend/i18n/trans/hu_HU.json | 2 ++ src/packages/frontend/i18n/trans/it_IT.json | 2 ++ src/packages/frontend/i18n/trans/ja_JP.json | 2 ++ src/packages/frontend/i18n/trans/ko_KR.json | 2 ++ src/packages/frontend/i18n/trans/pl_PL.json | 2 ++ src/packages/frontend/i18n/trans/pt_PT.json | 2 ++ src/packages/frontend/i18n/trans/ru_RU.json | 2 ++ src/packages/frontend/i18n/trans/tr_TR.json | 2 ++ src/packages/frontend/i18n/trans/zh_CN.json | 2 ++ src/packages/util/compute-states.ts | 10 ++++++++-- 17 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/packages/frontend/i18n/README.md b/src/packages/frontend/i18n/README.md index c337b9ad74..4f5044ccb5 100644 --- a/src/packages/frontend/i18n/README.md +++ b/src/packages/frontend/i18n/README.md @@ -41,15 +41,17 @@ After introducing new messages, these are the steps to get all translations into 1. `pnpm i18n:download` - Will grab the updated files like `zh_CN.json` and save them in the `i18n` folder. + Will grab the updated files containing the translated strings (e.g. `zh_CN.json`) and save them in the `./i18n/trans/` folder. + The source of truth for these files is always the remotely stored data – hence do not ever edit these files directly. 1. `pnpm i18n:compile` - This transforms the `[locale].json` files to `[locale].compiles.json`. + This transforms the `[locale].json` translation files from the step before to `[locale].compiled.json`. This could also reveal problems, when conditional ICU messages aren't properly formatted. - E.g. `"Sí, cerrar sesión{en todas partes, seleccionar, verdadero { en todas partes} otro {}}" with ID "account.sign-out.button.ok" in file "./i18n/es_ES.json"`: In the brackets, it has to start according to the syntax: `{everywhere, select, true {..} other {}}`. + E.g. `"Sí, cerrar sesión{en todas partes, seleccionar, verdadero { en todas partes} otro {}}" with ID "account.sign-out.button.ok" in file "./i18n/es_ES.json"`: + In the brackets, it has to start according to the syntax: `{everywhere, select, true {..} other {}}`, i.e. the variable `everywhere` must stay in English. -1. Reload the `frontend` after a compile, such that `await import...` will load the updated translation file for the set locale. +1. Reload the `frontend` after a compile, such that `await import...` will load the updated compiled translation file for the configured locale. Note: if just a translation has been updated, you only need to do the `i18n:download` & `i18n:compile` steps. diff --git a/src/packages/frontend/i18n/trans/ar_EG.json b/src/packages/frontend/i18n/trans/ar_EG.json index 932c57cd58..55556b99a4 100644 --- a/src/packages/frontend/i18n/trans/ar_EG.json +++ b/src/packages/frontend/i18n/trans/ar_EG.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "مغلق", "util.compute-states.closing.desc": "المشروع في طور الإغلاق.", "util.compute-states.closing.display": "إغلاق", + "util.compute-states.deleted.desc": "تم حذف المشروع", + "util.compute-states.deleted.display": "محذوف", "util.compute-states.opened.desc": "المشروع متاح وجاهز للتجربة والتشغيل.", "util.compute-states.opened.display": "توقف", "util.compute-states.opening.desc": "يتم استيراد المشروع؛ قد يستغرق هذا عدة دقائق حسب الحجم.", diff --git a/src/packages/frontend/i18n/trans/de_DE.json b/src/packages/frontend/i18n/trans/de_DE.json index ed5dee032d..7859e993b6 100644 --- a/src/packages/frontend/i18n/trans/de_DE.json +++ b/src/packages/frontend/i18n/trans/de_DE.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "Geschlossen", "util.compute-states.closing.desc": "Das Projekt wird gerade geschlossen.", "util.compute-states.closing.display": "Schließen", + "util.compute-states.deleted.desc": "Projekt ist gelöscht", + "util.compute-states.deleted.display": "Gelöscht", "util.compute-states.opened.desc": "Projekt ist verfügbar und bereit zum Ausführen.", "util.compute-states.opened.display": "Gestoppt", "util.compute-states.opening.desc": "Projekt wird importiert; dies kann je nach Größe mehrere Minuten dauern.", diff --git a/src/packages/frontend/i18n/trans/es_ES.json b/src/packages/frontend/i18n/trans/es_ES.json index 2a142d0ade..ab241dc01d 100644 --- a/src/packages/frontend/i18n/trans/es_ES.json +++ b/src/packages/frontend/i18n/trans/es_ES.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "Cerrado", "util.compute-states.closing.desc": "El proyecto está en proceso de cerrarse.", "util.compute-states.closing.display": "Cerrando", + "util.compute-states.deleted.desc": "El proyecto se elimina", + "util.compute-states.deleted.display": "Eliminado", "util.compute-states.opened.desc": "El proyecto está disponible y listo para intentar ejecutar.", "util.compute-states.opened.display": "Detenido", "util.compute-states.opening.desc": "El proyecto se está importando; esto puede tardar varios minutos dependiendo del tamaño.", diff --git a/src/packages/frontend/i18n/trans/fr_FR.json b/src/packages/frontend/i18n/trans/fr_FR.json index 2ac1e8772a..2aa978f34d 100644 --- a/src/packages/frontend/i18n/trans/fr_FR.json +++ b/src/packages/frontend/i18n/trans/fr_FR.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "Fermé", "util.compute-states.closing.desc": "Le projet est en cours de fermeture.", "util.compute-states.closing.display": "Fermeture", + "util.compute-states.deleted.desc": "Le projet est supprimé", + "util.compute-states.deleted.display": "Supprimé", "util.compute-states.opened.desc": "Le projet est disponible et prêt à essayer de fonctionner.", "util.compute-states.opened.display": "Arrêté", "util.compute-states.opening.desc": "Le projet est en cours d'importation ; cela peut prendre plusieurs minutes selon la taille.", diff --git a/src/packages/frontend/i18n/trans/he_IL.json b/src/packages/frontend/i18n/trans/he_IL.json index 5b1015a9ee..d1ba821922 100644 --- a/src/packages/frontend/i18n/trans/he_IL.json +++ b/src/packages/frontend/i18n/trans/he_IL.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "סגור", "util.compute-states.closing.desc": "הפרויקט בתהליך סגירה", "util.compute-states.closing.display": "סגירה", + "util.compute-states.deleted.desc": "הפרויקט נמחק", + "util.compute-states.deleted.display": "נמחק", "util.compute-states.opened.desc": "הפרויקט זמין ומוכן לנסות להריץ.", "util.compute-states.opened.display": "נעצר", "util.compute-states.opening.desc": "הפרויקט מיובא; זה עשוי לקחת מספר דקות בהתאם לגודל.", diff --git a/src/packages/frontend/i18n/trans/hi_IN.json b/src/packages/frontend/i18n/trans/hi_IN.json index 180fb287cc..c3a1476654 100644 --- a/src/packages/frontend/i18n/trans/hi_IN.json +++ b/src/packages/frontend/i18n/trans/hi_IN.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "बंद", "util.compute-states.closing.desc": "प्रोजेक्ट को बंद किया जा रहा है।", "util.compute-states.closing.display": "समाप्त कर रहा है", + "util.compute-states.deleted.desc": "प्रोजेक्ट हटा दिया गया है", + "util.compute-states.deleted.display": "हटाया गया", "util.compute-states.opened.desc": "प्रोजेक्ट उपलब्ध है और चलाने के लिए तैयार है.", "util.compute-states.opened.display": "रुका हुआ", "util.compute-states.opening.desc": "प्रोजेक्ट को आयात किया जा रहा है; आकार के अनुसार इसमें कुछ मिनट लग सकते हैं", diff --git a/src/packages/frontend/i18n/trans/hu_HU.json b/src/packages/frontend/i18n/trans/hu_HU.json index 2a81a1090b..c6406b49a3 100644 --- a/src/packages/frontend/i18n/trans/hu_HU.json +++ b/src/packages/frontend/i18n/trans/hu_HU.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "Zárva", "util.compute-states.closing.desc": "A projekt lezárás alatt áll.", "util.compute-states.closing.display": "Bezárás", + "util.compute-states.deleted.desc": "A projekt törölve van", + "util.compute-states.deleted.display": "Törölve", "util.compute-states.opened.desc": "A projekt elérhető és készen áll a futtatásra.", "util.compute-states.opened.display": "Leállítva", "util.compute-states.opening.desc": "A projekt importálása folyamatban van; ez néhány percig is eltarthat a mérettől függően.", diff --git a/src/packages/frontend/i18n/trans/it_IT.json b/src/packages/frontend/i18n/trans/it_IT.json index c21bf80843..215448190f 100644 --- a/src/packages/frontend/i18n/trans/it_IT.json +++ b/src/packages/frontend/i18n/trans/it_IT.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "Chiuso", "util.compute-states.closing.desc": "Il progetto è in fase di chiusura", "util.compute-states.closing.display": "Chiusura", + "util.compute-states.deleted.desc": "Il progetto è eliminato", + "util.compute-states.deleted.display": "Eliminato", "util.compute-states.opened.desc": "Il progetto è disponibile e pronto per essere eseguito.", "util.compute-states.opened.display": "Fermato", "util.compute-states.opening.desc": "Il progetto è in fase di importazione; potrebbero volerci diversi minuti a seconda delle dimensioni.", diff --git a/src/packages/frontend/i18n/trans/ja_JP.json b/src/packages/frontend/i18n/trans/ja_JP.json index e4e6d132e4..05abbcdd43 100644 --- a/src/packages/frontend/i18n/trans/ja_JP.json +++ b/src/packages/frontend/i18n/trans/ja_JP.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "閉じた", "util.compute-states.closing.desc": "プロジェクトは閉鎖中です", "util.compute-states.closing.display": "終了", + "util.compute-states.deleted.desc": "プロジェクトは削除されました", + "util.compute-states.deleted.display": "削除しました", "util.compute-states.opened.desc": "プロジェクトは利用可能で、実行準備ができています。", "util.compute-states.opened.display": "停止", "util.compute-states.opening.desc": "プロジェクトをインポート中です。サイズによっては数分かかる場合があります。", diff --git a/src/packages/frontend/i18n/trans/ko_KR.json b/src/packages/frontend/i18n/trans/ko_KR.json index 568b89e549..22622b92cb 100644 --- a/src/packages/frontend/i18n/trans/ko_KR.json +++ b/src/packages/frontend/i18n/trans/ko_KR.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "닫힘", "util.compute-states.closing.desc": "프로젝트가 닫히는 중입니다.", "util.compute-states.closing.display": "닫는 중", + "util.compute-states.deleted.desc": "프로젝트가 삭제되었습니다", + "util.compute-states.deleted.display": "삭제됨", "util.compute-states.opened.desc": "프로젝트가 사용 가능하며 실행할 준비가 되었습니다.", "util.compute-states.opened.display": "중지됨", "util.compute-states.opening.desc": "프로젝트가 가져오는 중입니다. 크기에 따라 몇 분 정도 걸릴 수 있습니다.", diff --git a/src/packages/frontend/i18n/trans/pl_PL.json b/src/packages/frontend/i18n/trans/pl_PL.json index e1e8fb08b9..2744017f82 100644 --- a/src/packages/frontend/i18n/trans/pl_PL.json +++ b/src/packages/frontend/i18n/trans/pl_PL.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "Zamknięte", "util.compute-states.closing.desc": "Projekt jest w trakcie zamykania.", "util.compute-states.closing.display": "Zamykanie", + "util.compute-states.deleted.desc": "Projekt został usunięty", + "util.compute-states.deleted.display": "Usunięto", "util.compute-states.opened.desc": "Projekt jest dostępny i gotowy do uruchomienia.", "util.compute-states.opened.display": "Zatrzymano", "util.compute-states.opening.desc": "Projekt jest importowany; może to potrwać kilka minut w zależności od rozmiaru.", diff --git a/src/packages/frontend/i18n/trans/pt_PT.json b/src/packages/frontend/i18n/trans/pt_PT.json index 49462f3023..ceabdbca15 100644 --- a/src/packages/frontend/i18n/trans/pt_PT.json +++ b/src/packages/frontend/i18n/trans/pt_PT.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "Fechado", "util.compute-states.closing.desc": "O projeto está em processo de encerramento.", "util.compute-states.closing.display": "Fechar", + "util.compute-states.deleted.desc": "O projeto foi eliminado", + "util.compute-states.deleted.display": "Eliminado", "util.compute-states.opened.desc": "O projeto está disponível e pronto para tentar executar.", "util.compute-states.opened.display": "Parado", "util.compute-states.opening.desc": "O projeto está a ser importado; isto pode demorar vários minutos dependendo do tamanho.", diff --git a/src/packages/frontend/i18n/trans/ru_RU.json b/src/packages/frontend/i18n/trans/ru_RU.json index c25309ffa6..2fcd4920fc 100644 --- a/src/packages/frontend/i18n/trans/ru_RU.json +++ b/src/packages/frontend/i18n/trans/ru_RU.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "Закрыто", "util.compute-states.closing.desc": "Проект находится в процессе закрытия.", "util.compute-states.closing.display": "Закрытие", + "util.compute-states.deleted.desc": "Проект удален", + "util.compute-states.deleted.display": "Удалено", "util.compute-states.opened.desc": "Проект доступен и готов к запуску.", "util.compute-states.opened.display": "Остановлено", "util.compute-states.opening.desc": "Проект импортируется; это может занять несколько минут в зависимости от размера.", diff --git a/src/packages/frontend/i18n/trans/tr_TR.json b/src/packages/frontend/i18n/trans/tr_TR.json index 4a3503745a..7a22bef5a2 100644 --- a/src/packages/frontend/i18n/trans/tr_TR.json +++ b/src/packages/frontend/i18n/trans/tr_TR.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "Kapalı", "util.compute-states.closing.desc": "Proje kapatılma sürecinde.", "util.compute-states.closing.display": "Kapatılıyor", + "util.compute-states.deleted.desc": "Proje silindi", + "util.compute-states.deleted.display": "Silindi", "util.compute-states.opened.desc": "Proje mevcut ve çalıştırılmaya hazır.", "util.compute-states.opened.display": "Durduruldu", "util.compute-states.opening.desc": "Proje içe aktarılıyor; boyutuna bağlı olarak bu birkaç dakika sürebilir.", diff --git a/src/packages/frontend/i18n/trans/zh_CN.json b/src/packages/frontend/i18n/trans/zh_CN.json index 416b334450..bb2465076b 100644 --- a/src/packages/frontend/i18n/trans/zh_CN.json +++ b/src/packages/frontend/i18n/trans/zh_CN.json @@ -931,6 +931,8 @@ "util.compute-states.closed.display": "已关闭", "util.compute-states.closing.desc": "项目正在关闭中", "util.compute-states.closing.display": "关闭", + "util.compute-states.deleted.desc": "项目已删除", + "util.compute-states.deleted.display": "已删除", "util.compute-states.opened.desc": "项目已可用并准备运行。", "util.compute-states.opened.display": "已停止", "util.compute-states.opening.desc": "项目正在导入中；根据大小可能需要几分钟。", diff --git a/src/packages/util/compute-states.ts b/src/packages/util/compute-states.ts index cbb42f1274..79b5f45de2 100644 --- a/src/packages/util/compute-states.ts +++ b/src/packages/util/compute-states.ts @@ -304,9 +304,15 @@ export const COMPUTE_STATES: ComputeStates = { // projects are deleted in hub -> postgres.delete-projects and this is a one-way operation deleted: { - desc: "Project is deleted", + desc: defineMessage({ + id: "util.compute-states.deleted.desc", + defaultMessage: "Project is deleted", + }), icon: "trash", - display: "Deleted", + display: defineMessage({ + id: "util.compute-states.deleted.display", + defaultMessage: "Deleted", + }), stable: true, to: {}, commands: [], From d1989d89f52dc25a1367b7f144dbce6d94c5a9c0 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Tue, 15 Oct 2024 11:29:57 +0200 Subject: [PATCH 18/20] frontend/project: translate project deleted banner and make it a proper banner --- src/packages/frontend/i18n/trans/ar_EG.json | 1 + src/packages/frontend/i18n/trans/de_DE.json | 1 + src/packages/frontend/i18n/trans/es_ES.json | 1 + src/packages/frontend/i18n/trans/fr_FR.json | 1 + src/packages/frontend/i18n/trans/he_IL.json | 1 + src/packages/frontend/i18n/trans/hi_IN.json | 1 + src/packages/frontend/i18n/trans/hu_HU.json | 1 + src/packages/frontend/i18n/trans/it_IT.json | 1 + src/packages/frontend/i18n/trans/ja_JP.json | 1 + src/packages/frontend/i18n/trans/ko_KR.json | 1 + src/packages/frontend/i18n/trans/pl_PL.json | 1 + src/packages/frontend/i18n/trans/pt_PT.json | 1 + src/packages/frontend/i18n/trans/ru_RU.json | 1 + src/packages/frontend/i18n/trans/tr_TR.json | 1 + src/packages/frontend/i18n/trans/zh_CN.json | 1 + .../frontend/project/warnings/deleted.tsx | 22 +++++++++++-------- 16 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/packages/frontend/i18n/trans/ar_EG.json b/src/packages/frontend/i18n/trans/ar_EG.json index 55556b99a4..8787b606c1 100644 --- a/src/packages/frontend/i18n/trans/ar_EG.json +++ b/src/packages/frontend/i18n/trans/ar_EG.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {بدء المشروع} other {ابدأ المشروع}}", "project.start-button.trial.description": "لا توجد سعة لمشاريع التجربة المجانية على CoCalc في الوقت الحالي. {br} قم بترقية مشروعك باستخدام رخصة أو {A4}.", "project.start-button.trial.message": "عدد كبير جدًا من المشاريع التجريبية المجانية", + "project.warnings.deleted.banner": "

{icon} تحذير: هذا المشروع محذوف!

إذا كنت تنوي استخدام هذا المشروع، ينبغي عليك استعادته في إعدادات المشروع.", "projects.filename-search.placeholder": "ابحث عن أسماء الملفات التي قمت بتحريرها...", "projects.load-all.label": "عرض كل المشاريع...", "projects.search.placeholder": "ابحث عن المشاريع (استخدم /re/ للبحث المنتظم)...", diff --git a/src/packages/frontend/i18n/trans/de_DE.json b/src/packages/frontend/i18n/trans/de_DE.json index 7859e993b6..133ea27cbc 100644 --- a/src/packages/frontend/i18n/trans/de_DE.json +++ b/src/packages/frontend/i18n/trans/de_DE.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {Projekt wird gestartet} other {Projekt starten}}", "project.start-button.trial.description": "Es gibt derzeit keine Kapazität mehr für Free Trial Projekte auf CoCalc. {br} Upgrade dieses Projekt mit einer Lizenz oder {A4}.", "project.start-button.trial.message": "Zu viele kostenlose Probeprojekte", + "project.warnings.deleted.banner": "

{icon} Warnung: dieses Projekt ist gelöscht!

Wenn Sie dieses Projekt nutzen möchten, sollten Sie es in den Projekteinstellungen wiederherstellen.", "projects.filename-search.placeholder": "Suche nach Dateien, die Sie bearbeitet haben...", "projects.load-all.label": "Zeige alle Projekte...", "projects.search.placeholder": "Suche nach Projekten (verwende /re/ für reguläre Ausdrücke)", diff --git a/src/packages/frontend/i18n/trans/es_ES.json b/src/packages/frontend/i18n/trans/es_ES.json index ab241dc01d..2045ff10bf 100644 --- a/src/packages/frontend/i18n/trans/es_ES.json +++ b/src/packages/frontend/i18n/trans/es_ES.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {Iniciando proyecto} other {Iniciar proyecto}}", "project.start-button.trial.description": "No hay más capacidad para proyectos de Prueba Gratuita en CoCalc en este momento. {br} Actualiza tu proyecto usando una licencia o {A4}.", "project.start-button.trial.message": "Demasiados Proyectos de Prueba Gratuita", + "project.warnings.deleted.banner": "

{icon} Advertencia: este proyecto está ¡eliminado!

Si tienes la intención de usar este proyecto, debes recuperarlo en la configuración del proyecto.", "projects.filename-search.placeholder": "Buscar nombres de archivos que editaste", "projects.load-all.label": "Mostrar todos los proyectos...", "projects.search.placeholder": "Buscar proyectos (use /re/ para regexp)...", diff --git a/src/packages/frontend/i18n/trans/fr_FR.json b/src/packages/frontend/i18n/trans/fr_FR.json index 2aa978f34d..dd02966b71 100644 --- a/src/packages/frontend/i18n/trans/fr_FR.json +++ b/src/packages/frontend/i18n/trans/fr_FR.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {Démarrage du projet} other {Démarrer le projet}}", "project.start-button.trial.description": "Il n'y a plus de capacité pour les projets d'essai gratuit sur CoCalc pour le moment. {br} Améliorez votre projet en utilisant une licence ou {A4}.", "project.start-button.trial.message": "Trop de projets d'essai gratuits", + "project.warnings.deleted.banner": "

{icon} Avertissement : ce projet est supprimé !

Si vous avez l'intention d'utiliser ce projet, vous devriez le restaurer dans les paramètres du projet.", "projects.filename-search.placeholder": "Rechercher les noms de fichiers que vous avez modifiés...", "projects.load-all.label": "Afficher tous les projets...", "projects.search.placeholder": "Rechercher des projets (utiliser /re/ pour les expressions régulières)...", diff --git a/src/packages/frontend/i18n/trans/he_IL.json b/src/packages/frontend/i18n/trans/he_IL.json index d1ba821922..b4369ef9e7 100644 --- a/src/packages/frontend/i18n/trans/he_IL.json +++ b/src/packages/frontend/i18n/trans/he_IL.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {מפעיל פרויקט} other {הפעל פרויקט}}", "project.start-button.trial.description": "אין יותר מקום עבור פרויקטים לניסיון חינם ב-CoCalc כרגע. {br} שדרג את הפרויקט שלך באמצעות רישיון או {A4}.", "project.start-button.trial.message": "יותר מדי פרויקטים של ניסיון חינם", + "project.warnings.deleted.banner": "

{icon} אזהרה: הפרויקט הזה נמחק!

אם אתה מתכוון להשתמש בפרויקט הזה, עליך לבטל את המחיקה שלו בהגדרות הפרויקט.", "projects.filename-search.placeholder": "חפש קבצים שערכת...", "projects.load-all.label": "הצג את כל הפרויקטים...", "projects.search.placeholder": "חפש פרויקטים (השתמש ב-/re/ לחיפוש רגולרי)...", diff --git a/src/packages/frontend/i18n/trans/hi_IN.json b/src/packages/frontend/i18n/trans/hi_IN.json index c3a1476654..7dff1e3adf 100644 --- a/src/packages/frontend/i18n/trans/hi_IN.json +++ b/src/packages/frontend/i18n/trans/hi_IN.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {परियोजना शुरू हो रही है} other {परियोजना शुरू करें}}", "project.start-button.trial.description": "CoCalc पर अभी फ्री ट्रायल प्रोजेक्ट्स के लिए और क्षमता नहीं है। {br} अपने प्रोजेक्ट को अपग्रेड करें लाइसेंस का उपयोग करके या {A4}।", "project.start-button.trial.message": "बहुत सारे मुफ्त ट्रायल प्रोजेक्ट्स", + "project.warnings.deleted.banner": "

{icon} चेतावनी: यह प्रोजेक्ट हटाया गया है!

यदि आप इस प्रोजेक्ट का उपयोग करना चाहते हैं, तो आपको प्रोजेक्ट सेटिंग्स में इसे अनहटाना चाहिए।", "projects.filename-search.placeholder": "संपादित की गई फ़ाइल नाम खोजें...", "projects.load-all.label": "सभी प्रोजेक्ट दिखाएँ...", "projects.search.placeholder": "प्रोजेक्ट खोजें (regexp के लिए /re/ का उपयोग करें)...", diff --git a/src/packages/frontend/i18n/trans/hu_HU.json b/src/packages/frontend/i18n/trans/hu_HU.json index c6406b49a3..61fcb62fcc 100644 --- a/src/packages/frontend/i18n/trans/hu_HU.json +++ b/src/packages/frontend/i18n/trans/hu_HU.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {Projekt indítása} other {Projekt indítása}}", "project.start-button.trial.description": "Nincs több kapacitás az Ingyenes Próba projektek számára a CoCalc-on jelenleg. {br} Frissítse projektjét licenc használatával vagy {A4}.", "project.start-button.trial.message": "Túl sok ingyenes próba projekt", + "project.warnings.deleted.banner": "

{icon} Figyelem: ez a projekt törölve van!

Ha használni szeretné ezt a projektet, akkor vissza kell állítania a projekt beállításaiban.", "projects.filename-search.placeholder": "Keressen a szerkesztett fájlnevek között...", "projects.load-all.label": "Összes projekt megjelenítése...", "projects.search.placeholder": "Projektek keresése (használja a /re/ a regexphez)...", diff --git a/src/packages/frontend/i18n/trans/it_IT.json b/src/packages/frontend/i18n/trans/it_IT.json index 215448190f..c953f4c46d 100644 --- a/src/packages/frontend/i18n/trans/it_IT.json +++ b/src/packages/frontend/i18n/trans/it_IT.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {Avvio progetto} other {Avvia progetto}}", "project.start-button.trial.description": "Non c'è più capacità per i progetti Free Trial su CoCalc in questo momento. {br} Aggiorna il tuo progetto utilizzando una licenza o {A4}.", "project.start-button.trial.message": "Troppi Progetti di Prova Gratuita", + "project.warnings.deleted.banner": "

{icon} Attenzione: questo progetto è eliminato!

Se intendi utilizzare questo progetto, dovresti ripristinarlo nelle impostazioni del progetto.", "projects.filename-search.placeholder": "Cerca i nomi dei file che hai modificato...", "projects.load-all.label": "Mostra tutti i progetti...", "projects.search.placeholder": "Cerca progetti (usa /re/ per regexp)...", diff --git a/src/packages/frontend/i18n/trans/ja_JP.json b/src/packages/frontend/i18n/trans/ja_JP.json index 05abbcdd43..2b6ec7e095 100644 --- a/src/packages/frontend/i18n/trans/ja_JP.json +++ b/src/packages/frontend/i18n/trans/ja_JP.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {プロジェクトを開始しています} other {プロジェクトを開始}}", "project.start-button.trial.description": "現在CoCalcでは無料試用プロジェクトの容量がありません。 {br} ライセンスを使用してプロジェクトをアップグレードするか{A4}。", "project.start-button.trial.message": "無料トライアルプロジェクトが多すぎます", + "project.warnings.deleted.banner": "

{icon} 警告: このプロジェクトは削除されました！

このプロジェクトを使用するつもりなら、プロジェクト設定で削除を取り消してください。", "projects.filename-search.placeholder": "編集したファイル名を検索...", "projects.load-all.label": "すべてのプロジェクトを表示...", "projects.search.placeholder": "プロジェクトを検索 (/re/ を正規表現として使用)...", diff --git a/src/packages/frontend/i18n/trans/ko_KR.json b/src/packages/frontend/i18n/trans/ko_KR.json index 22622b92cb..1e905ae183 100644 --- a/src/packages/frontend/i18n/trans/ko_KR.json +++ b/src/packages/frontend/i18n/trans/ko_KR.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {프로젝트 시작 중} other {프로젝트 시작}}", "project.start-button.trial.description": "현재 CoCalc에는 무료 체험 프로젝트의 추가 용량이 없습니다. {br} 라이선스를 사용하여 프로젝트를 업그레이드하거나 {A4}하십시오.", "project.start-button.trial.message": "무료 체험 프로젝트가 너무 많음", + "project.warnings.deleted.banner": "

{icon} 경고: 이 프로젝트는 삭제되었습니다!

이 프로젝트를 사용하려면 프로젝트 설정에서 삭제 취소를 해야 합니다.", "projects.filename-search.placeholder": "편집한 파일 이름 검색...", "projects.load-all.label": "모든 프로젝트 보기...", "projects.search.placeholder": "프로젝트 검색 (/re/ 정규식 사용)...", diff --git a/src/packages/frontend/i18n/trans/pl_PL.json b/src/packages/frontend/i18n/trans/pl_PL.json index 2744017f82..d97f1a1171 100644 --- a/src/packages/frontend/i18n/trans/pl_PL.json +++ b/src/packages/frontend/i18n/trans/pl_PL.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {Uruchamianie projektu} other {Uruchom projekt}}", "project.start-button.trial.description": "Nie ma już więcej miejsca na projekty w bezpłatnej wersji próbnej na CoCalc w tej chwili. {br} Uaktualnij swój projekt używając licencji lub {A4}.", "project.start-button.trial.message": "Zbyt Wiele Projektów w Darmowej Wersji Próbnej", + "project.warnings.deleted.banner": "

{icon} Ostrzeżenie: ten projekt jest usunięty!

Jeśli zamierzasz korzystać z tego projektu, powinieneś przywrócić go w ustawieniach projektu.", "projects.filename-search.placeholder": "Szukaj nazw plików, które edytowałeś...", "projects.load-all.label": "Pokaż wszystkie projekty...", "projects.search.placeholder": "Wyszukaj projekty (użyj /re/ dla wyrażeń regularnych)...", diff --git a/src/packages/frontend/i18n/trans/pt_PT.json b/src/packages/frontend/i18n/trans/pt_PT.json index ceabdbca15..acebeec01b 100644 --- a/src/packages/frontend/i18n/trans/pt_PT.json +++ b/src/packages/frontend/i18n/trans/pt_PT.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {A iniciar projeto} other {Iniciar projeto}}", "project.start-button.trial.description": "Não há mais capacidade para projetos de Teste Gratuito no CoCalc neste momento. {br} Atualize o seu projeto usando uma licença ou {A4}.", "project.start-button.trial.message": "Demasiados Projetos de Teste Gratuitos", + "project.warnings.deleted.banner": "

{icon} Aviso: este projeto está eliminado!

Se pretender usar este projeto, deve restaurá-lo nas definições do projeto.", "projects.filename-search.placeholder": "Procurar por nomes de ficheiros que editou...", "projects.load-all.label": "Mostrar todos os projetos...", "projects.search.placeholder": "Procurar projetos (use /re/ para regexp)...", diff --git a/src/packages/frontend/i18n/trans/ru_RU.json b/src/packages/frontend/i18n/trans/ru_RU.json index 2fcd4920fc..6ec6a8d6b4 100644 --- a/src/packages/frontend/i18n/trans/ru_RU.json +++ b/src/packages/frontend/i18n/trans/ru_RU.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {Запуск проекта} other {Запустить проект}}", "project.start-button.trial.description": "На CoCalc сейчас нет свободных ресурсов для бесплатных пробных проектов. {br} Обновите свой проект с помощью лицензии или {A4}.", "project.start-button.trial.message": "Слишком много пробных проектов", + "project.warnings.deleted.banner": "

{icon} Внимание: этот проект удален!

Если вы собираетесь использовать этот проект, вам следует восстановить его в настройках проекта.", "projects.filename-search.placeholder": "Искать измененные вами файлы...", "projects.load-all.label": "Показать все проекты...", "projects.search.placeholder": "Искать проекты (используйте /re/ для regexp)...", diff --git a/src/packages/frontend/i18n/trans/tr_TR.json b/src/packages/frontend/i18n/trans/tr_TR.json index 7a22bef5a2..e9355a4c0a 100644 --- a/src/packages/frontend/i18n/trans/tr_TR.json +++ b/src/packages/frontend/i18n/trans/tr_TR.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {Proje başlatılıyor} other {Projeyi başlat}}", "project.start-button.trial.description": "Şu anda CoCalc üzerinde Ücretsiz Deneme projeleri için daha fazla kapasite yok. {br} Projenizi yükseltin bir lisans kullanarak veya {A4}.", "project.start-button.trial.message": "Çok Fazla Ücretsiz Deneme Projesi", + "project.warnings.deleted.banner": "

{icon} Uyarı: bu proje silindi!

Bu projeyi kullanmayı düşünüyorsanız, proje ayarlarında silinmesini geri alın.", "projects.filename-search.placeholder": "Düzenlediğiniz dosya adlarını arayın...", "projects.load-all.label": "Tüm projeleri göster...", "projects.search.placeholder": "Projeleri ara (/re/ kullanarak düzenli ifade)...", diff --git a/src/packages/frontend/i18n/trans/zh_CN.json b/src/packages/frontend/i18n/trans/zh_CN.json index bb2465076b..d07ebc7655 100644 --- a/src/packages/frontend/i18n/trans/zh_CN.json +++ b/src/packages/frontend/i18n/trans/zh_CN.json @@ -909,6 +909,7 @@ "project.start-button.button.txt": "{starting, select, true {启动项目} other {启动项目}}", "project.start-button.trial.description": "目前CoCalc上没有更多的免费试用项目容量。{br} 使用许可证或{A4} 升级您的项目。", "project.start-button.trial.message": "免费试用项目过多", + "project.warnings.deleted.banner": "

{icon} 警告：此项目已删除！

如果您打算使用此项目，应该在项目设置中取消删除。", "projects.filename-search.placeholder": "搜索你编辑过的文件名", "projects.load-all.label": "显示所有项目...", "projects.search.placeholder": "搜索项目（使用 /re/ 进行正则表达式搜索）…", diff --git a/src/packages/frontend/project/warnings/deleted.tsx b/src/packages/frontend/project/warnings/deleted.tsx index 6bdc5b7ce3..a7d119518f 100644 --- a/src/packages/frontend/project/warnings/deleted.tsx +++ b/src/packages/frontend/project/warnings/deleted.tsx @@ -3,21 +3,25 @@ * License: MS-RSL – see LICENSE.md for details */ +import { FormattedMessage } from "react-intl"; + import { Alert } from "@cocalc/frontend/antd-bootstrap"; import { Icon } from "@cocalc/frontend/components"; // A warning to put on pages when the project is deleted export const DeletedProjectWarning: React.FC = () => { return ( - -

- Warning: this project is{" "} - deleted! -

-

- If you intend to use this project, you should{" "} - undelete it in project settings. -

+ + {icon} Warning: this project is deleted! + If you intend to use this project, you should undelete it in project settings.`} + values={{ + icon: , + strong: (c) => {c}, + h4: (c) =>

{c}

, + }} + /> ); }; From 5d797ef0435311535a3ac5510eef3a2043487cd8 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Tue, 15 Oct 2024 12:35:49 +0200 Subject: [PATCH 19/20] hub/delete-projects: reset more fields and clear more table entries --- .../database/postgres/delete-projects.ts | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index 9620c337f2..e73fbfa10d 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -181,9 +181,11 @@ export async function cleanup_old_projects_data( numProj += 1; let delRows = 0; - // Clean up data *on* a given project. For now, remove all site licenses. + // Clean up data *on* a given project. For now, remove all site licenses, status and last_active. await pool.query( - `UPDATE projects SET site_license = NULL WHERE project_id = $1`, + `UPDATE projects + SET site_license = NULL, status = NULL, last_active = NULL, run_quota = NULL + WHERE project_id = $1`, [project_id], ); @@ -237,17 +239,23 @@ async function delete_associated_project_data( L2: WinstonLogger["debug"], project_id: string, ): Promise { + // TODO: two tables reference a project, but become useless. + // There should be a fallback strategy to move these objects to another project or surface them as being orphaned. + // tables: cloud_filesystems, compute_servers + let total = 0; // collecting tables, where the primary key is the default (i.e. "id") and // the field to check is always called "project_id" const tables = [ - "public_paths", - "project_log", - "file_use", + "blobs", "file_access_log", + "file_use", "jupyter_api_log", + "mentions", "openai_chatgpt_log", - "blobs", + "project_log", + "public_paths", + "shopping_cart_items", ] as const; for (const table of tables) { From ee488881eb0cff177a00b99d6023313a9dc864b3 Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Mon, 25 Nov 2024 15:15:06 +0100 Subject: [PATCH 20/20] database/bulk-delete: disable "blob" table for now --- src/packages/database/postgres/delete-projects.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index e73fbfa10d..e97bfb4580 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -247,7 +247,7 @@ async function delete_associated_project_data( // collecting tables, where the primary key is the default (i.e. "id") and // the field to check is always called "project_id" const tables = [ - "blobs", + //"blobs", // TODO: this is a bit tricky, because data could be used elsewhere. In the future, there will be an associated account_id! "file_access_log", "file_use", "jupyter_api_log",