From c0ecb225b5edfefd566c7662301a2c4a1801c4c6 Mon Sep 17 00:00:00 2001 From: Francis Duvivier Date: Fri, 3 Jan 2025 20:41:22 +0100 Subject: [PATCH 1/4] Implement writeDraftFile --- .gitignore | 4 + data/.gitkeep | 0 package-lock.json | 128 ++++++++++++++++++++++++++- package.json | 2 + public/swagger.json | 16 ++-- src/config.ts | 2 + src/controllers/private-rest.ts | 24 ++++- src/db/PostgreSQLBadgeHubMetadata.ts | 1 + src/domain/BadgeHubData.ts | 27 +++--- src/domain/BadgeHubFiles.ts | 4 +- src/domain/BadgeHubMetadata.ts | 1 + src/domain/UploadedFile.ts | 15 ++++ src/fs/NodeFSBadgeHubFiles.ts | 14 ++- src/generated/routes.ts | 23 +++-- 14 files changed, 227 insertions(+), 34 deletions(-) create mode 100644 data/.gitkeep create mode 100644 src/domain/UploadedFile.ts diff --git a/.gitignore b/.gitignore index 02c6db8..721c1fb 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,7 @@ dist .pnp.* /backup/data-backup.sql /backup/ + +# data dir +data +!data/.gitkeep diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/package-lock.json b/package-lock.json index 8e4ac9f..6b9a411 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "express": "^4.21.2", "jose": "^5.9.6", "moment": "^2.30.1", + "multer": "^1.4.5-lts.1", "pg": "^8.11.5", "pino-http": "^10.1.0", "pm2": "^5.4.0", @@ -26,6 +27,7 @@ }, "devDependencies": { "@types/express": "^4.17.21", + "@types/multer": "^1.4.12", "@types/node": "^20.17.6", "@types/pg": "^8.11.5", "@types/supertest": "^6.0.2", @@ -1739,6 +1741,11 @@ "node": ">= 8" } }, + "node_modules/append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -1920,6 +1927,17 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -2194,6 +2212,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "engines": [ + "node >= 0.8" + ], + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, "node_modules/concurrently": { "version": "8.2.2", "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-8.2.2.tgz", @@ -2259,6 +2291,11 @@ "integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==", "dev": true }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, "node_modules/cpu-features": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.2.tgz", @@ -3602,6 +3639,11 @@ "url": "https://github.com/sponsors/mesqueeb" } }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -3993,6 +4035,23 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, + "node_modules/multer": { + "version": "1.4.5-lts.1", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.1.tgz", + "integrity": "sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.0.0", + "concat-stream": "^1.5.2", + "mkdirp": "^0.5.4", + "object-assign": "^4.1.1", + "type-is": "^1.6.4", + "xtend": "^4.0.0" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -4120,6 +4179,14 @@ "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", "integrity": "sha512-K7J4xq5xAD5jHsGM5ReWXRTFa3JRGofHiMcVgQ8PRwgWxzjHpMWCIzsmyf60+mh8KLsqYPcjUMa0AC4hd6lPyQ==" }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-inspect": { "version": "1.13.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", @@ -4848,6 +4915,11 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "dev": true }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, "node_modules/process-warning": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-4.0.0.tgz", @@ -4982,6 +5054,25 @@ "node": ">=0.8" } }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/readable-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -5472,6 +5563,27 @@ "integrity": "sha512-Bc3YwwCB+OzldMxOXJIIvC6cPRWr/LxOp48CdQTOkPyk/t4JWWJbrilwBd7RJzKV8QW7tJkcgAmeuLLJugl5/w==", "dev": true }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", @@ -5671,9 +5783,9 @@ } }, "node_modules/systeminformation": { - "version": "5.23.14", - "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.23.14.tgz", - "integrity": "sha512-mUHEuDQJJOpphvjcIrTY0iwLnoNo/qotr6SuN7v0ANOO0L3j89mfCrEuIVheS/9S9KGRt4Osqxh9GoF7BX49UA==", + "version": "5.23.5", + "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.23.5.tgz", + "integrity": "sha512-PEpJwhRYxZgBCAlWZhWIgfMTjXLqfcaZ1pJsJn9snWNfBW/Z1YQg1mbIUSWrEV3ErAHF7l/OoVLQeaZDlPzkpA==", "optional": true, "os": [ "darwin", @@ -5909,6 +6021,11 @@ "node": ">= 0.6" } }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" + }, "node_modules/typescript": { "version": "5.7.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.2.tgz", @@ -5954,6 +6071,11 @@ "node": ">= 0.8" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", diff --git a/package.json b/package.json index 2a4b97e..a73140a 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "express": "^4.21.2", "jose": "^5.9.6", "moment": "^2.30.1", + "multer": "^1.4.5-lts.1", "pg": "^8.11.5", "pino-http": "^10.1.0", "pm2": "^5.4.0", @@ -41,6 +42,7 @@ }, "devDependencies": { "@types/express": "^4.17.21", + "@types/multer": "^1.4.12", "@types/node": "^20.17.6", "@types/pg": "^8.11.5", "@types/supertest": "^6.0.2", diff --git a/public/swagger.json b/public/swagger.json index 6da012d..19fe632 100644 --- a/public/swagger.json +++ b/public/swagger.json @@ -1264,16 +1264,16 @@ "requestBody": { "required": true, "content": { - "application/json": { + "multipart/form-data": { "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/components/schemas/Uint8Array" + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" } - ] + }, + "required": ["file"] } } } diff --git a/src/config.ts b/src/config.ts index ce396e9..cad5ad3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,5 @@ import { config } from "dotenv"; +import * as path from "node:path"; config(); export const EXPRESS_PORT = 8081; @@ -8,3 +9,4 @@ export const POSTGRES_PASSWORD = process.env.POSTGRES_PASSWORD; export const POSTGRES_HOST = process.env.DB_HOST; export const POSTGRES_PORT = 5432; export const NODE_ENV = process.env.NODE_ENV; +export const DATA_DIR = path.resolve(process.env.DATA_DIR || "data"); diff --git a/src/controllers/private-rest.ts b/src/controllers/private-rest.ts index a961f29..c1f6689 100644 --- a/src/controllers/private-rest.ts +++ b/src/controllers/private-rest.ts @@ -1,4 +1,14 @@ -import { Body, Delete, Get, Patch, Path, Post, Route, Tags } from "tsoa"; +import { + Body, + Delete, + Get, + Patch, + Path, + Post, + Route, + Tags, + UploadedFile, +} from "tsoa"; import { BadgeHubData } from "@domain/BadgeHubData"; import { PostgreSQLBadgeHubMetadata } from "@db/PostgreSQLBadgeHubMetadata"; import type { ProjectSlug } from "@domain/readModels/app/Project"; @@ -10,7 +20,9 @@ import { NodeFSBadgeHubFiles } from "@fs/NodeFSBadgeHubFiles"; interface UserProps extends Omit {} interface ProjectProps extends Omit {} + interface ProjectPropsPartial extends Partial {} + interface DbInsertAppMetadataJSONPartial extends Partial {} @@ -74,9 +86,15 @@ export class PrivateRestController { public async writeFile( @Path() slug: string, @Path() filePath: string, - @Body() fileContent: string | Uint8Array + @UploadedFile() file: Express.Multer.File ): Promise { - await this.badgeHubData.writeDraftFile(slug, filePath, fileContent); + await this.badgeHubData.writeDraftFile(slug, filePath, { + mimetype: file.mimetype, + fileContent: file.buffer, + directory: file.destination, + fileName: file.filename, + size: file.size, + }); } /** diff --git a/src/db/PostgreSQLBadgeHubMetadata.ts b/src/db/PostgreSQLBadgeHubMetadata.ts index fba0b42..35e7406 100644 --- a/src/db/PostgreSQLBadgeHubMetadata.ts +++ b/src/db/PostgreSQLBadgeHubMetadata.ts @@ -37,6 +37,7 @@ import { getInsertKeysAndValuesSql, } from "@db/sqlHelpers/objectToSQL"; import { BadgeHubMetadata } from "@domain/BadgeHubMetadata"; +import { UploadedFile } from "@domain/UploadedFile"; function getUpdateAssigmentsSql(changes: Object) { const changeEntries = getEntriesWithDefinedValues(changes); diff --git a/src/domain/BadgeHubData.ts b/src/domain/BadgeHubData.ts index 155dafe..8d97f69 100644 --- a/src/domain/BadgeHubData.ts +++ b/src/domain/BadgeHubData.ts @@ -12,6 +12,7 @@ import { } from "@db/models/app/DBAppMetadataJSON"; import { BadgeHubMetadata } from "@domain/BadgeHubMetadata"; import { BadgeHubFiles } from "@domain/BadgeHubFiles"; +import { UploadedFile } from "@domain/UploadedFile"; export class BadgeHubData { constructor( @@ -105,14 +106,12 @@ export class BadgeHubData { async writeDraftFile( projectSlug: ProjectSlug, filePath: string, - contents: string | Uint8Array + uploadedFile: UploadedFile ): Promise { - await this._writeDraftFile(projectSlug, filePath.split("/"), contents); + await this._writeDraftFile(projectSlug, filePath.split("/"), uploadedFile); if (filePath === "metadata.json") { const appMetadata: DBAppMetadataJSON = JSON.parse( - typeof contents === "string" - ? contents - : new TextDecoder().decode(contents) + new TextDecoder().decode(uploadedFile.fileContent) ); await this.badgeHubMetadata.updateDraftMetadata(projectSlug, appMetadata); } @@ -132,18 +131,26 @@ export class BadgeHubData { const updatedDraftVersion = await this.badgeHubMetadata.getDraftVersion(slug); const updatedAppMetadata = updatedDraftVersion.app_metadata; - await this._writeDraftFile( - slug, - ["metadata.json"], + const fileContent = new TextEncoder().encode( JSON.stringify(updatedAppMetadata) ); + await this._writeDraftFile(slug, ["metadata.json"], { + mimetype: "application/json", + fileContent, + directory: undefined, + fileName: undefined, + size: fileContent.length, + }); } private async _writeDraftFile( slug: string, pathParts: string[], - content: Uint8Array | string + uploadedFile: UploadedFile ) { - return this.badgeHubFiles.writeFile([slug, "draft", ...pathParts], content); + return this.badgeHubFiles.writeFile( + [slug, "draft", ...pathParts], + uploadedFile + ); } } diff --git a/src/domain/BadgeHubFiles.ts b/src/domain/BadgeHubFiles.ts index 259e89b..736e90d 100644 --- a/src/domain/BadgeHubFiles.ts +++ b/src/domain/BadgeHubFiles.ts @@ -1,6 +1,8 @@ +import { UploadedFile } from "@domain/UploadedFile"; + export interface BadgeHubFiles { // Using path parts instead of a string to make it easier to work with paths in a cross-platform way - writeFile(pathParts: string[], content: string | Uint8Array): Promise; + writeFile(pathParts: string[], uploadedFile: UploadedFile): Promise; getFileContents(pathParts: string[]): Promise; } diff --git a/src/domain/BadgeHubMetadata.ts b/src/domain/BadgeHubMetadata.ts index 4e7984a..71925eb 100644 --- a/src/domain/BadgeHubMetadata.ts +++ b/src/domain/BadgeHubMetadata.ts @@ -6,6 +6,7 @@ import { Category } from "@domain/readModels/app/Category"; import { DBInsertUser } from "@db/models/app/DBUser"; import { DBInsertProject, DBProject } from "@db/models/app/DBProject"; import { DBInsertAppMetadataJSON } from "@db/models/app/DBAppMetadataJSON"; +import { UploadedFile } from "@domain/UploadedFile"; export interface BadgeHubMetadata { insertUser(user: DBInsertUser): Promise; diff --git a/src/domain/UploadedFile.ts b/src/domain/UploadedFile.ts new file mode 100644 index 0000000..754f29a --- /dev/null +++ b/src/domain/UploadedFile.ts @@ -0,0 +1,15 @@ +export type UploadedFile = + | { + mimetype: string; + fileContent: Uint8Array; + size: number; + directory?: undefined; + fileName?: undefined; + } + | { + mimetype: string; + size: number; + fileContent: Uint8Array; + directory: string; + fileName: string; + }; diff --git a/src/fs/NodeFSBadgeHubFiles.ts b/src/fs/NodeFSBadgeHubFiles.ts index 4912e0d..e0743c0 100644 --- a/src/fs/NodeFSBadgeHubFiles.ts +++ b/src/fs/NodeFSBadgeHubFiles.ts @@ -1,8 +1,16 @@ import { BadgeHubFiles } from "@domain/BadgeHubFiles"; - +import { UploadedFile } from "@domain/UploadedFile"; +import { DATA_DIR } from "@config"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; export class NodeFSBadgeHubFiles implements BadgeHubFiles { - writeFile(pathParts: string[], content: Uint8Array): Promise { - throw new Error("Method not implemented."); + async writeFile( + pathParts: string[], + uploadedFile: UploadedFile + ): Promise { + const fullPath = path.join(DATA_DIR, ...pathParts); + await fs.mkdir(path.dirname(fullPath), { recursive: true }); + await fs.writeFile(fullPath, uploadedFile.fileContent); } getFileContents(pathParts: string[]): Promise { diff --git a/src/generated/routes.ts b/src/generated/routes.ts index a3d701d..d29aaaa 100644 --- a/src/generated/routes.ts +++ b/src/generated/routes.ts @@ -13,6 +13,7 @@ import type { RequestHandler, Router, } from "express"; +import multer from "multer"; // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa @@ -463,12 +464,17 @@ const templateService = new ExpressTemplateService(models, { // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa -export function RegisterRoutes(app: Router) { +export function RegisterRoutes( + app: Router, + opts?: { multer?: ReturnType } +) { // ########################################################################################################### // NOTE: If you do not see routes for all of your controllers in this file, then you might not have informed tsoa of where to look // Please look into the "controllerPathGlobs" config option described in the readme: https://github.com/lukeautry/tsoa // ########################################################################################################### + const upload = opts?.multer || multer({ limits: { fileSize: 8388608 } }); + app.get( "/api/v3/devices", ...fetchMiddlewares(PublicRestController), @@ -1019,6 +1025,12 @@ export function RegisterRoutes(app: Router) { // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa app.post( "/api/v3/apps/:slug/draft/files/:filePath", + upload.fields([ + { + name: "file", + maxCount: 1, + }, + ]), ...fetchMiddlewares(PrivateRestController), ...fetchMiddlewares( PrivateRestController.prototype.writeFile @@ -1037,12 +1049,11 @@ export function RegisterRoutes(app: Router) { required: true, dataType: "string", }, - fileContent: { - in: "body", - name: "fileContent", + file: { + in: "formData", + name: "file", required: true, - dataType: "union", - subSchemas: [{ dataType: "string" }, { ref: "Uint8Array" }], + dataType: "file", }, }; From 87a1a49c30b29b26135caf470b2129aa2eec1457 Mon Sep 17 00:00:00 2001 From: Francis Duvivier Date: Fri, 3 Jan 2025 21:10:15 +0100 Subject: [PATCH 2/4] Rename draft in private-rest controller to include Draft We do this in order for the openapi spec to better reflect what the api does in the operation name --- public/swagger.json | 4 ++-- src/controllers/private-rest.ts | 4 ++-- src/generated/routes.ts | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/public/swagger.json b/public/swagger.json index 19fe632..930031f 100644 --- a/public/swagger.json +++ b/public/swagger.json @@ -1234,7 +1234,7 @@ }, "/api/v3/apps/{slug}/draft/files/{filePath}": { "post": { - "operationId": "WriteFile", + "operationId": "WriteDraftFile", "responses": { "204": { "description": "No content" @@ -1318,7 +1318,7 @@ }, "/api/v3/apps/{slug}/draft/metadata": { "patch": { - "operationId": "ChangeAppMetadata", + "operationId": "ChangeDraftAppMetadata", "responses": { "204": { "description": "No content" diff --git a/src/controllers/private-rest.ts b/src/controllers/private-rest.ts index c1f6689..15a4b9c 100644 --- a/src/controllers/private-rest.ts +++ b/src/controllers/private-rest.ts @@ -83,7 +83,7 @@ export class PrivateRestController { * Upload a file to the latest draft version of the project. */ @Post("/apps/{slug}/draft/files/{filePath}") - public async writeFile( + public async writeDraftFile( @Path() slug: string, @Path() filePath: string, @UploadedFile() file: Express.Multer.File @@ -101,7 +101,7 @@ export class PrivateRestController { * Change the metadata of the latest draft version of the project. */ @Patch("/apps/{slug}/draft/metadata") - public async changeAppMetadata( + public async changeDraftAppMetadata( @Path() slug: string, @Body() appMetadataChanges: DbInsertAppMetadataJSONPartial ): Promise { diff --git a/src/generated/routes.ts b/src/generated/routes.ts index d29aaaa..c25b93f 100644 --- a/src/generated/routes.ts +++ b/src/generated/routes.ts @@ -1033,10 +1033,10 @@ export function RegisterRoutes( ]), ...fetchMiddlewares(PrivateRestController), ...fetchMiddlewares( - PrivateRestController.prototype.writeFile + PrivateRestController.prototype.writeDraftFile ), - async function PrivateRestController_writeFile( + async function PrivateRestController_writeDraftFile( request: ExRequest, response: ExResponse, next: any @@ -1070,7 +1070,7 @@ export function RegisterRoutes( const controller = new PrivateRestController(); await templateService.apiHandler({ - methodName: "writeFile", + methodName: "writeDraftFile", controller, response, next, @@ -1087,10 +1087,10 @@ export function RegisterRoutes( "/api/v3/apps/:slug/draft/metadata", ...fetchMiddlewares(PrivateRestController), ...fetchMiddlewares( - PrivateRestController.prototype.changeAppMetadata + PrivateRestController.prototype.changeDraftAppMetadata ), - async function PrivateRestController_changeAppMetadata( + async function PrivateRestController_changeDraftAppMetadata( request: ExRequest, response: ExResponse, next: any @@ -1118,7 +1118,7 @@ export function RegisterRoutes( const controller = new PrivateRestController(); await templateService.apiHandler({ - methodName: "changeAppMetadata", + methodName: "changeDraftAppMetadata", controller, response, next, From 482dc70075cc04af5e779e30256f1f223c636213 Mon Sep 17 00:00:00 2001 From: Francis Duvivier Date: Fri, 3 Jan 2025 21:57:50 +0100 Subject: [PATCH 3/4] Implement getDraftFile --- public/swagger.json | 3 ++- src/controllers/private-rest.ts | 12 ++++++++++-- src/fs/NodeFSBadgeHubFiles.ts | 2 +- src/setupPopulateDBApi.ts | 18 +++++++++++------- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/public/swagger.json b/public/swagger.json index 930031f..c36262c 100644 --- a/public/swagger.json +++ b/public/swagger.json @@ -1287,7 +1287,8 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Uint8Array" + "type": "string", + "format": "byte" } } } diff --git a/src/controllers/private-rest.ts b/src/controllers/private-rest.ts index 15a4b9c..905547e 100644 --- a/src/controllers/private-rest.ts +++ b/src/controllers/private-rest.ts @@ -5,6 +5,7 @@ import { Patch, Path, Post, + Request, Route, Tags, UploadedFile, @@ -16,6 +17,8 @@ import type { DBInsertUser, DBUser } from "@db/models/app/DBUser"; import type { DBInsertProject } from "@db/models/app/DBProject"; import type { DBInsertAppMetadataJSON } from "@db/models/app/DBAppMetadataJSON"; import { NodeFSBadgeHubFiles } from "@fs/NodeFSBadgeHubFiles"; +import express from "express"; +import { Readable } from "node:stream"; interface UserProps extends Omit {} @@ -115,8 +118,13 @@ export class PrivateRestController { public async getDraftFile( @Path() slug: string, @Path() filePath: string - ): Promise { - return await this.badgeHubData.getFileContents(slug, "draft", filePath); + ): Promise { + const fileContents = await this.badgeHubData.getFileContents( + slug, + "draft", + filePath + ); + return Readable.from(fileContents); } /** diff --git a/src/fs/NodeFSBadgeHubFiles.ts b/src/fs/NodeFSBadgeHubFiles.ts index e0743c0..3008895 100644 --- a/src/fs/NodeFSBadgeHubFiles.ts +++ b/src/fs/NodeFSBadgeHubFiles.ts @@ -14,6 +14,6 @@ export class NodeFSBadgeHubFiles implements BadgeHubFiles { } getFileContents(pathParts: string[]): Promise { - throw new Error("Method not implemented."); + return fs.readFile(path.join(DATA_DIR, ...pathParts)); } } diff --git a/src/setupPopulateDBApi.ts b/src/setupPopulateDBApi.ts index eb014ef..ffd7283 100644 --- a/src/setupPopulateDBApi.ts +++ b/src/setupPopulateDBApi.ts @@ -17,6 +17,7 @@ import { DBInsertProjectStatusOnBadge } from "@db/models/DBProjectStatusOnBadge" import { BadgeHubData } from "@domain/BadgeHubData"; import { PostgreSQLBadgeHubMetadata } from "@db/PostgreSQLBadgeHubMetadata"; import { NodeFSBadgeHubFiles } from "@fs/NodeFSBadgeHubFiles"; +import { Readable } from "node:stream"; const CATEGORY_NAMES = [ "Uncategorised", @@ -342,11 +343,12 @@ async function insertProjects(badgeHubData: BadgeHubData, userCount: number) { updated_at: updatedAt, }; - await badgeHubData.writeDraftFile( - inserted.slug, - "metadata.json", - JSON.stringify(appMetadata) - ); + const fileContent = Buffer.from(JSON.stringify(appMetadata)); + await badgeHubData.writeDraftFile(inserted.slug, "metadata.json", { + mimetype: "application/json", + size: fileContent.length, + fileContent: fileContent, + }); } return projectSlugs.map((slug) => slug.toLowerCase()); @@ -365,7 +367,8 @@ async function badgeProjectCrossTable( }; const insert1 = getInsertKeysAndValuesSql(insertObject1); await client.query( - sql`insert into badgehub.project_statuses_on_badges (${insert1.keys}) values (${insert1.values})` + sql`insert into badgehub.project_statuses_on_badges (${insert1.keys}) + values (${insert1.values})` ); // Some project support two badges @@ -377,7 +380,8 @@ async function badgeProjectCrossTable( }; const insert2 = getInsertKeysAndValuesSql(insertObject2); await client.query( - sql`insert into badgehub.project_statuses_on_badges (${insert2.keys}) values (${insert2.values})` + sql`insert into badgehub.project_statuses_on_badges (${insert2.keys}) + values (${insert2.values})` ); } } From 23a8b6e8e86a661f5cf129c63c208f963ed4dc8e Mon Sep 17 00:00:00 2001 From: Francis Duvivier Date: Sun, 5 Jan 2025 08:50:27 +0100 Subject: [PATCH 4/4] Fix parent dir access vulnerability --- src/config.ts | 2 +- src/domain/BadgeHubData.ts | 13 +++----- src/domain/BadgeHubFiles.ts | 15 +++++++-- src/domain/readModels/app/Version.ts | 2 ++ src/fs/NodeFSBadgeHubFiles.ts | 50 ++++++++++++++++++++++++++-- 5 files changed, 68 insertions(+), 14 deletions(-) diff --git a/src/config.ts b/src/config.ts index cad5ad3..b12ae01 100644 --- a/src/config.ts +++ b/src/config.ts @@ -9,4 +9,4 @@ export const POSTGRES_PASSWORD = process.env.POSTGRES_PASSWORD; export const POSTGRES_HOST = process.env.DB_HOST; export const POSTGRES_PORT = 5432; export const NODE_ENV = process.env.NODE_ENV; -export const DATA_DIR = path.resolve(process.env.DATA_DIR || "data"); +export const DATA_DIR = process.env.DATA_DIR || "data"; diff --git a/src/domain/BadgeHubData.ts b/src/domain/BadgeHubData.ts index 8d97f69..a0a2be7 100644 --- a/src/domain/BadgeHubData.ts +++ b/src/domain/BadgeHubData.ts @@ -1,5 +1,5 @@ import { Project, ProjectSlug } from "@domain/readModels/app/Project"; -import { Version } from "@domain/readModels/app/Version"; +import { Version, VersionRevision } from "@domain/readModels/app/Version"; import { User } from "@domain/readModels/app/User"; import { FileMetadata } from "@domain/readModels/app/FileMetadata"; import { Badge } from "@domain/readModels/Badge"; @@ -71,11 +71,11 @@ export class BadgeHubData { // TODO file management: here we should get the file path from the DB in order to fetch the correct file throw new Error("Method not implemented."); } - return this.badgeHubFiles.getFileContents([ + return this.badgeHubFiles.getFileContents( projectSlug, versionRevision, - ...filePath.split("/"), - ]); + filePath.split("/") + ); } getVersionZipContents( @@ -148,9 +148,6 @@ export class BadgeHubData { pathParts: string[], uploadedFile: UploadedFile ) { - return this.badgeHubFiles.writeFile( - [slug, "draft", ...pathParts], - uploadedFile - ); + return this.badgeHubFiles.writeFile(slug, "draft", pathParts, uploadedFile); } } diff --git a/src/domain/BadgeHubFiles.ts b/src/domain/BadgeHubFiles.ts index 736e90d..96dd827 100644 --- a/src/domain/BadgeHubFiles.ts +++ b/src/domain/BadgeHubFiles.ts @@ -1,8 +1,19 @@ import { UploadedFile } from "@domain/UploadedFile"; +import { ProjectSlug } from "@domain/readModels/app/Project"; +import { VersionRevision } from "@domain/readModels/app/Version"; export interface BadgeHubFiles { // Using path parts instead of a string to make it easier to work with paths in a cross-platform way - writeFile(pathParts: string[], uploadedFile: UploadedFile): Promise; + writeFile( + projectSlug: ProjectSlug, + versionRevision: VersionRevision, + pathParts: string[], + uploadedFile: UploadedFile + ): Promise; - getFileContents(pathParts: string[]): Promise; + getFileContents( + projectSlug: ProjectSlug, + versionRevision: VersionRevision, + pathParts: string[] + ): Promise; } diff --git a/src/domain/readModels/app/Version.ts b/src/domain/readModels/app/Version.ts index 7d2ecb8..0b0cbf1 100644 --- a/src/domain/readModels/app/Version.ts +++ b/src/domain/readModels/app/Version.ts @@ -3,6 +3,8 @@ import { DatedData } from "./DatedData"; import { FileMetadata } from "./FileMetadata"; import { Project } from "@domain/readModels/app/Project"; +export type VersionRevision = number | "draft" | "latest"; + export interface VersionRelation { version: Version; } diff --git a/src/fs/NodeFSBadgeHubFiles.ts b/src/fs/NodeFSBadgeHubFiles.ts index 3008895..66d9257 100644 --- a/src/fs/NodeFSBadgeHubFiles.ts +++ b/src/fs/NodeFSBadgeHubFiles.ts @@ -3,17 +3,61 @@ import { UploadedFile } from "@domain/UploadedFile"; import { DATA_DIR } from "@config"; import * as fs from "node:fs/promises"; import * as path from "node:path"; +import { ProjectSlug } from "@domain/readModels/app/Project"; +import { VersionRevision } from "@domain/readModels/app/Version"; + +// We want to ensure here that the given pathparts are valid and do not cause the full path to be outside of the project+version's directory +// We cannot do this check in a higher layer because that would require knowledge of the file storage implementation. +const getAndCheckFullPath = ( + projectSlug: ProjectSlug, + versionRevision: VersionRevision, + pathParts: string[] +) => { + const versionDir = + typeof versionRevision === "number" + ? `v${versionRevision}` + : versionRevision; + const resolvedVersionDir = path.resolve(DATA_DIR, projectSlug, versionDir); + const fullPath = path.resolve(resolvedVersionDir, ...pathParts); + + if (!fullPath.startsWith(resolvedVersionDir + path.sep)) { + console.error( + "Malicious intent detected, path validity failed for arguments:", + projectSlug, + versionDir, + fullPath + ); + throw new Error( + `Given path is invalid [${projectSlug}, ${versionDir}, ${pathParts.join("/")}], this request will be reported.` + ); + } + + return fullPath; +}; + export class NodeFSBadgeHubFiles implements BadgeHubFiles { async writeFile( + projectSlug: ProjectSlug, + versionRevision: VersionRevision, pathParts: string[], uploadedFile: UploadedFile ): Promise { - const fullPath = path.join(DATA_DIR, ...pathParts); + const fullPath = getAndCheckFullPath( + projectSlug, + versionRevision, + pathParts + ); await fs.mkdir(path.dirname(fullPath), { recursive: true }); await fs.writeFile(fullPath, uploadedFile.fileContent); } - getFileContents(pathParts: string[]): Promise { - return fs.readFile(path.join(DATA_DIR, ...pathParts)); + getFileContents( + projectSlug: ProjectSlug, + versionRevision: VersionRevision, + pathParts: string[] + ): Promise { + return fs.readFile( + getAndCheckFullPath(projectSlug, versionRevision, pathParts) + ); } }