diff --git a/migrations/tenant/0026-unicode-object-names.sql b/migrations/tenant/0026-unicode-object-names.sql new file mode 100644 index 00000000..4ff818c1 --- /dev/null +++ b/migrations/tenant/0026-unicode-object-names.sql @@ -0,0 +1,162 @@ +ALTER TABLE "storage"."objects" + ADD CONSTRAINT objects_name_check + CHECK (name SIMILAR TO '[\x09\x0A\x0D\x20-\xD7FF\xE000-\xFFFD\x00010000-\x0010ffff]+'); + +CREATE OR REPLACE FUNCTION storage.search ( + prefix TEXT, + bucketname TEXT, + limits INT DEFAULT 100, + levels INT DEFAULT 1, + offsets INT DEFAULT 0, + search TEXT DEFAULT '', + sortcolumn TEXT DEFAULT 'name', + sortorder TEXT DEFAULT 'asc' +) RETURNS TABLE ( + name TEXT, + id UUID, + updated_at TIMESTAMPTZ, + created_at TIMESTAMPTZ, + last_accessed_at TIMESTAMPTZ, + metadata JSONB + ) +AS $$ +DECLARE + v_order_by TEXT; + v_sort_order TEXT; +BEGIN + CASE + WHEN sortcolumn = 'name' THEN + v_order_by = 'name'; + WHEN sortcolumn = 'updated_at' THEN + v_order_by = 'updated_at'; + WHEN sortcolumn = 'created_at' THEN + v_order_by = 'created_at'; + WHEN sortcolumn = 'last_accessed_at' THEN + v_order_by = 'last_accessed_at'; + ELSE + v_order_by = 'name'; + END CASE; + + CASE + WHEN sortorder = 'asc' THEN + v_sort_order = 'asc'; + WHEN sortorder = 'desc' THEN + v_sort_order = 'desc'; + ELSE + v_sort_order = 'asc'; + END CASE; + + v_order_by = v_order_by || ' ' || v_sort_order; + + RETURN QUERY EXECUTE + 'WITH folders AS ( + SELECT path_tokens[$1] AS folder + FROM storage.objects + WHERE STARTS_WITH(LOWER(objects.name), $2 || $3) + AND bucket_id = $4 + AND ARRAY_LENGTH(objects.path_tokens, 1) <> $1 + GROUP BY folder + ORDER BY folder ' || v_sort_order || ' + ) + (SELECT folder AS "name", + NULL AS id, + NULL AS updated_at, + NULL AS created_at, + NULL AS last_accessed_at, + NULL AS metadata FROM folders) + UNION ALL + (SELECT path_tokens[$1] AS "name", + id, + updated_at, + created_at, + last_accessed_at, + metadata + FROM storage.objects + WHERE STARTS_WITH(LOWER(objects.name), $2 || $3) + AND bucket_id = $4 + AND ARRAY_LENGTH(objects.path_tokens, 1) = $1 + ORDER BY ' || v_order_by || ') + LIMIT $5 + OFFSET $6' USING levels, LOWER(prefix), LOWER(search), bucketname, limits, offsets; +END; +$$ LANGUAGE plpgsql STABLE; + +CREATE OR REPLACE FUNCTION storage.list_objects_with_delimiter(bucket_id TEXT, prefix_param TEXT, delimiter_param TEXT, max_keys INTEGER DEFAULT 100, start_after TEXT DEFAULT '', next_token TEXT DEFAULT '') + RETURNS TABLE (name TEXT, id UUID, metadata JSONB, updated_at TIMESTAMPTZ) AS +$$ +BEGIN + RETURN QUERY EXECUTE + 'SELECT DISTINCT ON(name COLLATE "C") * FROM ( + SELECT + CASE + WHEN POSITION($2 IN SUBSTRING(name FROM LENGTH($1) + 1)) > 0 THEN + SUBSTRING(name FROM 1 for LENGTH($1) + POSITION($2 IN SUBSTRING(name FROM LENGTH($1) + 1))) + ELSE + name + END AS name, id, metadata, updated_at + FROM + storage.objects + WHERE + bucket_id = $5 AND + STARTS_WITH(LOWER(name), $1) AND + CASE + WHEN $6 != '''' THEN + name COLLATE "C" > $6 + ELSE true END + AND CASE + WHEN $4 != '''' THEN + CASE + WHEN POSITION($2 IN SUBSTRING(name FROM LENGTH($1) + 1)) > 0 THEN + SUBSTRING(name FROM 1 FOR LENGTH($1) + POSITION($2 IN SUBSTRING(name FROM LENGTH($1) + 1))) COLLATE "C" > $4 + ELSE + name COLLATE "C" > $4 + END + ELSE + TRUE + END + ORDER BY + name COLLATE "C" ASC) AS e ORDER BY name COLLATE "C" LIMIT $3' + USING LOWER(prefix_param), delimiter_param, max_keys, next_token, bucket_id, start_after; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION storage.list_multipart_uploads_with_delimiter(bucket_id text, prefix_param text, delimiter_param text, max_keys integer default 100, next_key_token text DEFAULT '', next_upload_token text default '') + RETURNS TABLE (key text, id text, created_at timestamptz) AS +$$ +BEGIN + RETURN QUERY EXECUTE + 'SELECT DISTINCT ON(key COLLATE "C") * FROM ( + SELECT + CASE + WHEN POSITION($2 IN SUBSTRING(key FROM LENGTH($1) + 1)) > 0 THEN + SUBSTRING(key FROM 1 FOR LENGTH($1) + POSITION($2 IN SUBSTRING(key FROM LENGTH($1) + 1))) + ELSE + key + END AS key, id, created_at + FROM + storage.s3_multipart_uploads + WHERE + bucket_id = $5 AND + STARTS_WITH(LOWER(key), $1) AND + CASE + WHEN $4 != '''' AND $6 = '''' THEN + CASE + WHEN POSITION($2 IN SUBSTRING(key FROM LENGTH($1) + 1)) > 0 THEN + SUBSTRING(key FROM 1 FOR LENGTH($1) + POSITION($2 IN SUBSTRING(key FROM LENGTH($1) + 1))) COLLATE "C" > $4 + ELSE + key COLLATE "C" > $4 + END + ELSE + TRUE + END AND + CASE + WHEN $6 != '''' THEN + id COLLATE "C" > $6 + ELSE + TRUE + END + ORDER BY + key COLLATE "C" ASC, created_at ASC) AS e ORDER BY key COLLATE "C" LIMIT $3' + USING LOWER(prefix_param), delimiter_param, max_keys, next_key_token, bucket_id, next_upload_token; +END; +$$ LANGUAGE plpgsql; diff --git a/src/http/plugins/xml.ts b/src/http/plugins/xml.ts index 6d474161..8d7a665b 100644 --- a/src/http/plugins/xml.ts +++ b/src/http/plugins/xml.ts @@ -21,6 +21,10 @@ export const xmlParser = fastifyPlugin( isArray: (_: string, jpath: string) => { return opts.parseAsArray?.includes(jpath) }, + tagValueProcessor: (name: string, value: string) => + value.replace(/&#x([0-9a-fA-F]{1,6});/g, (_, str: string) => + String.fromCharCode(Number.parseInt(str, 16)) + ), }) } diff --git a/src/internal/database/migrations/types.ts b/src/internal/database/migrations/types.ts index b283fc2d..6c473237 100644 --- a/src/internal/database/migrations/types.ts +++ b/src/internal/database/migrations/types.ts @@ -24,4 +24,5 @@ export const DBMigration = { 'optimize-search-function': 22, 'operation-function': 23, 'custom-metadata': 24, + 'unicode-object-names': 25, } as const diff --git a/src/internal/errors/codes.ts b/src/internal/errors/codes.ts index 1274b7d5..ef002519 100644 --- a/src/internal/errors/codes.ts +++ b/src/internal/errors/codes.ts @@ -265,7 +265,7 @@ export const ERRORS = { code: ErrorCode.InvalidKey, resource: key, httpStatusCode: 400, - message: `Invalid key: ${key}`, + message: `Invalid key: ${encodeURIComponent(key)}`, originalError: e, }), diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index a8609a10..bbd8f5a8 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -877,7 +877,11 @@ export class DBError extends StorageBackendError implements RenderableError { code: pgError.code, }) default: - return ERRORS.DatabaseError(pgError.message, pgError).withMetadata({ + const errorMessage = + pgError.code === '23514' && pgError.constraint === 'objects_name_check' + ? 'Invalid object name' + : pgError.message + return ERRORS.DatabaseError(errorMessage, pgError).withMetadata({ query, code: pgError.code, }) diff --git a/src/storage/limits.ts b/src/storage/limits.ts index cff8627d..b6fd4f2e 100644 --- a/src/storage/limits.ts +++ b/src/storage/limits.ts @@ -47,9 +47,15 @@ export async function isImageTransformationEnabled(tenantId: string) { * @param key */ export function isValidKey(key: string): boolean { - // only allow s3 safe characters and characters which require special handling for now - // https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html - return key.length > 0 && /^(\w|\/|!|-|\.|\*|'|\(|\)| |&|\$|@|=|;|:|\+|,|\?)*$/.test(key) + // Allow any sequence of Unicode characters with UTF-8 encoding, + // except characters not allowed in XML 1.0. + // See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + // See: https://www.w3.org/TR/REC-xml/#charsets + // + const regex = + /[\0-\x08\x0B\f\x0E-\x1F\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/ + + return key.length > 0 && !regex.test(key) } /** diff --git a/src/storage/protocols/s3/s3-handler.ts b/src/storage/protocols/s3/s3-handler.ts index baa6a3ed..3321bcd8 100644 --- a/src/storage/protocols/s3/s3-handler.ts +++ b/src/storage/protocols/s3/s3-handler.ts @@ -506,6 +506,9 @@ export class S3ProtocolHandler { throw ERRORS.InvalidUploadId() } + mustBeValidBucketName(Bucket) + mustBeValidKey(Key) + await uploader.canUpload({ bucketId: Bucket as string, objectName: Key as string, @@ -601,6 +604,9 @@ export class S3ProtocolHandler { throw ERRORS.MissingContentLength() } + mustBeValidBucketName(Bucket) + mustBeValidKey(Key) + const bucket = await this.storage.asSuperUser().findBucket(Bucket, 'file_size_limit') const maxFileSize = await getFileSizeLimit(this.storage.db.tenantId, bucket?.file_size_limit) @@ -755,6 +761,9 @@ export class S3ProtocolHandler { throw ERRORS.MissingParameter('Key') } + mustBeValidBucketName(Bucket) + mustBeValidKey(Key) + const multipart = await this.storage.db .asSuperUser() .findMultipartUpload(UploadId, 'id,version') @@ -797,6 +806,9 @@ export class S3ProtocolHandler { throw ERRORS.MissingParameter('Bucket') } + mustBeValidBucketName(Bucket) + mustBeValidKey(Key) + const object = await this.storage .from(Bucket) .findObject(Key, 'metadata,user_metadata,created_at,updated_at') @@ -836,6 +848,9 @@ export class S3ProtocolHandler { throw ERRORS.MissingParameter('Key') } + mustBeValidBucketName(Bucket) + mustBeValidKey(Key) + const object = await this.storage.from(Bucket).findObject(Key, 'id') if (!object) { @@ -864,6 +879,9 @@ export class S3ProtocolHandler { const bucket = command.Bucket as string const key = command.Key as string + mustBeValidBucketName(bucket) + mustBeValidKey(key) + const object = await this.storage.from(bucket).findObject(key, 'version,user_metadata') const response = await this.storage.backend.getObject( storageS3Bucket, @@ -916,6 +934,9 @@ export class S3ProtocolHandler { throw ERRORS.MissingParameter('Key') } + mustBeValidBucketName(Bucket) + mustBeValidKey(Key) + await this.storage.from(Bucket).deleteObject(Key) return {} @@ -947,6 +968,9 @@ export class S3ProtocolHandler { return {} } + mustBeValidBucketName(Bucket) + Delete.Objects.forEach((o) => mustBeValidKey(o.Key)) + const deletedResult = await this.storage .from(Bucket) .deleteObjects(Delete.Objects.map((o) => o.Key || '')) @@ -1017,6 +1041,11 @@ export class S3ProtocolHandler { command.MetadataDirective = 'COPY' } + mustBeValidBucketName(Bucket) + mustBeValidKey(Key) + mustBeValidBucketName(sourceBucket) + mustBeValidKey(sourceKey) + const copyResult = await this.storage.from(sourceBucket).copyObject({ sourceKey, destinationBucket: Bucket, @@ -1147,6 +1176,11 @@ export class S3ProtocolHandler { throw ERRORS.NoSuchKey('') } + mustBeValidBucketName(Bucket) + mustBeValidKey(Key) + mustBeValidBucketName(sourceBucketName) + mustBeValidKey(sourceKey) + // Check if copy source exists const copySource = await this.storage.db.findObject( sourceBucketName, diff --git a/src/test/common.ts b/src/test/common.ts index 2ee80c40..09b8b897 100644 --- a/src/test/common.ts +++ b/src/test/common.ts @@ -8,6 +8,43 @@ export const adminApp = app({}) const ENV = process.env +/** + * Should support all Unicode characters with UTF-8 encoding according to AWS S3 object naming guide, including: + * - Safe characters: 0-9 a-z A-Z !-_.*'() + * - Characters that might require special handling: &$@=;/:+,? and Space and ASCII characters \t, \n, and \r. + * - Characters: \{}^%`[]"<>~#| and non-printable ASCII characters (128–255 decimal characters). + * - Astral code points + * + * The following characters are not allowed: + * - ASCII characters 0x00–0x1F, except 0x09, 0x0A, and 0x0D. + * - Unicode \u{FFFE} and \u{FFFF}. + * - Lone surrogate characters. + * See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + * See: https://www.w3.org/TR/REC-xml/#charsets + */ +export function getUnicodeObjectName(): string { + const objectName = 'test' + .concat("!-_*.'()") + // Characters that might require special handling + .concat('&$@=;:+,? \x09\x0A\x0D') + // Characters to avoid + .concat('\\{}^%`[]"<>~#|\xFF') + // MinIO max. length for each '/' separated segment is 255 + .concat('/') + .concat([...Array(127).keys()].map((i) => String.fromCodePoint(i + 128)).join('')) + .concat('/') + // Some special Unicode characters + .concat('\u2028\u202F\u{0001FFFF}') + // Some other Unicode characters + .concat('일이삼\u{0001f642}') + + return objectName +} + +export function getBadObjectName(): string { + return 'test '.concat('\x01\x02\x03') +} + export function useMockQueue() { const queueSpy: jest.SpyInstance | undefined = undefined beforeEach(() => { diff --git a/src/test/object.test.ts b/src/test/object.test.ts index 844ebc4c..c7bbeb59 100644 --- a/src/test/object.test.ts +++ b/src/test/object.test.ts @@ -6,7 +6,7 @@ import app from '../app' import { getConfig, mergeConfig } from '../config' import { signJWT } from '@internal/auth' import { Obj, backends } from '../storage' -import { useMockObject, useMockQueue } from './common' +import { getUnicodeObjectName, useMockObject, useMockQueue } from './common' import { getServiceKeyUser, getPostgresConnection } from '@internal/database' import { Knex } from 'knex' import { ErrorCode, StorageBackendError } from '@internal/errors' @@ -2456,3 +2456,59 @@ describe('testing list objects', () => { expect(responseJSON[1].name).toBe('sadcat-upload23.png') }) }) + +describe('Object key with Unicode characters', () => { + test('can upload, get and list', async () => { + const objectName = getUnicodeObjectName() + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + const headers = Object.assign({}, form.getHeaders(), { + authorization: `Bearer ${serviceKey}`, + 'x-upsert': 'true', + }) + const uploadResponse = await app().inject({ + method: 'POST', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + ...headers, + ...form.getHeaders(), + }, + payload: form, + }) + expect(uploadResponse.statusCode).toBe(200) + + const getResponse = await app().inject({ + method: 'GET', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(getResponse.statusCode).toBe(200) + expect(getResponse.headers['etag']).toBe('abc') + expect(getResponse.headers['last-modified']).toBe('Thu, 12 Aug 2021 16:00:00 GMT') + expect(getResponse.headers['cache-control']).toBe('no-cache') + + const listResponse = await app().inject({ + method: 'POST', + url: `/object/list/bucket2`, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + payload: { prefix: '', search: objectName }, + }) + expect(listResponse.statusCode).toBe(200) + const listResponseJSON = JSON.parse(listResponse.body) + expect(listResponseJSON).toHaveLength(1) + expect(listResponseJSON[0].name).toBe(objectName.split('/')[0]) + + const deleteResponse = await app().inject({ + method: 'DELETE', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(deleteResponse.statusCode).toBe(200) + }) +}) diff --git a/src/test/s3-protocol.test.ts b/src/test/s3-protocol.test.ts index 6d505298..cac40486 100644 --- a/src/test/s3-protocol.test.ts +++ b/src/test/s3-protocol.test.ts @@ -32,6 +32,7 @@ import { randomUUID } from 'crypto' import { getSignedUrl } from '@aws-sdk/s3-request-presigner' import axios from 'axios' import { createPresignedPost } from '@aws-sdk/s3-presigned-post' +import { getBadObjectName, getUnicodeObjectName } from './common' const { s3ProtocolAccessKeySecret, s3ProtocolAccessKeyId, storageS3Region } = getConfig() @@ -1402,5 +1403,167 @@ describe('S3 Protocol', () => { expect(resp.ok).toBeTruthy() }) }) + + describe('Object key with Unicode characters', () => { + it('can be used with MultipartUpload commands', async () => { + const bucketName = await createBucket(client) + const objectName = getUnicodeObjectName() + const createMultiPartUpload = new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + ContentType: 'image/jpg', + CacheControl: 'max-age=2000', + }) + const createMultipartResp = await client.send(createMultiPartUpload) + expect(createMultipartResp.UploadId).toBeTruthy() + const uploadId = createMultipartResp.UploadId + + const listMultipartUploads = new ListMultipartUploadsCommand({ + Bucket: bucketName, + }) + const listMultipartResp = await client.send(listMultipartUploads) + expect(listMultipartResp.Uploads?.length).toBe(1) + expect(listMultipartResp.Uploads?.[0].Key).toBe(objectName) + + const data = Buffer.alloc(1024 * 1024 * 2) + const uploadPart = new UploadPartCommand({ + Bucket: bucketName, + Key: objectName, + ContentLength: data.length, + UploadId: uploadId, + Body: data, + PartNumber: 1, + }) + const uploadPartResp = await client.send(uploadPart) + expect(uploadPartResp.ETag).toBeTruthy() + + const listParts = new ListPartsCommand({ + Bucket: bucketName, + Key: objectName, + UploadId: uploadId, + }) + const listPartsResp = await client.send(listParts) + expect(listPartsResp.Parts?.length).toBe(1) + + const completeMultiPartUpload = new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { + PartNumber: 1, + ETag: uploadPartResp.ETag, + }, + ], + }, + }) + const completeMultipartResp = await client.send(completeMultiPartUpload) + expect(completeMultipartResp.$metadata.httpStatusCode).toBe(200) + expect(completeMultipartResp.Key).toEqual(objectName) + }) + + it('can be used with Put, List, and Delete Object commands', async () => { + const bucketName = await createBucket(client) + const objectName = getUnicodeObjectName() + const putObject = new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: Buffer.alloc(1024 * 1024 * 12), + }) + const putObjectResp = await client.send(putObject) + expect(putObjectResp.$metadata.httpStatusCode).toEqual(200) + + const listObjects = new ListObjectsCommand({ + Bucket: bucketName, + }) + const listObjectsResp = await client.send(listObjects) + expect(listObjectsResp.Contents?.length).toBe(1) + expect(listObjectsResp.Contents?.[0].Key).toBe(objectName) + + const listObjectsV2 = new ListObjectsV2Command({ + Bucket: bucketName, + }) + const listObjectsV2Resp = await client.send(listObjectsV2) + expect(listObjectsV2Resp.Contents?.length).toBe(1) + expect(listObjectsV2Resp.Contents?.[0].Key).toBe(objectName) + + const getObject = new GetObjectCommand({ + Bucket: bucketName, + Key: objectName, + }) + const getObjectResp = await client.send(getObject) + const getObjectRespData = await getObjectResp.Body?.transformToByteArray() + expect(getObjectRespData).toBeTruthy() + expect(getObjectResp.ETag).toBeTruthy() + + const deleteObjects = new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: [ + { + Key: objectName, + }, + ], + }, + }) + const deleteObjectsResp = await client.send(deleteObjects) + expect(deleteObjectsResp.Errors).toBeFalsy() + expect(deleteObjectsResp.Deleted).toEqual([ + { + Key: objectName, + }, + ]) + + const getObjectDeleted = new GetObjectCommand({ + Bucket: bucketName, + Key: objectName, + }) + try { + await client.send(getObjectDeleted) + throw new Error('Should not reach here') + } catch (e) { + expect((e as S3ServiceException).$metadata.httpStatusCode).toEqual(404) + } + }) + }) + describe('Object key with bad characters', () => { + it('should fail on Put and Delete Object commands', async () => { + const bucketName = await createBucket(client) + const badObjectName = getBadObjectName() + try { + await uploadFile(client, bucketName, `${badObjectName}`, 1) + const getObject = new GetObjectCommand({ + Bucket: bucketName, + Key: badObjectName, + }) + await client.send(getObject) + throw new Error('Should not reach here') + } catch (e) { + expect((e as Error).message).not.toBe('Should not reach here') + expect((e as Error).name).toBe('InvalidKey') + expect((e as S3ServiceException).$metadata.httpStatusCode).toBe(400) + } + + try { + const deleteObjects = new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: [ + { + Key: badObjectName, + }, + ], + }, + }) + await client.send(deleteObjects) + throw new Error('Should not reach here') + } catch (e) { + expect((e as Error).message).not.toBe('Should not reach here') + expect((e as Error).name).toBe('InvalidKey') + expect((e as S3ServiceException).$metadata.httpStatusCode).toBe(400) + } + }) + }) }) }) diff --git a/src/test/tenant.test.ts b/src/test/tenant.test.ts index d53bc0d1..080b96c8 100644 --- a/src/test/tenant.test.ts +++ b/src/test/tenant.test.ts @@ -16,7 +16,7 @@ const payload = { serviceKey: 'd', jwks: { keys: [] }, migrationStatus: 'COMPLETED', - migrationVersion: 'custom-metadata', + migrationVersion: 'unicode-object-names', tracingMode: 'basic', features: { imageTransformation: { @@ -40,7 +40,7 @@ const payload2 = { serviceKey: 'h', jwks: null, migrationStatus: 'COMPLETED', - migrationVersion: 'custom-metadata', + migrationVersion: 'unicode-object-names', tracingMode: 'basic', features: { imageTransformation: { diff --git a/src/test/tus.test.ts b/src/test/tus.test.ts index cdbcc933..8e5cf784 100644 --- a/src/test/tus.test.ts +++ b/src/test/tus.test.ts @@ -13,7 +13,7 @@ import { logger } from '@internal/monitoring' import { getServiceKeyUser, getPostgresConnection } from '@internal/database' import { getConfig } from '../config' import app from '../app' -import { checkBucketExists } from './common' +import { checkBucketExists, getUnicodeObjectName } from './common' import { Storage, backends, StorageKnexDB } from '../storage' const { serviceKey, tenantId, storageS3Bucket, storageBackendType } = getConfig() @@ -525,4 +525,89 @@ describe('Tus multipart', () => { } }) }) + + describe('Object key with Unicode characters', () => { + it('can be uploaded with the TUS protocol', async () => { + const objectName = randomUUID() + '-' + getUnicodeObjectName() + + const bucket = await storage.createBucket({ + id: bucketName, + name: bucketName, + public: true, + }) + + const result = await new Promise((resolve, reject) => { + const upload = new tus.Upload(oneChunkFile, { + endpoint: `${localServerAddress}/upload/resumable`, + onShouldRetry: () => false, + uploadDataDuringCreation: false, + headers: { + authorization: `Bearer ${serviceKey}`, + 'x-upsert': 'true', + }, + metadata: { + bucketName: bucketName, + objectName: objectName, + contentType: 'image/jpeg', + cacheControl: '3600', + metadata: JSON.stringify({ + test1: 'test1', + test2: 'test2', + }), + }, + onError: function (error) { + console.log('Failed because: ' + error) + reject(error) + }, + onSuccess: () => { + resolve(true) + }, + }) + + upload.start() + }) + + expect(result).toEqual(true) + + const dbAsset = await storage.from(bucket.id).findObject(objectName, '*') + expect(dbAsset).toEqual({ + bucket_id: bucket.id, + created_at: expect.any(Date), + id: expect.any(String), + last_accessed_at: expect.any(Date), + metadata: { + cacheControl: 'max-age=3600', + contentLength: 29526, + eTag: '"53e1323c929d57b09b95fbe6d531865c-1"', + httpStatusCode: 200, + lastModified: expect.any(String), + mimetype: 'image/jpeg', + size: 29526, + }, + user_metadata: { + test1: 'test1', + test2: 'test2', + }, + name: objectName, + owner: null, + owner_id: null, + path_tokens: objectName.split('/'), + updated_at: expect.any(Date), + version: expect.any(String), + }) + + const getResponse = await app().inject({ + method: 'GET', + url: `/object/${bucketName}/${encodeURIComponent(objectName)}`, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(getResponse.statusCode).toBe(200) + expect(getResponse.headers['etag']).toBe('"53e1323c929d57b09b95fbe6d531865c-1"') + expect(getResponse.headers['cache-control']).toBe('max-age=3600') + expect(getResponse.headers['content-length']).toBe('29526') + expect(getResponse.headers['content-type']).toBe('image/jpeg') + }) + }) })