From 642feaca6aec6400e06f92ee03dd6a8609f05e42 Mon Sep 17 00:00:00 2001 From: Daniel San Date: Tue, 17 Nov 2020 00:41:28 -0300 Subject: [PATCH 01/15] Add option to generate a presigned url with a expiration time Signed-off-by: Daniel San --- index.js | 6 ++++++ lib/optionsFromArguments.js | 2 ++ 2 files changed, 8 insertions(+) diff --git a/index.js b/index.js index d5b3196..61f2d4f 100644 --- a/index.js +++ b/index.js @@ -36,6 +36,8 @@ class S3Adapter { this._baseUrlDirect = options.baseUrlDirect; this._signatureVersion = options.signatureVersion; this._globalCacheControl = options.globalCacheControl; + this._presignedUrl = options.presignedUrl; + this._presignedUrlExpires = parseInt(options.presignedUrlExpires, 10); this._encryption = options.ServerSideEncryption; this._generateKey = options.generateKey; // Optional FilesAdaptor method @@ -159,6 +161,10 @@ class S3Adapter { getFileLocation(config, filename) { const fileName = filename.split('/').map(encodeURIComponent).join('/'); if (this._directAccess) { + if (this._presignedUrl) { + const params = { Bucket: this._bucket, Key: fileName, Expires: this._presignedUrlExpires }; + return this._s3Client.getSignedUrl('getObject', params); + } if (this._baseUrl) { if (typeof this._baseUrl === 'function') { if (this._baseUrlDirect) { diff --git a/lib/optionsFromArguments.js b/lib/optionsFromArguments.js index 4de8c19..7280040 100644 --- a/lib/optionsFromArguments.js +++ b/lib/optionsFromArguments.js @@ -94,6 +94,8 @@ const optionsFromArguments = function optionsFromArguments(args) { options = fromEnvironmentOrDefault(options, 'baseUrlDirect', 'S3_BASE_URL_DIRECT', false); options = fromEnvironmentOrDefault(options, 'signatureVersion', 'S3_SIGNATURE_VERSION', 'v4'); options = fromEnvironmentOrDefault(options, 'globalCacheControl', 'S3_GLOBAL_CACHE_CONTROL', null); + options = fromEnvironmentOrDefault(options, 'presignedUrl', 'S3_PRESIGNED_URL', false); + options = fromEnvironmentOrDefault(options, 'presignedUrlExpires', 'S3_PRESIGNED_URL_EXPIRES', 300); options = fromOptionsDictionaryOrDefault(options, 'generateKey', null); options = fromOptionsDictionaryOrDefault(options, 'validateFilename', null); From 6495047d3aa1169fcede2b984af9cd3efcec692f Mon Sep 17 00:00:00 2001 From: Daniel San Date: Tue, 17 Nov 2020 00:59:05 -0300 Subject: [PATCH 02/15] Update README.md Signed-off-by: Daniel San --- README.md | 54 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 6d98fd4..3323f59 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,8 @@ The preferred method is to use the default AWS credentials pattern. If no AWS c "baseUrlDirect": false, // default value "signatureVersion": 'v4', // default value "globalCacheControl": null, // default value. Or 'public, max-age=86400' for 24 hrs Cache-Control + "presignedUrl": false, // default value + "presignedUrlExpires": 300, // default value (300 seconds or 5 minutes) "ServerSideEncryption": 'AES256|aws:kms', //AES256 or aws:kms, or if you do not pass this, encryption won't be done "validateFilename": null, // Default to parse-server FilesAdapter::validateFilename. "generateKey": null // Will default to Parse.FilesController.preserveFileName @@ -114,29 +116,35 @@ And update your config / options ``` var S3Adapter = require('@parse/s3-files-adapter'); -var s3Adapter = new S3Adapter('accessKey', - 'secretKey', bucket, { - region: 'us-east-1' - bucketPrefix: '', - directAccess: false, - baseUrl: 'http://images.example.com', - signatureVersion: 'v4', - globalCacheControl: 'public, max-age=86400', // 24 hrs Cache-Control. - validateFilename: (filename) => { - if (filename.length > 1024) { - return 'Filename too long.'; - } - return null; // Return null on success - }, - generateKey: (filename) => { - return `${Date.now()}_${filename}`; // unique prefix for every filename - } - }); +var s3Adapter = new S3Adapter( + 'accessKey', + 'secretKey', + 'bucket', + { + region: 'us-east-1' + bucketPrefix: '', + directAccess: false, + baseUrl: 'http://images.example.com', + signatureVersion: 'v4', + globalCacheControl: 'public, max-age=86400', // 24 hrs Cache-Control. + presignedUrl: false, + presignedUrlExpires: 300, + validateFilename: (filename) => { + if (filename.length > 1024) { + return 'Filename too long.'; + } + return null; // Return null on success + }, + generateKey: (filename) => { + return `${Date.now()}_${filename}`; // unique prefix for every filename + } + } +); var api = new ParseServer({ - appId: 'my_app', - masterKey: 'master_key', - filesAdapter: s3adapter + appId: 'my_app', + masterKey: 'master_key', + filesAdapter: s3adapter }) ``` **Note:** there are a few ways you can pass arguments: @@ -167,6 +175,8 @@ var s3Options = { "baseUrl": null // default value "signatureVersion": 'v4', // default value "globalCacheControl": null, // default value. Or 'public, max-age=86400' for 24 hrs Cache-Control + "presignedUrl": false, // default value + "presignedUrlExpires": 300, // default value (300 seconds or 5 minutes) "validateFilename": () => null, // Anything goes! "generateKey": (filename) => filename, // Ensure Parse.FilesController.preserveFileName is true! } @@ -193,6 +203,8 @@ var s3Options = { region: process.env.SPACES_REGION, directAccess: true, globalCacheControl: "public, max-age=31536000", + presignedUrl: false, + presignedUrlExpires: 300, bucketPrefix: process.env.SPACES_BUCKET_PREFIX, s3overrides: { accessKeyId: process.env.SPACES_ACCESS_KEY, From ea46045e5b2547530168ba20cea8876aae9920b1 Mon Sep 17 00:00:00 2001 From: Daniel San Date: Wed, 18 Nov 2020 21:26:00 -0300 Subject: [PATCH 03/15] Uses AWS S3 presigned url together with baseUrl Signed-off-by: Daniel San --- index.js | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index 61f2d4f..ed35e8b 100644 --- a/index.js +++ b/index.js @@ -161,25 +161,41 @@ class S3Adapter { getFileLocation(config, filename) { const fileName = filename.split('/').map(encodeURIComponent).join('/'); if (this._directAccess) { + let presignedUrl = ''; if (this._presignedUrl) { - const params = { Bucket: this._bucket, Key: fileName, Expires: this._presignedUrlExpires }; - return this._s3Client.getSignedUrl('getObject', params); + const params = { + Bucket: this._bucket, + Key: this._bucketPrefix + fileName, + Expires: this._presignedUrlExpires, + }; + presignedUrl = this._s3Client.getSignedUrl('getObject', params); } if (this._baseUrl) { + let directAccessUrl; + if (typeof this._baseUrl === 'function') { if (this._baseUrlDirect) { - return `${this._baseUrl(config, filename)}/${fileName}`; + directAccessUrl = `${this._baseUrl(config, filename)}/${fileName}`; + } else { + directAccessUrl = `${this._baseUrl(config, filename)}/${this._bucketPrefix + fileName}`; } - return `${this._baseUrl(config, filename)}/${this._bucketPrefix + fileName}`; + } else if (this._baseUrlDirect) { + directAccessUrl = `${this._baseUrl}/${fileName}`; + } else { + directAccessUrl = `${this._baseUrl}/${this._bucketPrefix + fileName}`; } - if (this._baseUrlDirect) { - return `${this._baseUrl}/${fileName}`; + + if (this._presignedUrl) { + directAccessUrl += presignedUrl.substring(presignedUrl.indexOf('?')); } - return `${this._baseUrl}/${this._bucketPrefix + fileName}`; + return directAccessUrl; + } + if (this._presignedUrl) { + return presignedUrl; } return `https://${this._bucket}.s3.amazonaws.com/${this._bucketPrefix + fileName}`; } - return (`${config.mount}/files/${config.applicationId}/${fileName}`); + return `${config.mount}/files/${config.applicationId}/${fileName}`; } handleFileStream(filename, req, res) { From 1c4948ca98bdf1f276d5dcfe6923a40f7430dee7 Mon Sep 17 00:00:00 2001 From: Daniel San Date: Thu, 19 Nov 2020 18:09:20 -0300 Subject: [PATCH 04/15] Improves code to decrease complexity Signed-off-by: Daniel San --- index.js | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/index.js b/index.js index ed35e8b..2cb364a 100644 --- a/index.js +++ b/index.js @@ -161,28 +161,24 @@ class S3Adapter { getFileLocation(config, filename) { const fileName = filename.split('/').map(encodeURIComponent).join('/'); if (this._directAccess) { + const fileKey = `${this._bucketPrefix}${fileName}`; + let presignedUrl = ''; if (this._presignedUrl) { - const params = { - Bucket: this._bucket, - Key: this._bucketPrefix + fileName, - Expires: this._presignedUrlExpires, - }; + const params = { Bucket: this._bucket, Key: fileKey, Expires: this._presignedUrlExpires }; presignedUrl = this._s3Client.getSignedUrl('getObject', params); } if (this._baseUrl) { - let directAccessUrl; + let directAccessFileKey = fileKey; + if (this._baseUrlDirect) { + directAccessFileKey = fileName; + } + let directAccessUrl; if (typeof this._baseUrl === 'function') { - if (this._baseUrlDirect) { - directAccessUrl = `${this._baseUrl(config, filename)}/${fileName}`; - } else { - directAccessUrl = `${this._baseUrl(config, filename)}/${this._bucketPrefix + fileName}`; - } - } else if (this._baseUrlDirect) { - directAccessUrl = `${this._baseUrl}/${fileName}`; + directAccessUrl = `${this._baseUrl(config, filename)}/${directAccessFileKey}`; } else { - directAccessUrl = `${this._baseUrl}/${this._bucketPrefix + fileName}`; + directAccessUrl = `${this._baseUrl}/${directAccessFileKey}`; } if (this._presignedUrl) { @@ -193,7 +189,7 @@ class S3Adapter { if (this._presignedUrl) { return presignedUrl; } - return `https://${this._bucket}.s3.amazonaws.com/${this._bucketPrefix + fileName}`; + return `https://${this._bucket}.s3.amazonaws.com/${fileKey}`; } return `${config.mount}/files/${config.applicationId}/${fileName}`; } From 1f82746ea88c30b1ef14453a4ccec458c8f22ade Mon Sep 17 00:00:00 2001 From: Daniel San Date: Thu, 19 Nov 2020 21:32:20 -0300 Subject: [PATCH 05/15] Simplify code complexity using "function early return" concept Signed-off-by: Daniel San --- index.js | 56 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/index.js b/index.js index 2cb364a..9800578 100644 --- a/index.js +++ b/index.js @@ -21,6 +21,21 @@ const serialize = (obj) => { return str.join('&'); }; +function buildDirectAccessUrl(baseUrl, baseUrlFileKey, presignedUrl, config, filename) { + let directAccessUrl; + if (typeof baseUrl === 'function') { + directAccessUrl = `${baseUrl(config, filename)}/${baseUrlFileKey}`; + } else { + directAccessUrl = `${baseUrl}/${baseUrlFileKey}`; + } + + if (presignedUrl) { + directAccessUrl += presignedUrl.substring(presignedUrl.indexOf('?')); + } + + return directAccessUrl; +} + class S3Adapter { // Creates an S3 session. // Providing AWS access, secret keys and bucket are mandatory @@ -160,38 +175,27 @@ class S3Adapter { // otherwise we serve the file through parse-server getFileLocation(config, filename) { const fileName = filename.split('/').map(encodeURIComponent).join('/'); - if (this._directAccess) { - const fileKey = `${this._bucketPrefix}${fileName}`; - - let presignedUrl = ''; - if (this._presignedUrl) { - const params = { Bucket: this._bucket, Key: fileKey, Expires: this._presignedUrlExpires }; - presignedUrl = this._s3Client.getSignedUrl('getObject', params); - } - if (this._baseUrl) { - let directAccessFileKey = fileKey; - if (this._baseUrlDirect) { - directAccessFileKey = fileName; - } + if (!this._directAccess) { + return `${config.mount}/files/${config.applicationId}/${fileName}`; + } - let directAccessUrl; - if (typeof this._baseUrl === 'function') { - directAccessUrl = `${this._baseUrl(config, filename)}/${directAccessFileKey}`; - } else { - directAccessUrl = `${this._baseUrl}/${directAccessFileKey}`; - } + const fileKey = `${this._bucketPrefix}${fileName}`; - if (this._presignedUrl) { - directAccessUrl += presignedUrl.substring(presignedUrl.indexOf('?')); - } - return directAccessUrl; - } - if (this._presignedUrl) { + let presignedUrl = ''; + if (this._presignedUrl) { + const params = { Bucket: this._bucket, Key: fileKey, Expires: this._presignedUrlExpires }; + presignedUrl = this._s3Client.getSignedUrl('getObject', params); + if (!this._baseUrl) { return presignedUrl; } + } + + if (!this._baseUrl) { return `https://${this._bucket}.s3.amazonaws.com/${fileKey}`; } - return `${config.mount}/files/${config.applicationId}/${fileName}`; + + const baseUrlFileKey = this._baseUrlDirect ? fileName : fileKey; + return buildDirectAccessUrl(this._baseUrl, baseUrlFileKey, presignedUrl, config, filename); } handleFileStream(filename, req, res) { From e901d517a8b32278545145c6d6d3e9f230d915c0 Mon Sep 17 00:00:00 2001 From: Daniel San Date: Thu, 19 Nov 2020 21:44:24 -0300 Subject: [PATCH 06/15] Update README.md Signed-off-by: Daniel San --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3323f59..efa754a 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,8 @@ The preferred method is to use the default AWS credentials pattern. If no AWS c "baseUrlDirect": false, // default value "signatureVersion": 'v4', // default value "globalCacheControl": null, // default value. Or 'public, max-age=86400' for 24 hrs Cache-Control - "presignedUrl": false, // default value - "presignedUrlExpires": 300, // default value (300 seconds or 5 minutes) + "presignedUrl": false, // Optional. Set to true if yo want a AWS S3 presigned URL. Default is false. + "presignedUrlExpires": 300, // Optional. Configure the time that the AWS S3 presigned URL should expire, in seconds. Default is 300 seconds. "ServerSideEncryption": 'AES256|aws:kms', //AES256 or aws:kms, or if you do not pass this, encryption won't be done "validateFilename": null, // Default to parse-server FilesAdapter::validateFilename. "generateKey": null // Will default to Parse.FilesController.preserveFileName From 1051f9898800a226cbd2a2e4ccd359e86d8d1c96 Mon Sep 17 00:00:00 2001 From: Daniel San Date: Thu, 19 Nov 2020 22:18:51 -0300 Subject: [PATCH 07/15] Fix get presingedUrl and presignedUrl from class constructor args Signed-off-by: Daniel San --- lib/optionsFromArguments.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/optionsFromArguments.js b/lib/optionsFromArguments.js index 7280040..36cd6f3 100644 --- a/lib/optionsFromArguments.js +++ b/lib/optionsFromArguments.js @@ -64,6 +64,8 @@ const optionsFromArguments = function optionsFromArguments(args) { options.baseUrlDirect = otherOptions.baseUrlDirect; options.signatureVersion = otherOptions.signatureVersion; options.globalCacheControl = otherOptions.globalCacheControl; + options.presignedUrl = otherOptions.presignedUrl; + options.presignedUrlExpires = otherOptions.presignedUrlExpires; options.ServerSideEncryption = otherOptions.ServerSideEncryption; options.generateKey = otherOptions.generateKey; options.validateFilename = otherOptions.validateFilename; From 97189346da5a26206eaacc0600e5ea2903e6ab91 Mon Sep 17 00:00:00 2001 From: Daniel San Date: Thu, 19 Nov 2020 23:47:22 -0300 Subject: [PATCH 08/15] Using the AWS S3 default expire time Signed-off-by: Daniel San --- README.md | 8 ++++---- lib/optionsFromArguments.js | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index efa754a..6baf440 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ The preferred method is to use the default AWS credentials pattern. If no AWS c "signatureVersion": 'v4', // default value "globalCacheControl": null, // default value. Or 'public, max-age=86400' for 24 hrs Cache-Control "presignedUrl": false, // Optional. Set to true if yo want a AWS S3 presigned URL. Default is false. - "presignedUrlExpires": 300, // Optional. Configure the time that the AWS S3 presigned URL should expire, in seconds. Default is 300 seconds. + "presignedUrlExpires": 900, // Optional. Configure the time that the AWS S3 presigned URL should expire, in seconds. Default is 900 seconds. "ServerSideEncryption": 'AES256|aws:kms', //AES256 or aws:kms, or if you do not pass this, encryption won't be done "validateFilename": null, // Default to parse-server FilesAdapter::validateFilename. "generateKey": null // Will default to Parse.FilesController.preserveFileName @@ -128,7 +128,7 @@ var s3Adapter = new S3Adapter( signatureVersion: 'v4', globalCacheControl: 'public, max-age=86400', // 24 hrs Cache-Control. presignedUrl: false, - presignedUrlExpires: 300, + presignedUrlExpires: 900, validateFilename: (filename) => { if (filename.length > 1024) { return 'Filename too long.'; @@ -176,7 +176,7 @@ var s3Options = { "signatureVersion": 'v4', // default value "globalCacheControl": null, // default value. Or 'public, max-age=86400' for 24 hrs Cache-Control "presignedUrl": false, // default value - "presignedUrlExpires": 300, // default value (300 seconds or 5 minutes) + "presignedUrlExpires": 900, // default value (900 seconds) "validateFilename": () => null, // Anything goes! "generateKey": (filename) => filename, // Ensure Parse.FilesController.preserveFileName is true! } @@ -204,7 +204,7 @@ var s3Options = { directAccess: true, globalCacheControl: "public, max-age=31536000", presignedUrl: false, - presignedUrlExpires: 300, + presignedUrlExpires: 900, bucketPrefix: process.env.SPACES_BUCKET_PREFIX, s3overrides: { accessKeyId: process.env.SPACES_ACCESS_KEY, diff --git a/lib/optionsFromArguments.js b/lib/optionsFromArguments.js index 36cd6f3..3430c19 100644 --- a/lib/optionsFromArguments.js +++ b/lib/optionsFromArguments.js @@ -97,7 +97,7 @@ const optionsFromArguments = function optionsFromArguments(args) { options = fromEnvironmentOrDefault(options, 'signatureVersion', 'S3_SIGNATURE_VERSION', 'v4'); options = fromEnvironmentOrDefault(options, 'globalCacheControl', 'S3_GLOBAL_CACHE_CONTROL', null); options = fromEnvironmentOrDefault(options, 'presignedUrl', 'S3_PRESIGNED_URL', false); - options = fromEnvironmentOrDefault(options, 'presignedUrlExpires', 'S3_PRESIGNED_URL_EXPIRES', 300); + options = fromEnvironmentOrDefault(options, 'presignedUrlExpires', 'S3_PRESIGNED_URL_EXPIRES', 900); options = fromOptionsDictionaryOrDefault(options, 'generateKey', null); options = fromOptionsDictionaryOrDefault(options, 'validateFilename', null); From 140bdf95c5372c6accdc6ad01e1f266f8b9fcd7b Mon Sep 17 00:00:00 2001 From: Daniel San Date: Thu, 19 Nov 2020 23:49:17 -0300 Subject: [PATCH 09/15] When using s3client.getSignedUrl(), avoid return old path-style URL Signed-off-by: Daniel San --- spec/test.spec.js | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/spec/test.spec.js b/spec/test.spec.js index 3ce6e88..6a16d40 100644 --- a/spec/test.spec.js +++ b/spec/test.spec.js @@ -235,7 +235,7 @@ describe('S3Adapter tests', () => { describe('getFileStream', () => { it('should handle range bytes', () => { - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket'); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket'); s3._s3Client = { createBucket: (callback) => callback(), getObject: (params, callback) => { @@ -266,7 +266,7 @@ describe('S3Adapter tests', () => { }); it('should handle range bytes error', () => { - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket'); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket'); s3._s3Client = { createBucket: (callback) => callback(), getObject: (params, callback) => { @@ -290,7 +290,7 @@ describe('S3Adapter tests', () => { }); it('should handle range bytes no data', () => { - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket'); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket'); const data = { Error: 'NoBody' }; s3._s3Client = { createBucket: (callback) => callback(), @@ -331,26 +331,26 @@ describe('S3Adapter tests', () => { }); it('should get using the baseUrl', () => { - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket', options); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/foo/bar/test.png'); }); it('should get direct to baseUrl', () => { options.baseUrlDirect = true; - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket', options); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/test.png'); }); it('should get without directAccess', () => { options.directAccess = false; - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket', options); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://my.server.com/parse/files/xxxx/test.png'); }); it('should go directly to amazon', () => { delete options.baseUrl; - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket', options); - expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('https://myBucket.s3.amazonaws.com/foo/bar/test.png'); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); + expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('https://my-bucket.s3.amazonaws.com/foo/bar/test.png'); }); }); describe('getFileLocation', () => { @@ -374,26 +374,26 @@ describe('S3Adapter tests', () => { }); it('should get using the baseUrl', () => { - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket', options); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/foo/bar/test.png'); }); it('should get direct to baseUrl', () => { options.baseUrlDirect = true; - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket', options); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/test.png'); }); it('should get without directAccess', () => { options.directAccess = false; - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket', options); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://my.server.com/parse/files/xxxx/test.png'); }); it('should go directly to amazon', () => { delete options.baseUrl; - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket', options); - expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('https://myBucket.s3.amazonaws.com/foo/bar/test.png'); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); + expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('https://my-bucket.s3.amazonaws.com/foo/bar/test.png'); }); }); @@ -407,7 +407,7 @@ describe('S3Adapter tests', () => { }); it('should be null by default', () => { - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket', options); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); expect(s3.validateFilename === null).toBe(true); }); @@ -421,7 +421,7 @@ describe('S3Adapter tests', () => { } return null; }; - const s3 = new S3Adapter('accessKey', 'secretKey', 'myBucket', options); + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); expect(s3.validateFilename('foo/bar') instanceof Parse.Error).toBe(true); }); }); From 7a3c5ca3edf5b73b93769a3fa0c607cba767dbec Mon Sep 17 00:00:00 2001 From: Daniel San Date: Thu, 19 Nov 2020 23:54:11 -0300 Subject: [PATCH 10/15] Remove redundant code Signed-off-by: Daniel San --- spec/test.spec.js | 103 ++++++++++++++-------------------------------- 1 file changed, 31 insertions(+), 72 deletions(-) diff --git a/spec/test.spec.js b/spec/test.spec.js index 6a16d40..f469979 100644 --- a/spec/test.spec.js +++ b/spec/test.spec.js @@ -316,84 +316,43 @@ describe('S3Adapter tests', () => { }); describe('getFileLocation', () => { - const testConfig = { - mount: 'http://my.server.com/parse', - applicationId: 'xxxx', - }; - let options; - - beforeEach(() => { - options = { - directAccess: true, - bucketPrefix: 'foo/bar/', - baseUrl: 'http://example.com/files', + ['http://example.com/files', () => 'http://example.com/files'].forEach((baseUrl) => { + const testConfig = { + mount: 'http://my.server.com/parse', + applicationId: 'xxxx', }; - }); - - it('should get using the baseUrl', () => { - const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); - expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/foo/bar/test.png'); - }); - - it('should get direct to baseUrl', () => { - options.baseUrlDirect = true; - const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); - expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/test.png'); - }); - - it('should get without directAccess', () => { - options.directAccess = false; - const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); - expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://my.server.com/parse/files/xxxx/test.png'); - }); - - it('should go directly to amazon', () => { - delete options.baseUrl; - const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); - expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('https://my-bucket.s3.amazonaws.com/foo/bar/test.png'); - }); - }); - describe('getFileLocation', () => { - const testConfig = { - mount: 'http://my.server.com/parse', - applicationId: 'xxxx', - }; - let options; + let options; - beforeEach(() => { - options = { - directAccess: true, - bucketPrefix: 'foo/bar/', - baseUrl: (fileconfig, filename) => { - if (filename.length > 12) { - return 'http://example.com/files'; - } - return 'http://example.com/files'; - }, - }; - }); + beforeEach(() => { + options = { + directAccess: true, + bucketPrefix: 'foo/bar/', + baseUrl, + }; + }); - it('should get using the baseUrl', () => { - const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); - expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/foo/bar/test.png'); - }); + it('should get using the baseUrl', () => { + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); + expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/foo/bar/test.png'); + }); - it('should get direct to baseUrl', () => { - options.baseUrlDirect = true; - const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); - expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/test.png'); - }); + it('should get direct to baseUrl', () => { + options.baseUrlDirect = true; + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); + expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/test.png'); + }); - it('should get without directAccess', () => { - options.directAccess = false; - const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); - expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://my.server.com/parse/files/xxxx/test.png'); - }); + it('should get without directAccess', () => { + options.directAccess = false; + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); + expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://my.server.com/parse/files/xxxx/test.png'); + }); - it('should go directly to amazon', () => { - delete options.baseUrl; - const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); - expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('https://my-bucket.s3.amazonaws.com/foo/bar/test.png'); + it('should go directly to amazon', () => { + delete options.baseUrl; + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); + expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('https://my-bucket.s3.amazonaws.com/foo/bar/test.png'); + }); }); }); From 21e4d9c947b65ad6c5271e870f21f6be49467d6a Mon Sep 17 00:00:00 2001 From: Daniel San Date: Thu, 19 Nov 2020 23:55:18 -0300 Subject: [PATCH 11/15] Add tests for presigned url Signed-off-by: Daniel San --- spec/test.spec.js | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/spec/test.spec.js b/spec/test.spec.js index f469979..a32205a 100644 --- a/spec/test.spec.js +++ b/spec/test.spec.js @@ -325,6 +325,7 @@ describe('S3Adapter tests', () => { beforeEach(() => { options = { + presignedUrl: false, directAccess: true, bucketPrefix: 'foo/bar/', baseUrl, @@ -336,6 +337,20 @@ describe('S3Adapter tests', () => { expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/foo/bar/test.png'); }); + it('should get using the baseUrl and amazon using presigned URL', () => { + options.presignedUrl = true; + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); + + const fileLocation = s3.getFileLocation(testConfig, 'test.png'); + expect(fileLocation).toMatch(/^http:\/\/example.com\/files\/foo\/bar\/test.png\?/); + expect(fileLocation).toMatch(/X-Amz-Credential=accessKey%2F\d{8}%2Fus-east-1%2Fs3%2Faws4_request/); + expect(fileLocation).toMatch(/X-Amz-Date=\d{8}T\d{6}Z/); + expect(fileLocation).toMatch(/X-Amz-Signature=.{64}/); + expect(fileLocation).toContain('X-Amz-Algorithm=AWS4-HMAC-SHA256'); + expect(fileLocation).toContain('X-Amz-Expires=900'); + expect(fileLocation).toContain('X-Amz-SignedHeaders=host'); + }); + it('should get direct to baseUrl', () => { options.baseUrlDirect = true; const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); @@ -353,6 +368,21 @@ describe('S3Adapter tests', () => { const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('https://my-bucket.s3.amazonaws.com/foo/bar/test.png'); }); + + it('should go directly to amazon using presigned URL', () => { + delete options.baseUrl; + options.presignedUrl = true; + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); + + const fileLocation = s3.getFileLocation(testConfig, 'test.png'); + expect(fileLocation).toMatch(/^https:\/\/my-bucket.s3.amazonaws.com\/foo\/bar\/test.png\?/); + expect(fileLocation).toMatch(/X-Amz-Credential=accessKey%2F\d{8}%2Fus-east-1%2Fs3%2Faws4_request/); + expect(fileLocation).toMatch(/X-Amz-Date=\d{8}T\d{6}Z/); + expect(fileLocation).toMatch(/X-Amz-Signature=.{64}/); + expect(fileLocation).toContain('X-Amz-Algorithm=AWS4-HMAC-SHA256'); + expect(fileLocation).toContain('X-Amz-Expires=900'); + expect(fileLocation).toContain('X-Amz-SignedHeaders=host'); + }); }); }); From d761881000eb42c0a77988f782a48a5486257c00 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Tue, 24 Nov 2020 20:12:01 +0100 Subject: [PATCH 12/15] added new parameters to parameter table --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6baf440..4eb3017 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ The preferred method is to use the default AWS credentials pattern. If no AWS c | Parameter | Optional | Default value | Environment variable | Description | |-----------|----------|---------------|----------------------|-------------| | `fileAcl` | yes | `undefined` | S3_FILE_ACL | Sets the [Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl) of the file when storing it in the S3 bucket. Setting this parameter overrides the file ACL that would otherwise depend on the `directAccess` parameter. Setting the value `'none'` causes any ACL parameter to be removed that would otherwise be set. | +| `presignedUrl` | yes | `false` | S3_PRESIGNED_URL | If `true` a [presigned URL](https://docs.aws.amazon.com/AmazonS3/latest/dev/ShareObjectPreSignedURL.html) is returned when requesting the URL of file. The URL is only valid for a specified duration, see parameter `presignedUrlExpires`. | +| `presignedUrlExpires` | yes | `900` | S3_PRESIGNED_URL_EXPIRES | Sets the duration in seconds after which the [presigned URL](https://docs.aws.amazon.com/AmazonS3/latest/dev/ShareObjectPreSignedURL.html) of the file expires. This parameter requires `presignedUrl` to be `true`. | ### Using a config file @@ -75,8 +77,8 @@ The preferred method is to use the default AWS credentials pattern. If no AWS c "baseUrlDirect": false, // default value "signatureVersion": 'v4', // default value "globalCacheControl": null, // default value. Or 'public, max-age=86400' for 24 hrs Cache-Control - "presignedUrl": false, // Optional. Set to true if yo want a AWS S3 presigned URL. Default is false. - "presignedUrlExpires": 900, // Optional. Configure the time that the AWS S3 presigned URL should expire, in seconds. Default is 900 seconds. + "presignedUrl": false, // Optional. If true a presigned URL is returned when requesting the URL of file. The URL is only valid for a specified duration, see parameter `presignedUrlExpires`. Default is false. + "presignedUrlExpires": 900, // Optional. Sets the duration in seconds after which the presigned URL of the file expires. Default is 900 seconds. "ServerSideEncryption": 'AES256|aws:kms', //AES256 or aws:kms, or if you do not pass this, encryption won't be done "validateFilename": null, // Default to parse-server FilesAdapter::validateFilename. "generateKey": null // Will default to Parse.FilesController.preserveFileName From 4b5dd8d1380565e051f9cc4ba328f5f1bfc3de4e Mon Sep 17 00:00:00 2001 From: Daniel San Date: Fri, 4 Dec 2020 01:31:35 -0300 Subject: [PATCH 13/15] Small test improvement Signed-off-by: Daniel San --- spec/test.spec.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/test.spec.js b/spec/test.spec.js index a32205a..d66be5b 100644 --- a/spec/test.spec.js +++ b/spec/test.spec.js @@ -343,11 +343,11 @@ describe('S3Adapter tests', () => { const fileLocation = s3.getFileLocation(testConfig, 'test.png'); expect(fileLocation).toMatch(/^http:\/\/example.com\/files\/foo\/bar\/test.png\?/); - expect(fileLocation).toMatch(/X-Amz-Credential=accessKey%2F\d{8}%2Fus-east-1%2Fs3%2Faws4_request/); + expect(fileLocation).toMatch(/X-Amz-Credential=accessKey%2F\d{8}%2F\w{2}-\w{1,9}-\d%2Fs3%2Faws4_request/); expect(fileLocation).toMatch(/X-Amz-Date=\d{8}T\d{6}Z/); expect(fileLocation).toMatch(/X-Amz-Signature=.{64}/); + expect(fileLocation).toMatch(/X-Amz-Expires=\d{1,6}/); expect(fileLocation).toContain('X-Amz-Algorithm=AWS4-HMAC-SHA256'); - expect(fileLocation).toContain('X-Amz-Expires=900'); expect(fileLocation).toContain('X-Amz-SignedHeaders=host'); }); @@ -379,8 +379,8 @@ describe('S3Adapter tests', () => { expect(fileLocation).toMatch(/X-Amz-Credential=accessKey%2F\d{8}%2Fus-east-1%2Fs3%2Faws4_request/); expect(fileLocation).toMatch(/X-Amz-Date=\d{8}T\d{6}Z/); expect(fileLocation).toMatch(/X-Amz-Signature=.{64}/); + expect(fileLocation).toMatch(/X-Amz-Expires=\d{1,6}/); expect(fileLocation).toContain('X-Amz-Algorithm=AWS4-HMAC-SHA256'); - expect(fileLocation).toContain('X-Amz-Expires=900'); expect(fileLocation).toContain('X-Amz-SignedHeaders=host'); }); }); From 2330ac6504c597fe55291e54fd066ee6aba0f7a8 Mon Sep 17 00:00:00 2001 From: Daniel San Date: Fri, 4 Dec 2020 00:49:25 -0300 Subject: [PATCH 14/15] Add test to ensure that the AWS S3 client receives getObject as the operation in the getSignedUrl function Signed-off-by: Daniel San --- spec/test.spec.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spec/test.spec.js b/spec/test.spec.js index d66be5b..fca0d42 100644 --- a/spec/test.spec.js +++ b/spec/test.spec.js @@ -337,6 +337,22 @@ describe('S3Adapter tests', () => { expect(s3.getFileLocation(testConfig, 'test.png')).toEqual('http://example.com/files/foo/bar/test.png'); }); + it('when use presigned URL should use S3 \'getObject\' operation', () => { + options.presignedUrl = true; + const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); + const originalS3Client = s3._s3Client; + let getSignedUrlOperation = ''; + s3._s3Client = { + getSignedUrl: (operation, params, callback) => { + getSignedUrlOperation = operation; + return originalS3Client.getSignedUrl(operation, params, callback); + }, + }; + + s3.getFileLocation(testConfig, 'test.png'); + expect(getSignedUrlOperation).toBe('getObject'); + }); + it('should get using the baseUrl and amazon using presigned URL', () => { options.presignedUrl = true; const s3 = new S3Adapter('accessKey', 'secretKey', 'my-bucket', options); From 087369894d80b0d8937bbe9351fa25b04dcbcbff Mon Sep 17 00:00:00 2001 From: Daniel San Date: Tue, 2 Feb 2021 09:37:47 -0300 Subject: [PATCH 15/15] Add comment to explain security concerns when using presigned URL Signed-off-by: Daniel San --- index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/index.js b/index.js index 9800578..ddaac93 100644 --- a/index.js +++ b/index.js @@ -184,6 +184,8 @@ class S3Adapter { let presignedUrl = ''; if (this._presignedUrl) { const params = { Bucket: this._bucket, Key: fileKey, Expires: this._presignedUrlExpires }; + // Always use the "getObject" operation, and we recommend that you protect the URL + // appropriately: https://docs.aws.amazon.com/AmazonS3/latest/dev/ShareObjectPreSignedURL.html presignedUrl = this._s3Client.getSignedUrl('getObject', params); if (!this._baseUrl) { return presignedUrl;