From 4fc41ad6f6d6e09f18eece33ee75aa78819908c0 Mon Sep 17 00:00:00 2001 From: Maksim_Hadalau Date: Fri, 15 Dec 2023 17:30:54 +0100 Subject: [PATCH 1/5] support user buckets --- README.md | 3 +- .../java/com/epam/aidial/core/AiDial.java | 6 +- src/main/java/com/epam/aidial/core/Proxy.java | 7 +- .../epam/aidial/core/config/Encryption.java | 9 + .../core/controller/BucketController.java | 25 ++ .../core/controller/ControllerSelector.java | 77 ++++--- .../core/controller/DeleteFileController.java | 18 +- .../controller/DownloadFileController.java | 31 ++- .../controller/FileMetadataController.java | 27 ++- .../core/controller/UploadFileController.java | 29 ++- .../com/epam/aidial/core/data/Bucket.java | 4 + .../epam/aidial/core/data/FileMetadata.java | 9 +- .../aidial/core/data/FileMetadataBase.java | 4 +- .../epam/aidial/core/data/FolderMetadata.java | 29 ++- .../core/security/EncryptionService.java | 62 +++++ .../epam/aidial/core/storage/BlobStorage.java | 72 +++--- .../aidial/core/storage/BlobStorageUtil.java | 87 ++----- .../aidial/core/storage/BlobWriteStream.java | 18 +- .../core/storage/ResourceDescription.java | 85 +++++++ .../aidial/core/storage/ResourceType.java | 15 ++ .../com/epam/aidial/core/FileApiTest.java | 215 +++++++++++++----- .../controller/ControllerSelectorTest.java | 41 ++-- .../core/storage/BlobStorageUtilTest.java | 51 ----- src/test/resources/aidial.settings.json | 4 + 24 files changed, 607 insertions(+), 321 deletions(-) create mode 100644 src/main/java/com/epam/aidial/core/config/Encryption.java create mode 100644 src/main/java/com/epam/aidial/core/controller/BucketController.java create mode 100644 src/main/java/com/epam/aidial/core/data/Bucket.java create mode 100644 src/main/java/com/epam/aidial/core/security/EncryptionService.java create mode 100644 src/main/java/com/epam/aidial/core/storage/ResourceDescription.java create mode 100644 src/main/java/com/epam/aidial/core/storage/ResourceType.java delete mode 100644 src/test/java/com/epam/aidial/core/storage/BlobStorageUtilTest.java diff --git a/README.md b/README.md index 8cc988e9a..06cb06bec 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ Static settings are used on startup and cannot be changed while application is r | storage.credential | - |Blob storage secret key | storage.bucket | - |Blob storage bucket | storage.createBucket | false |Indicates whether bucket should be created on start-up - +| encryption.password | - |Password used for AES encryption +| encryption.salt | - |Salt used for AES encryption ### Dynamic settings Dynamic settings are stored in JSON files, specified via "config.files" static setting, and reloaded at interval, specified via "config.reload" static setting. Dynamic settings include: diff --git a/src/main/java/com/epam/aidial/core/AiDial.java b/src/main/java/com/epam/aidial/core/AiDial.java index 11706950d..35bceffed 100644 --- a/src/main/java/com/epam/aidial/core/AiDial.java +++ b/src/main/java/com/epam/aidial/core/AiDial.java @@ -1,13 +1,14 @@ package com.epam.aidial.core; -import com.auth0.jwk.UrlJwkProvider; import com.epam.aidial.core.config.ConfigStore; +import com.epam.aidial.core.config.Encryption; import com.epam.aidial.core.config.FileConfigStore; import com.epam.aidial.core.config.Storage; import com.epam.aidial.core.limiter.RateLimiter; import com.epam.aidial.core.log.GfLogStore; import com.epam.aidial.core.log.LogStore; import com.epam.aidial.core.security.AccessTokenValidator; +import com.epam.aidial.core.security.EncryptionService; import com.epam.aidial.core.storage.BlobStorage; import com.epam.aidial.core.upstream.UpstreamBalancer; import com.epam.deltix.gflog.core.LogConfigurator; @@ -68,7 +69,8 @@ void start() throws Exception { Storage storageConfig = Json.decodeValue(settings("storage").toBuffer(), Storage.class); storage = new BlobStorage(storageConfig); } - Proxy proxy = new Proxy(vertx, client, configStore, logStore, rateLimiter, upstreamBalancer, accessTokenValidator, storage); + EncryptionService encryptionService = new EncryptionService(Json.decodeValue(settings("encryption").toBuffer(), Encryption.class)); + Proxy proxy = new Proxy(vertx, client, configStore, logStore, rateLimiter, upstreamBalancer, accessTokenValidator, storage, encryptionService); server = vertx.createHttpServer(new HttpServerOptions(settings("server"))).requestHandler(proxy); open(server, HttpServer::listen); diff --git a/src/main/java/com/epam/aidial/core/Proxy.java b/src/main/java/com/epam/aidial/core/Proxy.java index d4371a52d..bb453b2f9 100644 --- a/src/main/java/com/epam/aidial/core/Proxy.java +++ b/src/main/java/com/epam/aidial/core/Proxy.java @@ -8,6 +8,7 @@ import com.epam.aidial.core.limiter.RateLimiter; import com.epam.aidial.core.log.LogStore; import com.epam.aidial.core.security.AccessTokenValidator; +import com.epam.aidial.core.security.EncryptionService; import com.epam.aidial.core.security.ExtractedClaims; import com.epam.aidial.core.storage.BlobStorage; import com.epam.aidial.core.upstream.UpstreamBalancer; @@ -27,6 +28,7 @@ import java.net.URLDecoder; import java.nio.charset.StandardCharsets; +import java.util.Set; @Slf4j @Getter @@ -46,6 +48,8 @@ public class Proxy implements Handler { public static final int REQUEST_BODY_MAX_SIZE_BYTES = 16 * 1024 * 1024; public static final int FILES_REQUEST_BODY_MAX_SIZE_BYTES = 512 * 1024 * 1024; + private static final Set ALLOWED_HTTP_METHODS = Set.of(HttpMethod.GET, HttpMethod.POST, HttpMethod.PUT, HttpMethod.DELETE); + private final Vertx vertx; private final HttpClient client; private final ConfigStore configStore; @@ -54,6 +58,7 @@ public class Proxy implements Handler { private final UpstreamBalancer upstreamBalancer; private final AccessTokenValidator tokenValidator; private final BlobStorage storage; + private final EncryptionService encryptionService; @Override public void handle(HttpServerRequest request) { @@ -79,7 +84,7 @@ private void handleRequest(HttpServerRequest request) { } HttpMethod requestMethod = request.method(); - if (requestMethod != HttpMethod.GET && requestMethod != HttpMethod.POST && requestMethod != HttpMethod.DELETE) { + if (!ALLOWED_HTTP_METHODS.contains(requestMethod)) { respond(request, HttpStatus.METHOD_NOT_ALLOWED); return; } diff --git a/src/main/java/com/epam/aidial/core/config/Encryption.java b/src/main/java/com/epam/aidial/core/config/Encryption.java new file mode 100644 index 000000000..81107ddc5 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/config/Encryption.java @@ -0,0 +1,9 @@ +package com.epam.aidial.core.config; + +import lombok.Data; + +@Data +public class Encryption { + String password; + String salt; +} diff --git a/src/main/java/com/epam/aidial/core/controller/BucketController.java b/src/main/java/com/epam/aidial/core/controller/BucketController.java new file mode 100644 index 000000000..2f46be9e9 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/controller/BucketController.java @@ -0,0 +1,25 @@ +package com.epam.aidial.core.controller; + +import com.epam.aidial.core.Proxy; +import com.epam.aidial.core.ProxyContext; +import com.epam.aidial.core.data.Bucket; +import com.epam.aidial.core.security.EncryptionService; +import com.epam.aidial.core.storage.BlobStorageUtil; +import com.epam.aidial.core.util.HttpStatus; +import io.vertx.core.Future; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class BucketController { + + private final Proxy proxy; + private final ProxyContext context; + + public Future getBucket() { + EncryptionService encryptionService = proxy.getEncryptionService(); + String bucketLocation = BlobStorageUtil.buildUserBucket(context); + String encryptedBucket = encryptionService.encrypt(bucketLocation); + + return context.respond(HttpStatus.OK, new Bucket(encryptedBucket)); + } +} diff --git a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java index 8fbd5b696..0ffd5b128 100644 --- a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java +++ b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java @@ -33,7 +33,11 @@ public class ControllerSelector { private static final Pattern PATTERN_APPLICATIONS = Pattern.compile("/+openai/applications"); - private static final Pattern PATTERN_FILES = Pattern.compile("/v1/files(.*)"); + private static final Pattern PATTERN_BUCKET = Pattern.compile("/v1/bucket"); + + private static final Pattern PATTERN_FILES = Pattern.compile("/v1/files/([^/]*)/(.*)"); + + private static final Pattern PATTERN_FILES_METADATA = Pattern.compile("/v1/files/metadata/([^/]*)/(.*)"); private static final Pattern PATTERN_RATE_RESPONSE = Pattern.compile("/+v1/([-.@a-zA-Z0-9]+)/rate"); private static final Pattern PATTERN_TOKENIZE = Pattern.compile("/+v1/deployments/([-.@a-zA-Z0-9]+)/tokenize"); @@ -50,6 +54,8 @@ public Controller select(Proxy proxy, ProxyContext context) { controller = selectPost(proxy, context, path); } else if (method == HttpMethod.DELETE) { controller = selectDelete(proxy, context, path); + } else if (method == HttpMethod.PUT) { + controller = selectPut(proxy, context, path); } return (controller == null) ? new RouteController(proxy, context) : controller; @@ -123,17 +129,26 @@ private static Controller selectGet(Proxy proxy, ProxyContext context, String pa return controller::getApplications; } + match = match(PATTERN_FILES_METADATA, path); + if (match != null) { + String bucket = match.group(1); + String filePath = match.group(2); + FileMetadataController controller = new FileMetadataController(proxy, context); + return () -> controller.list(bucket, filePath); + } + match = match(PATTERN_FILES, path); if (match != null) { - String filePath = match.group(1); - String purpose = context.getRequest().params().get(DownloadFileController.PURPOSE_FILE_QUERY_PARAMETER); - if (DownloadFileController.QUERY_METADATA_QUERY_PARAMETER_VALUE.equals(purpose)) { - FileMetadataController controller = new FileMetadataController(proxy, context); - return () -> controller.list(filePath); - } else { - DownloadFileController controller = new DownloadFileController(proxy, context); - return () -> controller.download(filePath); - } + String bucket = match.group(1); + String filePath = match.group(2); + DownloadFileController controller = new DownloadFileController(proxy, context); + return () -> controller.download(bucket, filePath); + } + + match = match(PATTERN_BUCKET, path); + if (match != null) { + BucketController controller = new BucketController(proxy, context); + return controller::getBucket; } return null; @@ -148,22 +163,15 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return () -> controller.handle(deploymentId, deploymentApi); } - match = match(PATTERN_FILES, path); - if (match != null) { - String relativeFilePath = match.group(1); - UploadFileController controller = new UploadFileController(proxy, context); - return () -> controller.upload(relativeFilePath); - } - match = match(PATTERN_RATE_RESPONSE, path); if (match != null) { String deploymentId = match.group(1); Function getter = (model) -> { return Optional.ofNullable(model) - .map(d -> d.getFeatures()) - .map(t -> t.getRateEndpoint()) - .orElse(null); + .map(d -> d.getFeatures()) + .map(t -> t.getRateEndpoint()) + .orElse(null); }; DeploymentFeatureController controller = new DeploymentFeatureController(proxy, context); @@ -176,9 +184,9 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p Function getter = (model) -> { return Optional.ofNullable(model) - .map(d -> d.getFeatures()) - .map(t -> t.getTokenizeEndpoint()) - .orElse(null); + .map(d -> d.getFeatures()) + .map(t -> t.getTokenizeEndpoint()) + .orElse(null); }; DeploymentFeatureController controller = new DeploymentFeatureController(proxy, context); @@ -191,9 +199,9 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p Function getter = (model) -> { return Optional.ofNullable(model) - .map(d -> d.getFeatures()) - .map(t -> t.getTruncatePromptEndpoint()) - .orElse(null); + .map(d -> d.getFeatures()) + .map(t -> t.getTruncatePromptEndpoint()) + .orElse(null); }; DeploymentFeatureController controller = new DeploymentFeatureController(proxy, context); @@ -206,9 +214,22 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p private static Controller selectDelete(Proxy proxy, ProxyContext context, String path) { Matcher match = match(PATTERN_FILES, path); if (match != null) { - String relativeFilePath = match.group(1); + String bucket = match.group(1); + String filePath = match.group(2); DeleteFileController controller = new DeleteFileController(proxy, context); - return () -> controller.delete(relativeFilePath); + return () -> controller.delete(bucket, filePath); + } + + return null; + } + + private static Controller selectPut(Proxy proxy, ProxyContext context, String path) { + Matcher match = match(PATTERN_FILES, path); + if (match != null) { + String bucket = match.group(1); + String filePath = match.group(2); + UploadFileController controller = new UploadFileController(proxy, context); + return () -> controller.upload(bucket, filePath); } return null; diff --git a/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java b/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java index ae9945cf6..aa65d1039 100644 --- a/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java @@ -4,6 +4,8 @@ import com.epam.aidial.core.ProxyContext; import com.epam.aidial.core.storage.BlobStorage; import com.epam.aidial.core.storage.BlobStorageUtil; +import com.epam.aidial.core.storage.ResourceDescription; +import com.epam.aidial.core.storage.ResourceType; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; import lombok.AllArgsConstructor; @@ -17,12 +19,20 @@ public class DeleteFileController { /** * Deletes file from storage. - * Current API implementation requires a relative path, absolute path will be calculated based on authentication context * - * @param path relative path, for example: /inputs/data.csv + * @param filePath relative path, for example: inputs/data.csv */ - public Future delete(String path) { - String absoluteFilePath = BlobStorageUtil.buildAbsoluteFilePath(context, path); + public Future delete(String bucket, String filePath) { + String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); + String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); + + if (!expectedUserBucket.equals(decryptedBucket)) { + return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); + } + + ResourceDescription resource = ResourceDescription.from(ResourceType.FILES, bucket, decryptedBucket, filePath); + String absoluteFilePath = resource.getAbsoluteFilePath(); + BlobStorage storage = proxy.getStorage(); Future result = proxy.getVertx().executeBlocking(() -> { try { diff --git a/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java b/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java index 801af823f..cc1b7a132 100644 --- a/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java @@ -4,6 +4,8 @@ import com.epam.aidial.core.ProxyContext; import com.epam.aidial.core.storage.BlobStorageUtil; import com.epam.aidial.core.storage.InputStreamReader; +import com.epam.aidial.core.storage.ResourceDescription; +import com.epam.aidial.core.storage.ResourceType; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; import io.vertx.core.Promise; @@ -21,13 +23,6 @@ @AllArgsConstructor public class DownloadFileController { - private static final String PATH_TYPE_QUERY_PARAMETER = "path"; - private static final String RELATIVE_PATH_TYPE = "relative"; - - static final String PURPOSE_FILE_QUERY_PARAMETER = "purpose"; - - static final String QUERY_METADATA_QUERY_PARAMETER_VALUE = "metadata"; - private final Proxy proxy; private final ProxyContext context; @@ -36,18 +31,20 @@ public class DownloadFileController { * Path can be either absolute or relative. * Path type determined by "path" query parameter which can be "absolute" or "relative"(default value) * - * @param path file path; absolute or relative + * @param filePath file path; absolute or relative */ - public Future download(String path) { - String pathType = context.getRequest().params().get(PATH_TYPE_QUERY_PARAMETER); - String absoluteFilePath; - if (RELATIVE_PATH_TYPE.equals(pathType)) { - absoluteFilePath = BlobStorageUtil.buildAbsoluteFilePath(context, path); - } else { - absoluteFilePath = path; + public Future download(String bucket, String filePath) { + String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); + String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); + + if (!expectedUserBucket.equals(decryptedBucket)) { + return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); } + + ResourceDescription resource = ResourceDescription.from(ResourceType.FILES, bucket, decryptedBucket, filePath); + Future blobFuture = proxy.getVertx().executeBlocking(() -> - proxy.getStorage().load(BlobStorageUtil.removeLeadingPathSeparator(absoluteFilePath))); + proxy.getStorage().load(resource.getAbsoluteFilePath())); Promise result = Promise.promise(); blobFuture.onSuccess(blob -> { @@ -78,7 +75,7 @@ public Future download(String path) { result.fail(e); } }).onFailure(error -> context.respond(HttpStatus.INTERNAL_SERVER_ERROR, - "Failed to fetch file with path " + path)); + "Failed to fetch file with path %s/%s".formatted(bucket, filePath))); return result.future(); } diff --git a/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java b/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java index aabb1fb0e..98bfbf597 100644 --- a/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java +++ b/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java @@ -5,13 +5,13 @@ import com.epam.aidial.core.data.FileMetadataBase; import com.epam.aidial.core.storage.BlobStorage; import com.epam.aidial.core.storage.BlobStorageUtil; +import com.epam.aidial.core.storage.ResourceDescription; +import com.epam.aidial.core.storage.ResourceType; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import java.util.List; - @AllArgsConstructor @Slf4j public class FileMetadataController { @@ -20,17 +20,28 @@ public class FileMetadataController { /** * Lists all files and folders that belong to the provided path. - * Current API implementation requires a relative path, absolute path will be calculated based on authentication context * - * @param path relative path, for example: /inputs + * @param path relative path, for example: inputs/ */ - public Future list(String path) { + public Future list(String bucket, String path) { + String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); + String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); + + if (!expectedUserBucket.equals(decryptedBucket)) { + return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); + } + BlobStorage storage = proxy.getStorage(); return proxy.getVertx().executeBlocking(() -> { try { - String absolutePath = BlobStorageUtil.buildAbsoluteFilePath(context, path); - List metadata = storage.listMetadata(absolutePath); - context.respond(HttpStatus.OK, metadata); + String filePath = path.isEmpty() ? BlobStorageUtil.PATH_SEPARATOR : path; + ResourceDescription resource = ResourceDescription.from(ResourceType.FILES, bucket, decryptedBucket, filePath); + FileMetadataBase metadata = storage.listMetadata(resource); + if (metadata != null) { + context.respond(HttpStatus.OK, metadata); + } else { + context.respond(HttpStatus.NOT_FOUND); + } } catch (Exception ex) { log.error("Failed to list files", ex); context.respond(HttpStatus.INTERNAL_SERVER_ERROR, "Failed to list files by path %s".formatted(path)); diff --git a/src/main/java/com/epam/aidial/core/controller/UploadFileController.java b/src/main/java/com/epam/aidial/core/controller/UploadFileController.java index 9486f72d0..2fa599a23 100644 --- a/src/main/java/com/epam/aidial/core/controller/UploadFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/UploadFileController.java @@ -4,6 +4,8 @@ import com.epam.aidial.core.ProxyContext; import com.epam.aidial.core.storage.BlobStorageUtil; import com.epam.aidial.core.storage.BlobWriteStream; +import com.epam.aidial.core.storage.ResourceDescription; +import com.epam.aidial.core.storage.ResourceType; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; import io.vertx.core.Promise; @@ -22,25 +24,34 @@ public class UploadFileController { /** * Uploads file to the storage. - * Current API implementation requires a relative path, absolute path will be calculated based on authentication context. - * File name defined in multipart/form-data context + * Current API implementation requires bucket and relative file path * - * @param path relative path, for example: /inputs + * @param bucket bucket to write + * @param filePath relative path according to the bucket, for example: folder1/file.txt */ - public Future upload(String path) { - String absoluteFilePath = BlobStorageUtil.buildAbsoluteFilePath(context, path); + public Future upload(String bucket, String filePath) { + String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); + String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); + + if (filePath.isEmpty() || BlobStorageUtil.isFolder(filePath)) { + return context.respond(HttpStatus.BAD_REQUEST, "File name is missing"); + } + + if (!expectedUserBucket.equals(decryptedBucket)) { + return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); + } + + ResourceDescription resource = ResourceDescription.from(ResourceType.FILES, bucket, decryptedBucket, filePath); Promise result = Promise.promise(); context.getRequest() .setExpectMultipart(true) .uploadHandler(upload -> { String contentType = upload.contentType(); - String filename = upload.filename(); Pipe pipe = new PipeImpl<>(upload).endOnFailure(false); BlobWriteStream writeStream = new BlobWriteStream( proxy.getVertx(), proxy.getStorage(), - filename, - absoluteFilePath, + resource, contentType); pipe.to(writeStream, result); @@ -49,7 +60,7 @@ public Future upload(String path) { .onFailure(error -> { writeStream.abortUpload(error); context.respond(HttpStatus.INTERNAL_SERVER_ERROR, - "Failed to upload file by path " + path); + "Failed to upload file by path %s/%s".formatted(bucket, filePath)); }); }); diff --git a/src/main/java/com/epam/aidial/core/data/Bucket.java b/src/main/java/com/epam/aidial/core/data/Bucket.java new file mode 100644 index 000000000..21ef09f57 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/data/Bucket.java @@ -0,0 +1,4 @@ +package com.epam.aidial.core.data; + +public record Bucket(String bucket) { +} diff --git a/src/main/java/com/epam/aidial/core/data/FileMetadata.java b/src/main/java/com/epam/aidial/core/data/FileMetadata.java index 1a1fe6403..6e70c5805 100644 --- a/src/main/java/com/epam/aidial/core/data/FileMetadata.java +++ b/src/main/java/com/epam/aidial/core/data/FileMetadata.java @@ -1,5 +1,6 @@ package com.epam.aidial.core.data; +import com.epam.aidial.core.storage.ResourceDescription; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -11,9 +12,13 @@ public class FileMetadata extends FileMetadataBase { long contentLength; String contentType; - public FileMetadata(String name, String path, long contentLength, String contentType) { - super(name, path, FileType.FILE); + public FileMetadata(String bucket, String name, String path, String url, long contentLength, String contentType) { + super(name, path, bucket, url, FileType.FILE); this.contentLength = contentLength; this.contentType = contentType; } + + public FileMetadata(ResourceDescription resource, long contentLength, String contentType) { + this(resource.getBucketName(), resource.getName(), resource.getParentPath(), resource.getUrl(), contentLength, contentType); + } } diff --git a/src/main/java/com/epam/aidial/core/data/FileMetadataBase.java b/src/main/java/com/epam/aidial/core/data/FileMetadataBase.java index d75dc6b8d..700feacda 100644 --- a/src/main/java/com/epam/aidial/core/data/FileMetadataBase.java +++ b/src/main/java/com/epam/aidial/core/data/FileMetadataBase.java @@ -9,6 +9,8 @@ @NoArgsConstructor public abstract class FileMetadataBase { private String name; - private String path; + private String parentPath; + private String bucket; + private String url; private FileType type; } diff --git a/src/main/java/com/epam/aidial/core/data/FolderMetadata.java b/src/main/java/com/epam/aidial/core/data/FolderMetadata.java index 16d060d51..1ea16174d 100644 --- a/src/main/java/com/epam/aidial/core/data/FolderMetadata.java +++ b/src/main/java/com/epam/aidial/core/data/FolderMetadata.java @@ -1,8 +1,33 @@ package com.epam.aidial.core.data; +import com.epam.aidial.core.storage.ResourceDescription; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.List; + +@Getter +@Setter +@NoArgsConstructor public class FolderMetadata extends FileMetadataBase { - public FolderMetadata(String name, String path) { - super(name, path, FileType.FOLDER); + private List files; + + public FolderMetadata(String bucket, String name, String path, String url, List files) { + super(name, path, bucket, url, FileType.FOLDER); + this.files = files; + } + + public FolderMetadata(String bucket, String name, String path, String url) { + this(bucket, name, path, url, null); + } + + public FolderMetadata(ResourceDescription resource) { + this(resource, null); + } + + public FolderMetadata(ResourceDescription resource, List files) { + this(resource.getBucketName(), resource.getName(), resource.getParentPath(), resource.getUrl(), files); } } diff --git a/src/main/java/com/epam/aidial/core/security/EncryptionService.java b/src/main/java/com/epam/aidial/core/security/EncryptionService.java new file mode 100644 index 000000000..fc97b325a --- /dev/null +++ b/src/main/java/com/epam/aidial/core/security/EncryptionService.java @@ -0,0 +1,62 @@ +package com.epam.aidial.core.security; + +import com.epam.aidial.core.config.Encryption; +import lombok.extern.slf4j.Slf4j; + +import java.security.spec.KeySpec; +import java.util.Base64; +import java.util.Objects; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; + +@Slf4j +public class EncryptionService { + + private static final String TRANSFORMATION = "AES/CBC/PKCS5Padding"; + + private final SecretKey key; + private final IvParameterSpec iv = new IvParameterSpec( + new byte[]{25, -13, -25, -119, -42, 117, -118, -128, -101, 20, -103, -81, -48, -23, -54, -113}); + + public EncryptionService(Encryption config) { + this(config.getPassword(), config.getSalt()); + } + + EncryptionService(String password, String salt) { + try { + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256"); + KeySpec spec = new PBEKeySpec(Objects.requireNonNull(password).toCharArray(), Objects.requireNonNull(salt).getBytes(), 3000, 256); + key = new SecretKeySpec(secretKeyFactory.generateSecret(spec).getEncoded(), "AES"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public String encrypt(String value) { + try { + Cipher cipher = Cipher.getInstance(TRANSFORMATION); + cipher.init(Cipher.ENCRYPT_MODE, key, iv); + return Base64.getEncoder() + .encodeToString(cipher.doFinal(value.getBytes())); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public String decrypt(String value) { + try { + Cipher cipher = Cipher.getInstance(TRANSFORMATION); + cipher.init(Cipher.DECRYPT_MODE, key, iv); + return new String(cipher.doFinal(Base64.getDecoder() + .decode(value))); + } catch (Exception e) { + log.error("Failed to decrypt value " + value, e); + + return null; + } + } +} diff --git a/src/main/java/com/epam/aidial/core/storage/BlobStorage.java b/src/main/java/com/epam/aidial/core/storage/BlobStorage.java index 440f7e7da..4aa12ea0e 100644 --- a/src/main/java/com/epam/aidial/core/storage/BlobStorage.java +++ b/src/main/java/com/epam/aidial/core/storage/BlobStorage.java @@ -59,13 +59,12 @@ public BlobStorage(Storage config, Properties overrides) { /** * Initialize multipart upload * - * @param fileName name of the file, for example: data.csv - * @param path absolute path according to the bucket, for example: Users/user1/files/input data - * @param contentType MIME type of the content, for example: text/csv + * @param absoluteFilePath absolute path according to the bucket, for example: Users/user1/files/input/file.txt + * @param contentType MIME type of the content, for example: text/csv */ @SuppressWarnings("UnstableApiUsage") // multipart upload uses beta API - public MultipartUpload initMultipartUpload(String fileName, String path, String contentType) { - BlobMetadata metadata = buildBlobMetadata(fileName, path, contentType, bucketName); + public MultipartUpload initMultipartUpload(String absoluteFilePath, String contentType) { + BlobMetadata metadata = buildBlobMetadata(absoluteFilePath, contentType, bucketName); return blobStore.initiateMultipartUpload(bucketName, metadata, PutOptions.NONE); } @@ -102,14 +101,12 @@ public void abortMultipartUpload(MultipartUpload multipart) { /** * Upload file in a single request * - * @param fileName name of the file, for example: data.csv - * @param path absolute path according to the bucket, for example: Users/user1/files/input data - * @param contentType MIME type of the content, for example: text/csv - * @param data whole content data + * @param absoluteFilePath absolute path according to the bucket, for example: Users/user1/files/input/file.txt + * @param contentType MIME type of the content, for example: text/csv + * @param data whole content data */ - public void store(String fileName, String path, String contentType, Buffer data) { - String filePath = BlobStorageUtil.buildFilePath(fileName, path); - Blob blob = blobStore.blobBuilder(filePath) + public void store(String absoluteFilePath, String contentType, Buffer data) { + Blob blob = blobStore.blobBuilder(absoluteFilePath) .payload(new BufferPayload(data)) .contentLength(data.length()) .contentType(contentType) @@ -138,14 +135,23 @@ public void delete(String filePath) { } /** - * List all files/folder metadata for a given path - * - * @param path absolute path for a folder, for example: Users/user1/files + * List all files/folder metadata for a given resource */ - public List listMetadata(String path) { - ListContainerOptions options = buildListContainerOptions(BlobStorageUtil.normalizePathForQuery(path)); - PageSet list = blobStore.list(bucketName, options); - return list.stream().map(BlobStorage::buildFileMetadata).toList(); + public FileMetadataBase listMetadata(ResourceDescription resource) { + ListContainerOptions options = buildListContainerOptions(resource.getAbsoluteFilePath()); + PageSet list = blobStore.list(this.bucketName, options); + List filesMetadata = list.stream().map(meta -> buildFileMetadata(resource, meta)).toList(); + + // listing folder + if (resource.isFolder()) { + return new FolderMetadata(resource, filesMetadata); + } else { + // listing file + if (filesMetadata.size() == 1) { + return filesMetadata.get(0); + } + return null; + } } @Override @@ -159,27 +165,27 @@ private static ListContainerOptions buildListContainerOptions(String prefix) { .delimiter(BlobStorageUtil.PATH_SEPARATOR); } - private static FileMetadataBase buildFileMetadata(StorageMetadata metadata) { - String absoluteFilePath = metadata.getName(); - String[] elements = absoluteFilePath.split(BlobStorageUtil.PATH_SEPARATOR); - String lastElement = elements[elements.length - 1]; - String path = absoluteFilePath.substring(0, absoluteFilePath.length() - lastElement.length() - 1); - String normalizedPath = BlobStorageUtil.normalizeParentPath(path); + private static FileMetadataBase buildFileMetadata(ResourceDescription resource, StorageMetadata metadata) { + String bucketName = resource.getBucketName(); + ResourceDescription resultResource = getResourceDescription(resource.getType(), bucketName, + resource.getBucketLocation(), metadata.getName()); return switch (metadata.getType()) { - case BLOB -> - new FileMetadata(lastElement, normalizedPath, metadata.getSize(), - ((BlobMetadata) metadata).getContentMetadata().getContentType()); - case FOLDER, RELATIVE_PATH -> - new FolderMetadata(lastElement, normalizedPath); + case BLOB -> new FileMetadata(resultResource, metadata.getSize(), + ((BlobMetadata) metadata).getContentMetadata().getContentType()); + case FOLDER, RELATIVE_PATH -> new FolderMetadata(resultResource); case CONTAINER -> throw new IllegalArgumentException("Can't list container"); }; } - private static BlobMetadata buildBlobMetadata(String fileName, String path, String contentType, String bucketName) { - String filePath = BlobStorageUtil.buildFilePath(fileName, path); + private static ResourceDescription getResourceDescription(ResourceType resourceType, String bucketName, String bucketLocation, String absoluteFilePath) { + String relativeFilePath = absoluteFilePath.substring(bucketLocation.length() + resourceType.getName().length()); + return ResourceDescription.from(resourceType, bucketName, bucketLocation, relativeFilePath); + } + + private static BlobMetadata buildBlobMetadata(String absoluteFilePath, String contentType, String bucketName) { ContentMetadata contentMetadata = buildContentMetadata(contentType); - return new BlobMetadataImpl(null, filePath, null, null, null, null, null, Map.of(), null, bucketName, contentMetadata, null, Tier.STANDARD); + return new BlobMetadataImpl(null, absoluteFilePath, null, null, null, null, null, Map.of(), null, bucketName, contentMetadata, null, Tier.STANDARD); } private static ContentMetadata buildContentMetadata(String contentType) { diff --git a/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java b/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java index c6d2a17f7..7505d5dd6 100644 --- a/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java +++ b/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java @@ -7,92 +7,37 @@ @UtilityClass public class BlobStorageUtil { - private static final String USER_ROOT_DIR_PATTERN = "Users/%s/files/%s"; + private static final String USER_BUCKET_PATTERN = "Users/%s/"; - private static final String API_KEY_ROOT_DIR_PATTERN = "Keys/%s/files/%s"; + private static final String API_KEY_BUCKET_PATTERN = "Keys/%s/"; public static final String PATH_SEPARATOR = "/"; - private static final char DELIMITER = PATH_SEPARATOR.charAt(0); - - /** - * Normalize provided path for listing files query by removing leading and adding trailing path separator. - * For example, path /Users/User1/files/folders will be transformed to Users/User1/files/folders/ - * - * @return normalized path - */ - public String normalizePathForQuery(String path) { - if (path == null || path.isBlank()) { - return null; - } - - if (path.equals(PATH_SEPARATOR)) { - return path; - } - - // remove leading separator - if (path.charAt(0) == DELIMITER) { - path = path.substring(1); - } - - // add trailing separator if needed - return path.charAt(path.length() - 1) == DELIMITER ? path : path + PATH_SEPARATOR; - } - - /** - * Normalize parent path for file metadata by adding leading path separator and removing trailing one. - * For example, path Users/User1/files/folder1/ will be transformed to /Users/User1/files/folder1 - * - * @return normalized path - */ - public String normalizeParentPath(String path) { - path = removeTrailingPathSeparator(path); - - return path.charAt(0) == DELIMITER ? path : PATH_SEPARATOR + path; - } - - public String removeLeadingPathSeparator(String path) { - if (path == null || path.isBlank()) { - return null; - } - return path.charAt(0) == DELIMITER ? path.substring(1) : path; - } - - public String buildFilePath(String fileName, String path) { - path = removeTrailingPathSeparator(path); - return path + PATH_SEPARATOR + fileName; - } public String getContentType(String fileName) { String mimeType = MimeMapping.getMimeTypeForFilename(fileName); return mimeType == null ? "application/octet-stream" : mimeType; } - public String buildAbsoluteFilePath(ProxyContext context, String path) { - return buildAbsoluteFilePath(context.getUserSub(), context.getProject(), path); - } + public String buildUserBucket(ProxyContext context) { + String userSub = context.getUserSub(); + String apiKeyId = context.getProject(); - private String buildAbsoluteFilePath(String userSub, String apiKeyId, String path) { - path = removeLeadingAndTrailingPathSeparators(path); if (userSub != null) { - return USER_ROOT_DIR_PATTERN.formatted(userSub, path); - } else { - return API_KEY_ROOT_DIR_PATTERN.formatted(apiKeyId, path); + return USER_BUCKET_PATTERN.formatted(userSub); } - } - String removeTrailingPathSeparator(String path) { - if (path == null || path.isBlank()) { - return null; + if (apiKeyId != null) { + return API_KEY_BUCKET_PATTERN.formatted(apiKeyId); } - int length = path.length(); - return path.charAt(length - 1) == DELIMITER ? path.substring(0, length - 1) : path; + + throw new IllegalArgumentException("Can't find user bucket. Either user sub or api-key project must be provided"); } - String removeLeadingAndTrailingPathSeparators(String path) { - if (path == null || path.isBlank() || path.equals(PATH_SEPARATOR)) { - return ""; - } - path = removeLeadingPathSeparator(path); - return removeTrailingPathSeparator(path); + public String buildAbsoluteFilePath(ResourceType resource, String bucket, String path) { + return bucket + resource.getName() + PATH_SEPARATOR + path; + } + + public boolean isFolder(String path) { + return path.endsWith(PATH_SEPARATOR); } } diff --git a/src/main/java/com/epam/aidial/core/storage/BlobWriteStream.java b/src/main/java/com/epam/aidial/core/storage/BlobWriteStream.java index 089589175..725dc45b2 100644 --- a/src/main/java/com/epam/aidial/core/storage/BlobWriteStream.java +++ b/src/main/java/com/epam/aidial/core/storage/BlobWriteStream.java @@ -28,8 +28,7 @@ public class BlobWriteStream implements WriteStream { private final Vertx vertx; private final BlobStorage storage; - private final String parentPath; - private final String fileName; + private final ResourceDescription resource; private final String contentType; private final Buffer chunkBuffer = Buffer.buffer(); @@ -51,14 +50,12 @@ public class BlobWriteStream implements WriteStream { public BlobWriteStream(Vertx vertx, BlobStorage storage, - String fileName, - String parentPath, + ResourceDescription resource, String contentType) { this.vertx = vertx; this.storage = storage; - this.fileName = fileName; - this.parentPath = parentPath; - this.contentType = contentType != null ? contentType : BlobStorageUtil.getContentType(fileName); + this.resource = resource; + this.contentType = contentType != null ? contentType : BlobStorageUtil.getContentType(resource.getName()); } @Override @@ -103,11 +100,10 @@ public void end(Handler> handler) { } Buffer lastChunk = chunkBuffer.slice(0, position); - metadata = new FileMetadata(fileName, BlobStorageUtil.normalizeParentPath(parentPath), - bytesHandled, contentType); + metadata = new FileMetadata(resource, bytesHandled, contentType); if (mpu == null) { log.info("Resource is too small for multipart upload, sending as a regular blob"); - storage.store(fileName, parentPath, contentType, lastChunk); + storage.store(resource.getAbsoluteFilePath(), contentType, lastChunk); } else { if (position != 0) { MultipartPart part = storage.storeMultipartPart(mpu, ++chunkNumber, lastChunk); @@ -150,7 +146,7 @@ public WriteStream drainHandler(Handler handler) { synchronized (BlobWriteStream.this) { try { if (mpu == null) { - mpu = storage.initMultipartUpload(fileName, parentPath, contentType); + mpu = storage.initMultipartUpload(resource.getAbsoluteFilePath(), contentType); } MultipartPart part = storage.storeMultipartPart(mpu, ++chunkNumber, chunkBuffer.slice(0, position)); parts.add(part); diff --git a/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java b/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java new file mode 100644 index 000000000..dc8c94082 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java @@ -0,0 +1,85 @@ +package com.epam.aidial.core.storage; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.stream.Collectors; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ResourceDescription { + ResourceType type; + String name; + List parentFolders; + String bucketName; + String bucketLocation; + boolean isFolder; + + public String getUrl() { + StringBuilder builder = new StringBuilder(); + builder.append(encodeToUrl(bucketName)) + .append(BlobStorageUtil.PATH_SEPARATOR); + if (parentFolders != null) { + String parentPath = parentFolders.stream() + .map(ResourceDescription::encodeToUrl) + .collect(Collectors.joining(BlobStorageUtil.PATH_SEPARATOR)); + builder.append(parentPath) + .append(BlobStorageUtil.PATH_SEPARATOR); + } + if (name != null && !name.equals(BlobStorageUtil.PATH_SEPARATOR)) { + builder.append(encodeToUrl(name)); + + if (isFolder) { + builder.append(BlobStorageUtil.PATH_SEPARATOR); + } + } + + return builder.toString(); + } + + public String getAbsoluteFilePath() { + StringBuilder builder = new StringBuilder(); + if (parentFolders != null) { + builder.append(getParentPath()); + } + if (name != null && !name.equals(BlobStorageUtil.PATH_SEPARATOR)) { + builder.append(BlobStorageUtil.PATH_SEPARATOR).append(name); + + if (isFolder) { + builder.append(BlobStorageUtil.PATH_SEPARATOR); + } + } + + return BlobStorageUtil.buildAbsoluteFilePath(type, bucketLocation, builder.toString()); + } + + public String getParentPath() { + return parentFolders == null ? null : String.join(BlobStorageUtil.PATH_SEPARATOR, parentFolders); + } + + public static ResourceDescription from(ResourceType type, String bucketName, String bucketLocation, String relativeFilePath) { + String[] elements = relativeFilePath.split(BlobStorageUtil.PATH_SEPARATOR); + List parentFolders = null; + String name = "/"; + if (elements.length > 0) { + name = elements[elements.length - 1]; + } + if (elements.length > 1) { + String parentPath = relativeFilePath.substring(0, relativeFilePath.length() - name.length() - 1); + if (!parentPath.isEmpty() && !parentPath.equals(BlobStorageUtil.PATH_SEPARATOR)) { + parentFolders = List.of(parentPath.split(BlobStorageUtil.PATH_SEPARATOR)); + } + } + + return new ResourceDescription(type, name, parentFolders, bucketName, bucketLocation, BlobStorageUtil.isFolder(relativeFilePath)); + } + + private static String encodeToUrl(String value) { + return URLEncoder.encode(value, StandardCharsets.UTF_8); + } +} diff --git a/src/main/java/com/epam/aidial/core/storage/ResourceType.java b/src/main/java/com/epam/aidial/core/storage/ResourceType.java new file mode 100644 index 000000000..f9a45ae7f --- /dev/null +++ b/src/main/java/com/epam/aidial/core/storage/ResourceType.java @@ -0,0 +1,15 @@ +package com.epam.aidial.core.storage; + +public enum ResourceType { + FILES("files"); + + private final String name; + + ResourceType(String name) { + this.name = name; + } + + public String getName() { + return name; + } +} diff --git a/src/test/java/com/epam/aidial/core/FileApiTest.java b/src/test/java/com/epam/aidial/core/FileApiTest.java index 05bd0e082..9b10d74a9 100644 --- a/src/test/java/com/epam/aidial/core/FileApiTest.java +++ b/src/test/java/com/epam/aidial/core/FileApiTest.java @@ -2,14 +2,14 @@ import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; +import com.epam.aidial.core.data.Bucket; import com.epam.aidial.core.data.FileMetadata; import com.epam.aidial.core.data.FolderMetadata; +import com.epam.aidial.core.util.ProxyUtil; import io.vertx.core.Future; import io.vertx.core.Promise; import io.vertx.core.Vertx; import io.vertx.core.buffer.Buffer; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; import io.vertx.ext.web.client.WebClient; import io.vertx.ext.web.codec.BodyCodec; import io.vertx.ext.web.multipart.MultipartForm; @@ -25,9 +25,9 @@ import org.junit.jupiter.api.extension.ExtendWith; import java.nio.file.Path; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertIterableEquals; import static org.junit.jupiter.api.Assertions.assertNull; @ExtendWith(VertxExtension.class) @@ -68,17 +68,34 @@ public static void destroy() { dial.stop(); } + @Test + public void testBucket(Vertx vertx, VertxTestContext context) { + WebClient client = WebClient.create(vertx); + client.get(serverPort, "localhost", "/v1/bucket") + .putHeader("Api-key", "proxyKey2") + .as(BodyCodec.json(Bucket.class)) + .send(context.succeeding(response -> { + context.verify(() -> { + assertEquals(200, response.statusCode()); + assertEquals(new Bucket("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM="), response.body()); + context.completeNow(); + }); + })); + } + @Test public void testEmptyFilesList(Vertx vertx, VertxTestContext context) { WebClient client = WebClient.create(vertx); - client.get(serverPort, "localhost", "/v1/files") + + FolderMetadata emptyBucketResponse = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "/", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/", List.of()); + client.get(serverPort, "localhost", "/v1/files/metadata/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=/") .putHeader("Api-key", "proxyKey2") - .addQueryParam("purpose", "metadata") - .as(BodyCodec.jsonArray()) + .as(BodyCodec.json(FolderMetadata.class)) .send(context.succeeding(response -> { context.verify(() -> { assertEquals(200, response.statusCode()); - assertEquals(JsonArray.of(), response.body()); + assertEquals(emptyBucketResponse, response.body()); context.completeNow(); }); })); @@ -87,7 +104,7 @@ public void testEmptyFilesList(Vertx vertx, VertxTestContext context) { @Test public void testFileNotFound(Vertx vertx, VertxTestContext context) { WebClient client = WebClient.create(vertx); - client.get(serverPort, "localhost", "/v1/files/test_file.txt") + client.get(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.buffer()) .send(context.succeeding(response -> { @@ -99,24 +116,115 @@ public void testFileNotFound(Vertx vertx, VertxTestContext context) { })); } + @Test + public void testInvalidFileUploadUrl(Vertx vertx, VertxTestContext context) { + WebClient client = WebClient.create(vertx); + client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/") + .putHeader("Api-key", "proxyKey2") + .as(BodyCodec.string()) + .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), + context.succeeding(response -> { + context.verify(() -> { + assertEquals(400, response.statusCode()); + assertEquals("File name is missing", response.body()); + context.completeNow(); + }); + }) + ); + } + + @Test + public void testInvalidFileUploadUrl2(Vertx vertx, VertxTestContext context) { + WebClient client = WebClient.create(vertx); + client.put(serverPort, "localhost", "/v1/files/bucket/") + .putHeader("Api-key", "proxyKey2") + .as(BodyCodec.string()) + .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), + context.succeeding(response -> { + context.verify(() -> { + assertEquals(400, response.statusCode()); + assertEquals("File name is missing", response.body()); + context.completeNow(); + }); + }) + ); + } + + @Test + public void testDownloadFromAnotherBucket(Vertx vertx, VertxTestContext context) { + Checkpoint checkpoint = context.checkpoint(2); + WebClient client = WebClient.create(vertx); + + FileMetadata expectedFileMetadata = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "file.txt", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt", 17, "text/custom"); + + Future.succeededFuture().compose((mapper) -> { + Promise promise = Promise.promise(); + client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") + .putHeader("Api-key", "proxyKey2") + .as(BodyCodec.json(FileMetadata.class)) + .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), + context.succeeding(response -> { + context.verify(() -> { + assertEquals(200, response.statusCode()); + assertEquals(expectedFileMetadata, response.body()); + checkpoint.flag(); + promise.complete(); + }); + }) + ); + return promise.future(); + }).andThen((mapper) -> { + client.get(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") + .putHeader("Api-key", "proxyKey1") + .as(BodyCodec.string()) + .send(context.succeeding(response -> { + context.verify(() -> { + assertEquals(403, response.statusCode()); + checkpoint.flag(); + }); + })); + }); + } + + @Test + public void testFileUploadIntoAnotherBucket(Vertx vertx, VertxTestContext context) { + WebClient client = WebClient.create(vertx); + client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") + .putHeader("Api-key", "proxyKey1") + .as(BodyCodec.string()) + .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), + context.succeeding(response -> { + context.verify(() -> { + assertEquals(403, response.statusCode()); + context.completeNow(); + }); + }) + ); + } + @Test public void testFileUpload(Vertx vertx, VertxTestContext context) { Checkpoint checkpoint = context.checkpoint(3); WebClient client = WebClient.create(vertx); - FileMetadata expectedFileMetadata = new FileMetadata("file.txt", "/Keys/EPM-RTC-RAIL/files", 17, "text/custom"); + FolderMetadata emptyFolderResponse = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "/", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/", List.of()); + FileMetadata expectedFileMetadata = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "file.txt", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt", 17, "text/custom"); + FolderMetadata expectedFolderMetadata = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "/", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/", List.of(expectedFileMetadata)); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // verify no files - client.get(serverPort, "localhost", "/v1/files") + client.get(serverPort, "localhost", "/v1/files/metadata/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/") .putHeader("Api-key", "proxyKey2") - .addQueryParam("purpose", "metadata") - .as(BodyCodec.jsonArray()) + .as(BodyCodec.json(FolderMetadata.class)) .send(context.succeeding(response -> { context.verify(() -> { assertEquals(200, response.statusCode()); - assertEquals(JsonArray.of(), response.body()); + assertEquals(emptyFolderResponse, response.body()); checkpoint.flag(); promise.complete(); }); @@ -126,7 +234,7 @@ public void testFileUpload(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // upload test file - client.post(serverPort, "localhost", "/v1/files") + client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -143,14 +251,13 @@ public void testFileUpload(Vertx vertx, VertxTestContext context) { return promise.future(); }).andThen((result) -> { // verify uploaded file can be listed - client.get(serverPort, "localhost", "/v1/files") + client.get(serverPort, "localhost", "/v1/files/metadata/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/") .putHeader("Api-key", "proxyKey2") - .addQueryParam("purpose", "metadata") - .as(BodyCodec.jsonArray()) + .as(BodyCodec.string()) .send(context.succeeding(response -> { context.verify(() -> { assertEquals(200, response.statusCode()); - assertIterableEquals(JsonArray.of(JsonObject.mapFrom(expectedFileMetadata)), response.body()); + assertEquals(ProxyUtil.MAPPER.writeValueAsString(expectedFolderMetadata), response.body()); checkpoint.flag(); }); })); @@ -159,15 +266,16 @@ public void testFileUpload(Vertx vertx, VertxTestContext context) { @Test public void testFileDownload(Vertx vertx, VertxTestContext context) { - Checkpoint checkpoint = context.checkpoint(3); + Checkpoint checkpoint = context.checkpoint(2); WebClient client = WebClient.create(vertx); - FileMetadata expectedFileMetadata = new FileMetadata("file.txt", "/Keys/EPM-RTC-RAIL/files/folder1", 17, "text/plain"); + FileMetadata expectedFileMetadata = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "file.txt", "folder1", "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/file.txt", 17, "text/plain"); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // upload test file - client.post(serverPort, "localhost", "/v1/files/folder1") + client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT), @@ -181,27 +289,9 @@ public void testFileDownload(Vertx vertx, VertxTestContext context) { }) ); - return promise.future(); - }).compose((mapper) -> { - Promise promise = Promise.promise(); - // download by relative path - client.get(serverPort, "localhost", "/v1/files/folder1/file.txt") - .putHeader("Api-key", "proxyKey2") - .addQueryParam("path", "relative") - .as(BodyCodec.string()) - .send(context.succeeding(response -> { - context.verify(() -> { - assertEquals(200, response.statusCode()); - assertEquals(TEST_FILE_CONTENT, response.body()); - checkpoint.flag(); - promise.complete(); - }); - })); - return promise.future(); }).andThen((result) -> { - // download by absolute path - client.get(serverPort, "localhost", "/v1/files/Keys/EPM-RTC-RAIL/files/folder1/file.txt") + client.get(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .send(context.succeeding(response -> { @@ -219,21 +309,29 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { Checkpoint checkpoint = context.checkpoint(4); WebClient client = WebClient.create(vertx); - FileMetadata expectedFileMetadata1 = new FileMetadata("file.txt", "/Keys/EPM-RTC-RAIL/files", 17, "text/custom"); - FileMetadata expectedFileMetadata2 = new FileMetadata("file.txt", "/Keys/EPM-RTC-RAIL/files/folder1", 17, "text/custom"); - FolderMetadata expectedFolderMetadata = new FolderMetadata("folder1", "/Keys/EPM-RTC-RAIL/files"); + FolderMetadata emptyFolderResponse = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "/", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/", List.of()); + + FileMetadata expectedFileMetadata1 = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "file.txt", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt", 17, "text/custom"); + FileMetadata expectedFileMetadata2 = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "file.txt", "folder1", "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/file.txt", 17, "text/custom"); + FolderMetadata expectedFolder1Metadata = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "folder1", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/"); + FolderMetadata expectedRootFolderMetadata = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "/", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/", + List.of(expectedFileMetadata1, expectedFolder1Metadata)); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // verify no files - client.get(serverPort, "localhost", "/v1/files") + client.get(serverPort, "localhost", "/v1/files/metadata/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/") .putHeader("Api-key", "proxyKey2") - .addQueryParam("purpose", "metadata") - .as(BodyCodec.jsonArray()) + .as(BodyCodec.json(FolderMetadata.class)) .send(context.succeeding(response -> { context.verify(() -> { assertEquals(200, response.statusCode()); - assertEquals(JsonArray.of(), response.body()); + assertEquals(emptyFolderResponse, response.body()); checkpoint.flag(); promise.complete(); }); @@ -243,7 +341,7 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // upload test file1 - client.post(serverPort, "localhost", "/v1/files") + client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -261,7 +359,7 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // upload test file2 - client.post(serverPort, "localhost", "/v1/files/folder1") + client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -278,17 +376,13 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { return promise.future(); }).andThen((result) -> { // verify uploaded files can be listed - client.get(serverPort, "localhost", "/v1/files") + client.get(serverPort, "localhost", "/v1/files/metadata/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/") .putHeader("Api-key", "proxyKey2") - .addQueryParam("purpose", "metadata") - .as(BodyCodec.jsonArray()) + .as(BodyCodec.string()) .send(context.succeeding(response -> { context.verify(() -> { assertEquals(200, response.statusCode()); - assertIterableEquals( - JsonArray.of(JsonObject.mapFrom(expectedFileMetadata1), - JsonObject.mapFrom(expectedFolderMetadata)), - response.body()); + assertEquals(ProxyUtil.MAPPER.writeValueAsString(expectedRootFolderMetadata), response.body()); checkpoint.flag(); }); })); @@ -300,12 +394,13 @@ public void testFileDelete(Vertx vertx, VertxTestContext context) { Checkpoint checkpoint = context.checkpoint(3); WebClient client = WebClient.create(vertx); - FileMetadata expectedFileMetadata = new FileMetadata("test_file.txt", "/Keys/EPM-RTC-RAIL/files", 17, "text/plain"); + FileMetadata expectedFileMetadata = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", + "test_file.txt", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/test_file.txt", 17, "text/plain"); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // upload test file - client.post(serverPort, "localhost", "/v1/files") + client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/test_file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("test_file.txt", TEST_FILE_CONTENT), @@ -323,7 +418,7 @@ public void testFileDelete(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // delete file - client.delete(serverPort, "localhost", "/v1/files/test_file.txt") + client.delete(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/test_file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .send(context.succeeding(response -> { @@ -337,7 +432,7 @@ public void testFileDelete(Vertx vertx, VertxTestContext context) { return promise.future(); }).andThen((mapper) -> { // try to download deleted file - client.get(serverPort, "localhost", "/v1/files/test_file.txt") + client.get(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/test_file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .send(context.succeeding(response -> { diff --git a/src/test/java/com/epam/aidial/core/controller/ControllerSelectorTest.java b/src/test/java/com/epam/aidial/core/controller/ControllerSelectorTest.java index b87f3ab0e..1a51d008b 100644 --- a/src/test/java/com/epam/aidial/core/controller/ControllerSelectorTest.java +++ b/src/test/java/com/epam/aidial/core/controller/ControllerSelectorTest.java @@ -2,7 +2,6 @@ import com.epam.aidial.core.Proxy; import com.epam.aidial.core.ProxyContext; -import io.vertx.core.MultiMap; import io.vertx.core.http.HttpMethod; import io.vertx.core.http.HttpServerRequest; import org.junit.jupiter.api.BeforeEach; @@ -20,7 +19,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -181,37 +179,36 @@ public void testSelectGetApplicationsController() { @Test public void testSelectListMetadataFileController() { - when(request.path()).thenReturn("/v1/files/folder1/file1?purpose=metadata"); + when(request.path()).thenReturn("/v1/files/metadata/bucket/file1"); when(request.method()).thenReturn(HttpMethod.GET); - MultiMap params = mock(MultiMap.class); - when(request.params()).thenReturn(params); - when(params.get("purpose")).thenReturn("metadata"); Controller controller = ControllerSelector.select(proxy, context); assertNotNull(controller); SerializedLambda lambda = getSerializedLambda(controller); assertNotNull(lambda); - assertEquals(2, lambda.getCapturedArgCount()); + assertEquals(3, lambda.getCapturedArgCount()); Object arg1 = lambda.getCapturedArg(0); Object arg2 = lambda.getCapturedArg(1); + Object arg3 = lambda.getCapturedArg(2); assertInstanceOf(FileMetadataController.class, arg1); - assertEquals("/folder1/file1?purpose=metadata", arg2); + assertEquals("bucket", arg2); + assertEquals("file1", arg3); } @Test public void testSelectDownloadFileController() { - when(request.path()).thenReturn("/v1/files/folder1/file1"); + when(request.path()).thenReturn("/v1/files/bucket/folder1/file1.txt"); when(request.method()).thenReturn(HttpMethod.GET); - MultiMap params = mock(MultiMap.class); - when(request.params()).thenReturn(params); Controller controller = ControllerSelector.select(proxy, context); assertNotNull(controller); SerializedLambda lambda = getSerializedLambda(controller); assertNotNull(lambda); - assertEquals(2, lambda.getCapturedArgCount()); + assertEquals(3, lambda.getCapturedArgCount()); Object arg1 = lambda.getCapturedArg(0); Object arg2 = lambda.getCapturedArg(1); + Object arg3 = lambda.getCapturedArg(2); assertInstanceOf(DownloadFileController.class, arg1); - assertEquals("/folder1/file1", arg2); + assertEquals("bucket", arg2); + assertEquals("folder1/file1.txt", arg3); } @Test @@ -233,32 +230,36 @@ public void testSelectPostDeploymentController() { @Test public void testSelectUploadFileController() { - when(request.path()).thenReturn("/v1/files/folder1/file1"); - when(request.method()).thenReturn(HttpMethod.POST); + when(request.path()).thenReturn("/v1/files/bucket/folder1/file1.txt"); + when(request.method()).thenReturn(HttpMethod.PUT); Controller controller = ControllerSelector.select(proxy, context); assertNotNull(controller); SerializedLambda lambda = getSerializedLambda(controller); assertNotNull(lambda); - assertEquals(2, lambda.getCapturedArgCount()); + assertEquals(3, lambda.getCapturedArgCount()); Object arg1 = lambda.getCapturedArg(0); Object arg2 = lambda.getCapturedArg(1); + Object arg3 = lambda.getCapturedArg(2); assertInstanceOf(UploadFileController.class, arg1); - assertEquals("/folder1/file1", arg2); + assertEquals("bucket", arg2); + assertEquals("folder1/file1.txt", arg3); } @Test public void testSelectDeleteFileController() { - when(request.path()).thenReturn("/v1/files/folder1/file1"); + when(request.path()).thenReturn("/v1/files/bucket/folder1/file1.txt"); when(request.method()).thenReturn(HttpMethod.DELETE); Controller controller = ControllerSelector.select(proxy, context); assertNotNull(controller); SerializedLambda lambda = getSerializedLambda(controller); assertNotNull(lambda); - assertEquals(2, lambda.getCapturedArgCount()); + assertEquals(3, lambda.getCapturedArgCount()); Object arg1 = lambda.getCapturedArg(0); Object arg2 = lambda.getCapturedArg(1); + Object arg3 = lambda.getCapturedArg(2); assertInstanceOf(DeleteFileController.class, arg1); - assertEquals("/folder1/file1", arg2); + assertEquals("bucket", arg2); + assertEquals("folder1/file1.txt", arg3); } @Test diff --git a/src/test/java/com/epam/aidial/core/storage/BlobStorageUtilTest.java b/src/test/java/com/epam/aidial/core/storage/BlobStorageUtilTest.java deleted file mode 100644 index eccdf6818..000000000 --- a/src/test/java/com/epam/aidial/core/storage/BlobStorageUtilTest.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.epam.aidial.core.storage; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import static com.epam.aidial.core.storage.BlobStorageUtil.normalizePathForQuery; -import static com.epam.aidial.core.storage.BlobStorageUtil.removeLeadingAndTrailingPathSeparators; -import static com.epam.aidial.core.storage.BlobStorageUtil.removeLeadingPathSeparator; -import static com.epam.aidial.core.storage.BlobStorageUtil.removeTrailingPathSeparator; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - -public class BlobStorageUtilTest { - - @Test - public void testNormalizePathForQuery() { - assertEquals("Users/User/files/", normalizePathForQuery("/Users/User/files")); - assertEquals("Users/User/files/", normalizePathForQuery("Users/User/files")); - assertEquals("folder/", normalizePathForQuery("folder")); - assertEquals("/", normalizePathForQuery("/")); - assertNull(normalizePathForQuery("")); - } - - @Test - public void testRemoveLeadingPathSeparator() { - assertEquals("Users/User/files/", removeLeadingPathSeparator("/Users/User/files/")); - assertEquals("Users/User/files", removeLeadingPathSeparator("Users/User/files")); - assertEquals("folder/", removeLeadingPathSeparator("/folder/")); - assertEquals("", removeLeadingPathSeparator("/")); - assertNull(removeLeadingPathSeparator("")); - } - - @Test - public void testRemoveTrailingPathSeparator() { - assertEquals("/Users/User/files", removeTrailingPathSeparator("/Users/User/files/")); - assertEquals("Users/User/files", removeTrailingPathSeparator("Users/User/files")); - assertEquals("/folder", removeTrailingPathSeparator("/folder/")); - assertEquals("", removeTrailingPathSeparator("/")); - assertNull(removeTrailingPathSeparator("")); - } - - @Test - public void testRemoveLeadingAndTrailingPathSeparators() { - assertEquals("Users/User/files", removeLeadingAndTrailingPathSeparators("/Users/User/files/")); - assertEquals("Users/User/files", removeLeadingAndTrailingPathSeparators("Users/User/files")); - assertEquals("folder", removeLeadingAndTrailingPathSeparators("/folder/")); - assertEquals("", removeLeadingAndTrailingPathSeparators(null)); - assertEquals("", removeLeadingAndTrailingPathSeparators("/")); - assertEquals("", removeLeadingAndTrailingPathSeparators("")); - } -} diff --git a/src/test/resources/aidial.settings.json b/src/test/resources/aidial.settings.json index 681cdecd6..c9b1cf14e 100644 --- a/src/test/resources/aidial.settings.json +++ b/src/test/resources/aidial.settings.json @@ -44,5 +44,9 @@ "rolePath": "roles", "issuerPattern": "issuer" } + }, + "encryption": { + "password": "password", + "salt": "salt" } } From be289d3efc3e164d09dff0df66bff3b31e99675b Mon Sep 17 00:00:00 2001 From: Maksim_Hadalau Date: Sun, 17 Dec 2023 17:46:05 +0100 Subject: [PATCH 2/5] address review comments --- .../AccessControlBaseController.java | 30 ++++ .../core/controller/ControllerSelector.java | 8 +- .../core/controller/DeleteFileController.java | 26 +--- .../controller/DownloadFileController.java | 31 ++--- .../controller/FileMetadataController.java | 29 ++-- .../core/controller/UploadFileController.java | 30 ++-- .../core/security/EncryptionService.java | 7 +- .../epam/aidial/core/storage/BlobStorage.java | 2 +- .../aidial/core/storage/BlobStorageUtil.java | 2 +- .../core/storage/ResourceDescription.java | 28 +++- .../aidial/core/storage/ResourceType.java | 12 +- .../com/epam/aidial/core/FileApiTest.java | 6 +- .../core/storage/ResourceDescriptionTest.java | 129 ++++++++++++++++++ 13 files changed, 232 insertions(+), 108 deletions(-) create mode 100644 src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java create mode 100644 src/test/java/com/epam/aidial/core/storage/ResourceDescriptionTest.java diff --git a/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java b/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java new file mode 100644 index 000000000..c5fb3e3ae --- /dev/null +++ b/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java @@ -0,0 +1,30 @@ +package com.epam.aidial.core.controller; + +import com.epam.aidial.core.Proxy; +import com.epam.aidial.core.ProxyContext; +import com.epam.aidial.core.storage.BlobStorageUtil; +import com.epam.aidial.core.util.HttpStatus; +import io.vertx.core.Future; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public abstract class AccessControlBaseController { + + final Proxy proxy; + final ProxyContext context; + + + public Future handle(String bucket, String filePath) { + String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); + String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); + + if (!expectedUserBucket.equals(decryptedBucket)) { + return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); + } + + return handle(bucket, decryptedBucket, filePath); + } + + protected abstract Future handle(String bucketName, String bucketLocation, String filePath); + +} diff --git a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java index 0ffd5b128..b1c400697 100644 --- a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java +++ b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java @@ -134,7 +134,7 @@ private static Controller selectGet(Proxy proxy, ProxyContext context, String pa String bucket = match.group(1); String filePath = match.group(2); FileMetadataController controller = new FileMetadataController(proxy, context); - return () -> controller.list(bucket, filePath); + return () -> controller.handle(bucket, filePath); } match = match(PATTERN_FILES, path); @@ -142,7 +142,7 @@ private static Controller selectGet(Proxy proxy, ProxyContext context, String pa String bucket = match.group(1); String filePath = match.group(2); DownloadFileController controller = new DownloadFileController(proxy, context); - return () -> controller.download(bucket, filePath); + return () -> controller.handle(bucket, filePath); } match = match(PATTERN_BUCKET, path); @@ -217,7 +217,7 @@ private static Controller selectDelete(Proxy proxy, ProxyContext context, String String bucket = match.group(1); String filePath = match.group(2); DeleteFileController controller = new DeleteFileController(proxy, context); - return () -> controller.delete(bucket, filePath); + return () -> controller.handle(bucket, filePath); } return null; @@ -229,7 +229,7 @@ private static Controller selectPut(Proxy proxy, ProxyContext context, String pa String bucket = match.group(1); String filePath = match.group(2); UploadFileController controller = new UploadFileController(proxy, context); - return () -> controller.upload(bucket, filePath); + return () -> controller.handle(bucket, filePath); } return null; diff --git a/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java b/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java index aa65d1039..bb84a15e5 100644 --- a/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java @@ -3,34 +3,22 @@ import com.epam.aidial.core.Proxy; import com.epam.aidial.core.ProxyContext; import com.epam.aidial.core.storage.BlobStorage; -import com.epam.aidial.core.storage.BlobStorageUtil; import com.epam.aidial.core.storage.ResourceDescription; import com.epam.aidial.core.storage.ResourceType; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j -@AllArgsConstructor -public class DeleteFileController { - private final Proxy proxy; - private final ProxyContext context; +public class DeleteFileController extends AccessControlBaseController { - /** - * Deletes file from storage. - * - * @param filePath relative path, for example: inputs/data.csv - */ - public Future delete(String bucket, String filePath) { - String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); - String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); - - if (!expectedUserBucket.equals(decryptedBucket)) { - return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); - } + public DeleteFileController(Proxy proxy, ProxyContext context) { + super(proxy, context); + } - ResourceDescription resource = ResourceDescription.from(ResourceType.FILES, bucket, decryptedBucket, filePath); + @Override + protected Future handle(String bucketName, String bucketLocation, String filePath) { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, bucketName, bucketLocation, filePath); String absoluteFilePath = resource.getAbsoluteFilePath(); BlobStorage storage = proxy.getStorage(); diff --git a/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java b/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java index cc1b7a132..cb144c14b 100644 --- a/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java @@ -2,7 +2,6 @@ import com.epam.aidial.core.Proxy; import com.epam.aidial.core.ProxyContext; -import com.epam.aidial.core.storage.BlobStorageUtil; import com.epam.aidial.core.storage.InputStreamReader; import com.epam.aidial.core.storage.ResourceDescription; import com.epam.aidial.core.storage.ResourceType; @@ -11,7 +10,6 @@ import io.vertx.core.Promise; import io.vertx.core.http.HttpHeaders; import io.vertx.core.http.HttpServerResponse; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jclouds.blobstore.domain.Blob; import org.jclouds.io.MutableContentMetadata; @@ -20,28 +18,15 @@ import java.io.IOException; @Slf4j -@AllArgsConstructor -public class DownloadFileController { +public class DownloadFileController extends AccessControlBaseController { - private final Proxy proxy; - private final ProxyContext context; - - /** - * Downloads file content from provided path. - * Path can be either absolute or relative. - * Path type determined by "path" query parameter which can be "absolute" or "relative"(default value) - * - * @param filePath file path; absolute or relative - */ - public Future download(String bucket, String filePath) { - String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); - String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); - - if (!expectedUserBucket.equals(decryptedBucket)) { - return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); - } + public DownloadFileController(Proxy proxy, ProxyContext context) { + super(proxy, context); + } - ResourceDescription resource = ResourceDescription.from(ResourceType.FILES, bucket, decryptedBucket, filePath); + @Override + protected Future handle(String bucketName, String bucketLocation, String filePath) { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, bucketName, bucketLocation, filePath); Future blobFuture = proxy.getVertx().executeBlocking(() -> proxy.getStorage().load(resource.getAbsoluteFilePath())); @@ -75,7 +60,7 @@ public Future download(String bucket, String filePath) { result.fail(e); } }).onFailure(error -> context.respond(HttpStatus.INTERNAL_SERVER_ERROR, - "Failed to fetch file with path %s/%s".formatted(bucket, filePath))); + "Failed to fetch file with path %s/%s".formatted(bucketName, filePath))); return result.future(); } diff --git a/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java b/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java index 98bfbf597..cebebe720 100644 --- a/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java +++ b/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java @@ -9,33 +9,22 @@ import com.epam.aidial.core.storage.ResourceType; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -@AllArgsConstructor @Slf4j -public class FileMetadataController { - private final Proxy proxy; - private final ProxyContext context; +public class FileMetadataController extends AccessControlBaseController { - /** - * Lists all files and folders that belong to the provided path. - * - * @param path relative path, for example: inputs/ - */ - public Future list(String bucket, String path) { - String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); - String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); - - if (!expectedUserBucket.equals(decryptedBucket)) { - return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); - } + public FileMetadataController(Proxy proxy, ProxyContext context) { + super(proxy, context); + } + @Override + protected Future handle(String bucketName, String bucketLocation, String filePath) { BlobStorage storage = proxy.getStorage(); return proxy.getVertx().executeBlocking(() -> { try { - String filePath = path.isEmpty() ? BlobStorageUtil.PATH_SEPARATOR : path; - ResourceDescription resource = ResourceDescription.from(ResourceType.FILES, bucket, decryptedBucket, filePath); + String path = filePath.isEmpty() ? BlobStorageUtil.PATH_SEPARATOR : filePath; + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, bucketName, bucketLocation, path); FileMetadataBase metadata = storage.listMetadata(resource); if (metadata != null) { context.respond(HttpStatus.OK, metadata); @@ -44,7 +33,7 @@ public Future list(String bucket, String path) { } } catch (Exception ex) { log.error("Failed to list files", ex); - context.respond(HttpStatus.INTERNAL_SERVER_ERROR, "Failed to list files by path %s".formatted(path)); + context.respond(HttpStatus.INTERNAL_SERVER_ERROR, "Failed to list files by path %s/%s".formatted(bucketName, filePath)); } return null; diff --git a/src/main/java/com/epam/aidial/core/controller/UploadFileController.java b/src/main/java/com/epam/aidial/core/controller/UploadFileController.java index 2fa599a23..a1bc08851 100644 --- a/src/main/java/com/epam/aidial/core/controller/UploadFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/UploadFileController.java @@ -12,36 +12,22 @@ import io.vertx.core.buffer.Buffer; import io.vertx.core.streams.Pipe; import io.vertx.core.streams.impl.PipeImpl; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j -@RequiredArgsConstructor -public class UploadFileController { +public class UploadFileController extends AccessControlBaseController { - private final Proxy proxy; - private final ProxyContext context; - - /** - * Uploads file to the storage. - * Current API implementation requires bucket and relative file path - * - * @param bucket bucket to write - * @param filePath relative path according to the bucket, for example: folder1/file.txt - */ - public Future upload(String bucket, String filePath) { - String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); - String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); + public UploadFileController(Proxy proxy, ProxyContext context) { + super(proxy, context); + } + @Override + protected Future handle(String bucketName, String bucketLocation, String filePath) { if (filePath.isEmpty() || BlobStorageUtil.isFolder(filePath)) { return context.respond(HttpStatus.BAD_REQUEST, "File name is missing"); } - if (!expectedUserBucket.equals(decryptedBucket)) { - return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); - } - - ResourceDescription resource = ResourceDescription.from(ResourceType.FILES, bucket, decryptedBucket, filePath); + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, bucketName, bucketLocation, filePath); Promise result = Promise.promise(); context.getRequest() .setExpectMultipart(true) @@ -60,7 +46,7 @@ public Future upload(String bucket, String filePath) { .onFailure(error -> { writeStream.abortUpload(error); context.respond(HttpStatus.INTERNAL_SERVER_ERROR, - "Failed to upload file by path %s/%s".formatted(bucket, filePath)); + "Failed to upload file by path %s/%s".formatted(bucketName, filePath)); }); }); diff --git a/src/main/java/com/epam/aidial/core/security/EncryptionService.java b/src/main/java/com/epam/aidial/core/security/EncryptionService.java index fc97b325a..5cfd4e970 100644 --- a/src/main/java/com/epam/aidial/core/security/EncryptionService.java +++ b/src/main/java/com/epam/aidial/core/security/EncryptionService.java @@ -6,6 +6,7 @@ import java.security.spec.KeySpec; import java.util.Base64; import java.util.Objects; +import javax.annotation.Nullable; import javax.crypto.Cipher; import javax.crypto.SecretKey; import javax.crypto.SecretKeyFactory; @@ -27,9 +28,11 @@ public EncryptionService(Encryption config) { } EncryptionService(String password, String salt) { + Objects.requireNonNull(password, "Encryption password is not set"); + Objects.requireNonNull(salt, "Encryption salt is not set"); try { SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256"); - KeySpec spec = new PBEKeySpec(Objects.requireNonNull(password).toCharArray(), Objects.requireNonNull(salt).getBytes(), 3000, 256); + KeySpec spec = new PBEKeySpec(password.toCharArray(), salt.getBytes(), 3000, 256); key = new SecretKeySpec(secretKeyFactory.generateSecret(spec).getEncoded(), "AES"); } catch (Exception e) { throw new RuntimeException(e); @@ -47,6 +50,7 @@ public String encrypt(String value) { } } + @Nullable public String decrypt(String value) { try { Cipher cipher = Cipher.getInstance(TRANSFORMATION); @@ -55,7 +59,6 @@ public String decrypt(String value) { .decode(value))); } catch (Exception e) { log.error("Failed to decrypt value " + value, e); - return null; } } diff --git a/src/main/java/com/epam/aidial/core/storage/BlobStorage.java b/src/main/java/com/epam/aidial/core/storage/BlobStorage.java index 4aa12ea0e..8dcf0d13c 100644 --- a/src/main/java/com/epam/aidial/core/storage/BlobStorage.java +++ b/src/main/java/com/epam/aidial/core/storage/BlobStorage.java @@ -179,7 +179,7 @@ private static FileMetadataBase buildFileMetadata(ResourceDescription resource, } private static ResourceDescription getResourceDescription(ResourceType resourceType, String bucketName, String bucketLocation, String absoluteFilePath) { - String relativeFilePath = absoluteFilePath.substring(bucketLocation.length() + resourceType.getName().length()); + String relativeFilePath = absoluteFilePath.substring(bucketLocation.length() + resourceType.getFolder().length()); return ResourceDescription.from(resourceType, bucketName, bucketLocation, relativeFilePath); } diff --git a/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java b/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java index 7505d5dd6..b32fce61b 100644 --- a/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java +++ b/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java @@ -34,7 +34,7 @@ public String buildUserBucket(ProxyContext context) { } public String buildAbsoluteFilePath(ResourceType resource, String bucket, String path) { - return bucket + resource.getName() + PATH_SEPARATOR + path; + return bucket + resource.getFolder() + PATH_SEPARATOR + path; } public boolean isFolder(String path) { diff --git a/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java b/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java index dc8c94082..bc9d31b48 100644 --- a/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java +++ b/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java @@ -1,8 +1,9 @@ package com.epam.aidial.core.storage; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Data; -import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -10,8 +11,7 @@ import java.util.stream.Collectors; @Data -@AllArgsConstructor -@NoArgsConstructor +@AllArgsConstructor(access = AccessLevel.PRIVATE) public class ResourceDescription { ResourceType type; String name; @@ -31,7 +31,7 @@ public String getUrl() { builder.append(parentPath) .append(BlobStorageUtil.PATH_SEPARATOR); } - if (name != null && !name.equals(BlobStorageUtil.PATH_SEPARATOR)) { + if (name != null && !isHomeFolder(name)) { builder.append(encodeToUrl(name)); if (isFolder) { @@ -45,10 +45,11 @@ public String getUrl() { public String getAbsoluteFilePath() { StringBuilder builder = new StringBuilder(); if (parentFolders != null) { - builder.append(getParentPath()); + builder.append(getParentPath()) + .append(BlobStorageUtil.PATH_SEPARATOR); } - if (name != null && !name.equals(BlobStorageUtil.PATH_SEPARATOR)) { - builder.append(BlobStorageUtil.PATH_SEPARATOR).append(name); + if (name != null && !isHomeFolder(name)) { + builder.append(name); if (isFolder) { builder.append(BlobStorageUtil.PATH_SEPARATOR); @@ -63,6 +64,9 @@ public String getParentPath() { } public static ResourceDescription from(ResourceType type, String bucketName, String bucketLocation, String relativeFilePath) { + verify(bucketLocation.endsWith(BlobStorageUtil.PATH_SEPARATOR), "Bucket location must end with /"); + verify(!StringUtils.isBlank(relativeFilePath), "Invalid relative path: " + relativeFilePath); + String[] elements = relativeFilePath.split(BlobStorageUtil.PATH_SEPARATOR); List parentFolders = null; String name = "/"; @@ -82,4 +86,14 @@ public static ResourceDescription from(ResourceType type, String bucketName, Str private static String encodeToUrl(String value) { return URLEncoder.encode(value, StandardCharsets.UTF_8); } + + private static boolean isHomeFolder(String path) { + return path.equals(BlobStorageUtil.PATH_SEPARATOR); + } + + private static void verify(boolean condition, String message) { + if (!condition) { + throw new IllegalArgumentException(message); + } + } } diff --git a/src/main/java/com/epam/aidial/core/storage/ResourceType.java b/src/main/java/com/epam/aidial/core/storage/ResourceType.java index f9a45ae7f..9b8285b82 100644 --- a/src/main/java/com/epam/aidial/core/storage/ResourceType.java +++ b/src/main/java/com/epam/aidial/core/storage/ResourceType.java @@ -1,15 +1,15 @@ package com.epam.aidial.core.storage; public enum ResourceType { - FILES("files"); + FILE("files"); - private final String name; + private final String folder; - ResourceType(String name) { - this.name = name; + ResourceType(String folder) { + this.folder = folder; } - public String getName() { - return name; + public String getFolder() { + return folder; } } diff --git a/src/test/java/com/epam/aidial/core/FileApiTest.java b/src/test/java/com/epam/aidial/core/FileApiTest.java index 9b10d74a9..edf8131fd 100644 --- a/src/test/java/com/epam/aidial/core/FileApiTest.java +++ b/src/test/java/com/epam/aidial/core/FileApiTest.java @@ -136,14 +136,14 @@ public void testInvalidFileUploadUrl(Vertx vertx, VertxTestContext context) { @Test public void testInvalidFileUploadUrl2(Vertx vertx, VertxTestContext context) { WebClient client = WebClient.create(vertx); - client.put(serverPort, "localhost", "/v1/files/bucket/") + client.put(serverPort, "localhost", "/v1/files/test-bucket/") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), context.succeeding(response -> { context.verify(() -> { - assertEquals(400, response.statusCode()); - assertEquals("File name is missing", response.body()); + assertEquals(403, response.statusCode()); + assertEquals("You don't have an access to the bucket test-bucket", response.body()); context.completeNow(); }); }) diff --git a/src/test/java/com/epam/aidial/core/storage/ResourceDescriptionTest.java b/src/test/java/com/epam/aidial/core/storage/ResourceDescriptionTest.java new file mode 100644 index 000000000..fc572d6f2 --- /dev/null +++ b/src/test/java/com/epam/aidial/core/storage/ResourceDescriptionTest.java @@ -0,0 +1,129 @@ +package com.epam.aidial.core.storage; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ResourceDescriptionTest { + + @Test + public void testHomeFolderDescription() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "aes/bucket/name", "buckets/location/", "/"); + assertEquals("/", resource.getName()); + assertEquals("aes/bucket/name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("aes%2Fbucket%2Fname/", resource.getUrl()); + assertEquals("buckets/location/files/", resource.getAbsoluteFilePath()); + assertTrue(resource.isFolder()); + assertNull(resource.getParentPath()); + assertNull(resource.getParentFolders()); + } + + @Test + public void testUserFolderDescription() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "folder1/"); + assertEquals("folder1", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name/folder1/", resource.getUrl()); + assertEquals("buckets/location/files/folder1/", resource.getAbsoluteFilePath()); + assertTrue(resource.isFolder()); + assertNull(resource.getParentPath()); + assertNull(resource.getParentFolders()); + } + + @Test + public void testUserFolderDescription2() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "folder1/folder2/"); + assertEquals("folder2", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name/folder1/folder2/", resource.getUrl()); + assertEquals("buckets/location/files/folder1/folder2/", resource.getAbsoluteFilePath()); + assertTrue(resource.isFolder()); + assertEquals("folder1", resource.getParentPath()); + assertIterableEquals(List.of("folder1"), resource.getParentFolders()); + } + + @Test + public void testUserFolderDescription3() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "folder1/folder2/folder3/"); + assertEquals("folder3", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name/folder1/folder2/folder3/", resource.getUrl()); + assertEquals("buckets/location/files/folder1/folder2/folder3/", resource.getAbsoluteFilePath()); + assertTrue(resource.isFolder()); + assertEquals("folder1/folder2", resource.getParentPath()); + assertIterableEquals(List.of("folder1", "folder2"), resource.getParentFolders()); + } + + @Test + public void testFileDescription1() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "file.txt"); + assertEquals("file.txt", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name/file.txt", resource.getUrl()); + assertEquals("buckets/location/files/file.txt", resource.getAbsoluteFilePath()); + assertFalse(resource.isFolder()); + assertNull(resource.getParentPath()); + assertNull(resource.getParentFolders()); + } + + @Test + public void testFileDescription2() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "folder1/file.txt"); + assertEquals("file.txt", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name/folder1/file.txt", resource.getUrl()); + assertEquals("buckets/location/files/folder1/file.txt", resource.getAbsoluteFilePath()); + assertFalse(resource.isFolder()); + assertEquals("folder1", resource.getParentPath()); + assertIterableEquals(List.of("folder1"), resource.getParentFolders()); + } + + @Test + public void testFileDescription3() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "folder1/folder2/file.txt"); + assertEquals("file.txt", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name/folder1/folder2/file.txt", resource.getUrl()); + assertEquals("buckets/location/files/folder1/folder2/file.txt", resource.getAbsoluteFilePath()); + assertFalse(resource.isFolder()); + assertEquals("folder1/folder2", resource.getParentPath()); + assertIterableEquals(List.of("folder1", "folder2"), resource.getParentFolders()); + } + + @Test + public void testInvalidBucketLocation() { + assertThrows(IllegalArgumentException.class, + () -> ResourceDescription.from(ResourceType.FILE, "bucket-name", "buckets/location", "file.txt")); + } + + @Test + public void testInvalidRelativePath() { + assertThrows(IllegalArgumentException.class, + () -> ResourceDescription.from(ResourceType.FILE, "bucket-name", "buckets/location/", "")); + assertThrows(IllegalArgumentException.class, + () -> ResourceDescription.from(ResourceType.FILE, "bucket-name", "buckets/location/", null)); + assertThrows(IllegalArgumentException.class, + () -> ResourceDescription.from(ResourceType.FILE, "bucket-name", "buckets/location/", " ")); + } +} From ed853b59b2b5d1fad0de7693d9c3d6d518808bcb Mon Sep 17 00:00:00 2001 From: Maksim_Hadalau Date: Sun, 17 Dec 2023 19:15:47 +0100 Subject: [PATCH 3/5] fix controller selector --- .../AccessControlBaseController.java | 11 +++- .../core/controller/ControllerSelector.java | 57 ++++++++++--------- .../epam/aidial/core/storage/BlobStorage.java | 2 +- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java b/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java index c5fb3e3ae..a01ab7d30 100644 --- a/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java +++ b/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java @@ -7,6 +7,9 @@ import io.vertx.core.Future; import lombok.AllArgsConstructor; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; + @AllArgsConstructor public abstract class AccessControlBaseController { @@ -15,14 +18,16 @@ public abstract class AccessControlBaseController { public Future handle(String bucket, String filePath) { + String decodedBucket = URLDecoder.decode(bucket, StandardCharsets.UTF_8); + String decodedFilePath = URLDecoder.decode(filePath, StandardCharsets.UTF_8); String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); - String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); + String decryptedBucket = proxy.getEncryptionService().decrypt(decodedBucket); if (!expectedUserBucket.equals(decryptedBucket)) { - return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); + return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + decodedBucket); } - return handle(bucket, decryptedBucket, filePath); + return handle(decodedBucket, decryptedBucket, decodedFilePath); } protected abstract Future handle(String bucketName, String bucketLocation, String filePath); diff --git a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java index b1c400697..59ff1ad20 100644 --- a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java +++ b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java @@ -44,92 +44,93 @@ public class ControllerSelector { private static final Pattern PATTERN_TRUNCATE_PROMPT = Pattern.compile("/+v1/deployments/([-.@a-zA-Z0-9]+)/truncate_prompt"); public Controller select(Proxy proxy, ProxyContext context) { - String path = URLDecoder.decode(context.getRequest().path(), StandardCharsets.UTF_8); + String providedPath = context.getRequest().path(); + String decodedPath = URLDecoder.decode(context.getRequest().path(), StandardCharsets.UTF_8); HttpMethod method = context.getRequest().method(); Controller controller = null; if (method == HttpMethod.GET) { - controller = selectGet(proxy, context, path); + controller = selectGet(proxy, context, providedPath, decodedPath); } else if (method == HttpMethod.POST) { - controller = selectPost(proxy, context, path); + controller = selectPost(proxy, context, providedPath, decodedPath); } else if (method == HttpMethod.DELETE) { - controller = selectDelete(proxy, context, path); + controller = selectDelete(proxy, context, providedPath, decodedPath); } else if (method == HttpMethod.PUT) { - controller = selectPut(proxy, context, path); + controller = selectPut(proxy, context, providedPath, decodedPath); } return (controller == null) ? new RouteController(proxy, context) : controller; } - private static Controller selectGet(Proxy proxy, ProxyContext context, String path) { + private static Controller selectGet(Proxy proxy, ProxyContext context, String providedPath, String decodedPath) { Matcher match; - match = match(PATTERN_DEPLOYMENT, path); + match = match(PATTERN_DEPLOYMENT, decodedPath); if (match != null) { DeploymentController controller = new DeploymentController(context); String deploymentId = match.group(1); return () -> controller.getDeployment(deploymentId); } - match = match(PATTERN_DEPLOYMENTS, path); + match = match(PATTERN_DEPLOYMENTS, decodedPath); if (match != null) { DeploymentController controller = new DeploymentController(context); return controller::getDeployments; } - match = match(PATTERN_MODEL, path); + match = match(PATTERN_MODEL, decodedPath); if (match != null) { ModelController controller = new ModelController(context); String modelId = match.group(1); return () -> controller.getModel(modelId); } - match = match(PATTERN_MODELS, path); + match = match(PATTERN_MODELS, decodedPath); if (match != null) { ModelController controller = new ModelController(context); return controller::getModels; } - match = match(PATTERN_ADDON, path); + match = match(PATTERN_ADDON, decodedPath); if (match != null) { AddonController controller = new AddonController(context); String addonId = match.group(1); return () -> controller.getAddon(addonId); } - match = match(PATTERN_ADDONS, path); + match = match(PATTERN_ADDONS, decodedPath); if (match != null) { AddonController controller = new AddonController(context); return controller::getAddons; } - match = match(PATTERN_ASSISTANT, path); + match = match(PATTERN_ASSISTANT, decodedPath); if (match != null) { AssistantController controller = new AssistantController(context); String assistantId = match.group(1); return () -> controller.getAssistant(assistantId); } - match = match(PATTERN_ASSISTANTS, path); + match = match(PATTERN_ASSISTANTS, decodedPath); if (match != null) { AssistantController controller = new AssistantController(context); return controller::getAssistants; } - match = match(PATTERN_APPLICATION, path); + match = match(PATTERN_APPLICATION, decodedPath); if (match != null) { ApplicationController controller = new ApplicationController(context); String application = match.group(1); return () -> controller.getApplication(application); } - match = match(PATTERN_APPLICATIONS, path); + match = match(PATTERN_APPLICATIONS, decodedPath); if (match != null) { ApplicationController controller = new ApplicationController(context); return controller::getApplications; } - match = match(PATTERN_FILES_METADATA, path); + match = match(PATTERN_FILES_METADATA, providedPath); if (match != null) { String bucket = match.group(1); String filePath = match.group(2); @@ -137,7 +138,7 @@ private static Controller selectGet(Proxy proxy, ProxyContext context, String pa return () -> controller.handle(bucket, filePath); } - match = match(PATTERN_FILES, path); + match = match(PATTERN_FILES, providedPath); if (match != null) { String bucket = match.group(1); String filePath = match.group(2); @@ -145,7 +146,7 @@ private static Controller selectGet(Proxy proxy, ProxyContext context, String pa return () -> controller.handle(bucket, filePath); } - match = match(PATTERN_BUCKET, path); + match = match(PATTERN_BUCKET, decodedPath); if (match != null) { BucketController controller = new BucketController(proxy, context); return controller::getBucket; @@ -154,8 +155,8 @@ private static Controller selectGet(Proxy proxy, ProxyContext context, String pa return null; } - private static Controller selectPost(Proxy proxy, ProxyContext context, String path) { - Matcher match = match(PATTERN_POST_DEPLOYMENT, path); + private static Controller selectPost(Proxy proxy, ProxyContext context, String providedPath, String decodedPath) { + Matcher match = match(PATTERN_POST_DEPLOYMENT, decodedPath); if (match != null) { String deploymentId = match.group(1); String deploymentApi = match.group(2); @@ -163,7 +164,7 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return () -> controller.handle(deploymentId, deploymentApi); } - match = match(PATTERN_RATE_RESPONSE, path); + match = match(PATTERN_RATE_RESPONSE, decodedPath); if (match != null) { String deploymentId = match.group(1); @@ -178,7 +179,7 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return () -> controller.handle(deploymentId, getter, false); } - match = match(PATTERN_TOKENIZE, path); + match = match(PATTERN_TOKENIZE, decodedPath); if (match != null) { String deploymentId = match.group(1); @@ -193,7 +194,7 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return () -> controller.handle(deploymentId, getter, true); } - match = match(PATTERN_TRUNCATE_PROMPT, path); + match = match(PATTERN_TRUNCATE_PROMPT, decodedPath); if (match != null) { String deploymentId = match.group(1); @@ -211,8 +212,8 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return null; } - private static Controller selectDelete(Proxy proxy, ProxyContext context, String path) { - Matcher match = match(PATTERN_FILES, path); + private static Controller selectDelete(Proxy proxy, ProxyContext context, String providedPath, String decodedPath) { + Matcher match = match(PATTERN_FILES, providedPath); if (match != null) { String bucket = match.group(1); String filePath = match.group(2); @@ -223,8 +224,8 @@ private static Controller selectDelete(Proxy proxy, ProxyContext context, String return null; } - private static Controller selectPut(Proxy proxy, ProxyContext context, String path) { - Matcher match = match(PATTERN_FILES, path); + private static Controller selectPut(Proxy proxy, ProxyContext context, String providedPath, String decodedPath) { + Matcher match = match(PATTERN_FILES, providedPath); if (match != null) { String bucket = match.group(1); String filePath = match.group(2); diff --git a/src/main/java/com/epam/aidial/core/storage/BlobStorage.java b/src/main/java/com/epam/aidial/core/storage/BlobStorage.java index 8dcf0d13c..9e9015fd0 100644 --- a/src/main/java/com/epam/aidial/core/storage/BlobStorage.java +++ b/src/main/java/com/epam/aidial/core/storage/BlobStorage.java @@ -179,7 +179,7 @@ private static FileMetadataBase buildFileMetadata(ResourceDescription resource, } private static ResourceDescription getResourceDescription(ResourceType resourceType, String bucketName, String bucketLocation, String absoluteFilePath) { - String relativeFilePath = absoluteFilePath.substring(bucketLocation.length() + resourceType.getFolder().length()); + String relativeFilePath = absoluteFilePath.substring(bucketLocation.length() + resourceType.getFolder().length() + 1); return ResourceDescription.from(resourceType, bucketName, bucketLocation, relativeFilePath); } From 5cca412fef2eccc413cd5b3b1278f4d61966df70 Mon Sep 17 00:00:00 2001 From: Maksim_Hadalau Date: Mon, 18 Dec 2023 11:35:32 +0100 Subject: [PATCH 4/5] address review comments --- .../AccessControlBaseController.java | 22 ++- .../core/controller/ControllerSelector.java | 57 +++--- .../core/controller/DeleteFileController.java | 8 +- .../controller/DownloadFileController.java | 9 +- .../controller/FileMetadataController.java | 9 +- .../core/controller/UploadFileController.java | 9 +- .../core/security/EncryptionService.java | 8 +- .../core/storage/ResourceDescription.java | 22 +-- .../com/epam/aidial/core/util/Base58.java | 163 ++++++++++++++++++ .../com/epam/aidial/core/FileApiTest.java | 88 +++++----- .../core/storage/ResourceDescriptionTest.java | 41 +++-- 11 files changed, 302 insertions(+), 134 deletions(-) create mode 100644 src/main/java/com/epam/aidial/core/util/Base58.java diff --git a/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java b/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java index a01ab7d30..fd5d9195e 100644 --- a/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java +++ b/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java @@ -3,13 +3,12 @@ import com.epam.aidial.core.Proxy; import com.epam.aidial.core.ProxyContext; import com.epam.aidial.core.storage.BlobStorageUtil; +import com.epam.aidial.core.storage.ResourceDescription; +import com.epam.aidial.core.storage.ResourceType; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; import lombok.AllArgsConstructor; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; - @AllArgsConstructor public abstract class AccessControlBaseController { @@ -18,18 +17,23 @@ public abstract class AccessControlBaseController { public Future handle(String bucket, String filePath) { - String decodedBucket = URLDecoder.decode(bucket, StandardCharsets.UTF_8); - String decodedFilePath = URLDecoder.decode(filePath, StandardCharsets.UTF_8); String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); - String decryptedBucket = proxy.getEncryptionService().decrypt(decodedBucket); + String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); if (!expectedUserBucket.equals(decryptedBucket)) { - return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + decodedBucket); + return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); + } + + ResourceDescription resource; + try { + resource = ResourceDescription.from(ResourceType.FILE, bucket, decryptedBucket, filePath); + } catch (Exception ex) { + return context.respond(HttpStatus.BAD_REQUEST, "Invalid file url provided"); } - return handle(decodedBucket, decryptedBucket, decodedFilePath); + return handle(resource); } - protected abstract Future handle(String bucketName, String bucketLocation, String filePath); + protected abstract Future handle(ResourceDescription resource); } diff --git a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java index 59ff1ad20..b1c400697 100644 --- a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java +++ b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java @@ -44,93 +44,92 @@ public class ControllerSelector { private static final Pattern PATTERN_TRUNCATE_PROMPT = Pattern.compile("/+v1/deployments/([-.@a-zA-Z0-9]+)/truncate_prompt"); public Controller select(Proxy proxy, ProxyContext context) { - String providedPath = context.getRequest().path(); - String decodedPath = URLDecoder.decode(context.getRequest().path(), StandardCharsets.UTF_8); + String path = URLDecoder.decode(context.getRequest().path(), StandardCharsets.UTF_8); HttpMethod method = context.getRequest().method(); Controller controller = null; if (method == HttpMethod.GET) { - controller = selectGet(proxy, context, providedPath, decodedPath); + controller = selectGet(proxy, context, path); } else if (method == HttpMethod.POST) { - controller = selectPost(proxy, context, providedPath, decodedPath); + controller = selectPost(proxy, context, path); } else if (method == HttpMethod.DELETE) { - controller = selectDelete(proxy, context, providedPath, decodedPath); + controller = selectDelete(proxy, context, path); } else if (method == HttpMethod.PUT) { - controller = selectPut(proxy, context, providedPath, decodedPath); + controller = selectPut(proxy, context, path); } return (controller == null) ? new RouteController(proxy, context) : controller; } - private static Controller selectGet(Proxy proxy, ProxyContext context, String providedPath, String decodedPath) { + private static Controller selectGet(Proxy proxy, ProxyContext context, String path) { Matcher match; - match = match(PATTERN_DEPLOYMENT, decodedPath); + match = match(PATTERN_DEPLOYMENT, path); if (match != null) { DeploymentController controller = new DeploymentController(context); String deploymentId = match.group(1); return () -> controller.getDeployment(deploymentId); } - match = match(PATTERN_DEPLOYMENTS, decodedPath); + match = match(PATTERN_DEPLOYMENTS, path); if (match != null) { DeploymentController controller = new DeploymentController(context); return controller::getDeployments; } - match = match(PATTERN_MODEL, decodedPath); + match = match(PATTERN_MODEL, path); if (match != null) { ModelController controller = new ModelController(context); String modelId = match.group(1); return () -> controller.getModel(modelId); } - match = match(PATTERN_MODELS, decodedPath); + match = match(PATTERN_MODELS, path); if (match != null) { ModelController controller = new ModelController(context); return controller::getModels; } - match = match(PATTERN_ADDON, decodedPath); + match = match(PATTERN_ADDON, path); if (match != null) { AddonController controller = new AddonController(context); String addonId = match.group(1); return () -> controller.getAddon(addonId); } - match = match(PATTERN_ADDONS, decodedPath); + match = match(PATTERN_ADDONS, path); if (match != null) { AddonController controller = new AddonController(context); return controller::getAddons; } - match = match(PATTERN_ASSISTANT, decodedPath); + match = match(PATTERN_ASSISTANT, path); if (match != null) { AssistantController controller = new AssistantController(context); String assistantId = match.group(1); return () -> controller.getAssistant(assistantId); } - match = match(PATTERN_ASSISTANTS, decodedPath); + match = match(PATTERN_ASSISTANTS, path); if (match != null) { AssistantController controller = new AssistantController(context); return controller::getAssistants; } - match = match(PATTERN_APPLICATION, decodedPath); + match = match(PATTERN_APPLICATION, path); if (match != null) { ApplicationController controller = new ApplicationController(context); String application = match.group(1); return () -> controller.getApplication(application); } - match = match(PATTERN_APPLICATIONS, decodedPath); + match = match(PATTERN_APPLICATIONS, path); if (match != null) { ApplicationController controller = new ApplicationController(context); return controller::getApplications; } - match = match(PATTERN_FILES_METADATA, providedPath); + match = match(PATTERN_FILES_METADATA, path); if (match != null) { String bucket = match.group(1); String filePath = match.group(2); @@ -138,7 +137,7 @@ private static Controller selectGet(Proxy proxy, ProxyContext context, String pr return () -> controller.handle(bucket, filePath); } - match = match(PATTERN_FILES, providedPath); + match = match(PATTERN_FILES, path); if (match != null) { String bucket = match.group(1); String filePath = match.group(2); @@ -146,7 +145,7 @@ private static Controller selectGet(Proxy proxy, ProxyContext context, String pr return () -> controller.handle(bucket, filePath); } - match = match(PATTERN_BUCKET, decodedPath); + match = match(PATTERN_BUCKET, path); if (match != null) { BucketController controller = new BucketController(proxy, context); return controller::getBucket; @@ -155,8 +154,8 @@ private static Controller selectGet(Proxy proxy, ProxyContext context, String pr return null; } - private static Controller selectPost(Proxy proxy, ProxyContext context, String providedPath, String decodedPath) { - Matcher match = match(PATTERN_POST_DEPLOYMENT, decodedPath); + private static Controller selectPost(Proxy proxy, ProxyContext context, String path) { + Matcher match = match(PATTERN_POST_DEPLOYMENT, path); if (match != null) { String deploymentId = match.group(1); String deploymentApi = match.group(2); @@ -164,7 +163,7 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return () -> controller.handle(deploymentId, deploymentApi); } - match = match(PATTERN_RATE_RESPONSE, decodedPath); + match = match(PATTERN_RATE_RESPONSE, path); if (match != null) { String deploymentId = match.group(1); @@ -179,7 +178,7 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return () -> controller.handle(deploymentId, getter, false); } - match = match(PATTERN_TOKENIZE, decodedPath); + match = match(PATTERN_TOKENIZE, path); if (match != null) { String deploymentId = match.group(1); @@ -194,7 +193,7 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return () -> controller.handle(deploymentId, getter, true); } - match = match(PATTERN_TRUNCATE_PROMPT, decodedPath); + match = match(PATTERN_TRUNCATE_PROMPT, path); if (match != null) { String deploymentId = match.group(1); @@ -212,8 +211,8 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return null; } - private static Controller selectDelete(Proxy proxy, ProxyContext context, String providedPath, String decodedPath) { - Matcher match = match(PATTERN_FILES, providedPath); + private static Controller selectDelete(Proxy proxy, ProxyContext context, String path) { + Matcher match = match(PATTERN_FILES, path); if (match != null) { String bucket = match.group(1); String filePath = match.group(2); @@ -224,8 +223,8 @@ private static Controller selectDelete(Proxy proxy, ProxyContext context, String return null; } - private static Controller selectPut(Proxy proxy, ProxyContext context, String providedPath, String decodedPath) { - Matcher match = match(PATTERN_FILES, providedPath); + private static Controller selectPut(Proxy proxy, ProxyContext context, String path) { + Matcher match = match(PATTERN_FILES, path); if (match != null) { String bucket = match.group(1); String filePath = match.group(2); diff --git a/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java b/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java index bb84a15e5..1fa329c5c 100644 --- a/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java @@ -4,7 +4,6 @@ import com.epam.aidial.core.ProxyContext; import com.epam.aidial.core.storage.BlobStorage; import com.epam.aidial.core.storage.ResourceDescription; -import com.epam.aidial.core.storage.ResourceType; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; import lombok.extern.slf4j.Slf4j; @@ -17,8 +16,11 @@ public DeleteFileController(Proxy proxy, ProxyContext context) { } @Override - protected Future handle(String bucketName, String bucketLocation, String filePath) { - ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, bucketName, bucketLocation, filePath); + protected Future handle(ResourceDescription resource) { + if (resource.isFolder()) { + return context.respond(HttpStatus.BAD_REQUEST, "Can't delete a folder"); + } + String absoluteFilePath = resource.getAbsoluteFilePath(); BlobStorage storage = proxy.getStorage(); diff --git a/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java b/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java index cb144c14b..d6752951e 100644 --- a/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java @@ -4,7 +4,6 @@ import com.epam.aidial.core.ProxyContext; import com.epam.aidial.core.storage.InputStreamReader; import com.epam.aidial.core.storage.ResourceDescription; -import com.epam.aidial.core.storage.ResourceType; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; import io.vertx.core.Promise; @@ -25,8 +24,10 @@ public DownloadFileController(Proxy proxy, ProxyContext context) { } @Override - protected Future handle(String bucketName, String bucketLocation, String filePath) { - ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, bucketName, bucketLocation, filePath); + protected Future handle(ResourceDescription resource) { + if (resource.isFolder()) { + return context.respond(HttpStatus.BAD_REQUEST, "Can't download a folder"); + } Future blobFuture = proxy.getVertx().executeBlocking(() -> proxy.getStorage().load(resource.getAbsoluteFilePath())); @@ -60,7 +61,7 @@ protected Future handle(String bucketName, String bucketLocation, String file result.fail(e); } }).onFailure(error -> context.respond(HttpStatus.INTERNAL_SERVER_ERROR, - "Failed to fetch file with path %s/%s".formatted(bucketName, filePath))); + "Failed to fetch file with path %s/%s".formatted(resource.getBucketName(), resource.getRelativePath()))); return result.future(); } diff --git a/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java b/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java index cebebe720..ed0035bda 100644 --- a/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java +++ b/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java @@ -4,9 +4,7 @@ import com.epam.aidial.core.ProxyContext; import com.epam.aidial.core.data.FileMetadataBase; import com.epam.aidial.core.storage.BlobStorage; -import com.epam.aidial.core.storage.BlobStorageUtil; import com.epam.aidial.core.storage.ResourceDescription; -import com.epam.aidial.core.storage.ResourceType; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; import lombok.extern.slf4j.Slf4j; @@ -19,12 +17,10 @@ public FileMetadataController(Proxy proxy, ProxyContext context) { } @Override - protected Future handle(String bucketName, String bucketLocation, String filePath) { + protected Future handle(ResourceDescription resource) { BlobStorage storage = proxy.getStorage(); return proxy.getVertx().executeBlocking(() -> { try { - String path = filePath.isEmpty() ? BlobStorageUtil.PATH_SEPARATOR : filePath; - ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, bucketName, bucketLocation, path); FileMetadataBase metadata = storage.listMetadata(resource); if (metadata != null) { context.respond(HttpStatus.OK, metadata); @@ -33,7 +29,8 @@ protected Future handle(String bucketName, String bucketLocation, String file } } catch (Exception ex) { log.error("Failed to list files", ex); - context.respond(HttpStatus.INTERNAL_SERVER_ERROR, "Failed to list files by path %s/%s".formatted(bucketName, filePath)); + context.respond(HttpStatus.INTERNAL_SERVER_ERROR, + "Failed to list files by path %s/%s".formatted(resource.getBucketName(), resource.getRelativePath())); } return null; diff --git a/src/main/java/com/epam/aidial/core/controller/UploadFileController.java b/src/main/java/com/epam/aidial/core/controller/UploadFileController.java index a1bc08851..3ea3e0ac1 100644 --- a/src/main/java/com/epam/aidial/core/controller/UploadFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/UploadFileController.java @@ -2,10 +2,8 @@ import com.epam.aidial.core.Proxy; import com.epam.aidial.core.ProxyContext; -import com.epam.aidial.core.storage.BlobStorageUtil; import com.epam.aidial.core.storage.BlobWriteStream; import com.epam.aidial.core.storage.ResourceDescription; -import com.epam.aidial.core.storage.ResourceType; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; import io.vertx.core.Promise; @@ -22,12 +20,11 @@ public UploadFileController(Proxy proxy, ProxyContext context) { } @Override - protected Future handle(String bucketName, String bucketLocation, String filePath) { - if (filePath.isEmpty() || BlobStorageUtil.isFolder(filePath)) { + protected Future handle(ResourceDescription resource) { + if (resource.isFolder()) { return context.respond(HttpStatus.BAD_REQUEST, "File name is missing"); } - ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, bucketName, bucketLocation, filePath); Promise result = Promise.promise(); context.getRequest() .setExpectMultipart(true) @@ -46,7 +43,7 @@ protected Future handle(String bucketName, String bucketLocation, String file .onFailure(error -> { writeStream.abortUpload(error); context.respond(HttpStatus.INTERNAL_SERVER_ERROR, - "Failed to upload file by path %s/%s".formatted(bucketName, filePath)); + "Failed to upload file by path %s/%s".formatted(resource.getBucketName(), resource.getRelativePath())); }); }); diff --git a/src/main/java/com/epam/aidial/core/security/EncryptionService.java b/src/main/java/com/epam/aidial/core/security/EncryptionService.java index 5cfd4e970..b2f9ce8fd 100644 --- a/src/main/java/com/epam/aidial/core/security/EncryptionService.java +++ b/src/main/java/com/epam/aidial/core/security/EncryptionService.java @@ -1,10 +1,10 @@ package com.epam.aidial.core.security; import com.epam.aidial.core.config.Encryption; +import com.epam.aidial.core.util.Base58; import lombok.extern.slf4j.Slf4j; import java.security.spec.KeySpec; -import java.util.Base64; import java.util.Objects; import javax.annotation.Nullable; import javax.crypto.Cipher; @@ -43,8 +43,7 @@ public String encrypt(String value) { try { Cipher cipher = Cipher.getInstance(TRANSFORMATION); cipher.init(Cipher.ENCRYPT_MODE, key, iv); - return Base64.getEncoder() - .encodeToString(cipher.doFinal(value.getBytes())); + return Base58.encode(cipher.doFinal(value.getBytes())); } catch (Exception e) { throw new RuntimeException(e); } @@ -55,8 +54,7 @@ public String decrypt(String value) { try { Cipher cipher = Cipher.getInstance(TRANSFORMATION); cipher.init(Cipher.DECRYPT_MODE, key, iv); - return new String(cipher.doFinal(Base64.getDecoder() - .decode(value))); + return new String(cipher.doFinal(Base58.decode(value))); } catch (Exception e) { log.error("Failed to decrypt value " + value, e); return null; diff --git a/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java b/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java index bc9d31b48..8376030de 100644 --- a/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java +++ b/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java @@ -8,7 +8,6 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.stream.Collectors; @Data @AllArgsConstructor(access = AccessLevel.PRIVATE) @@ -16,30 +15,28 @@ public class ResourceDescription { ResourceType type; String name; List parentFolders; + String relativePath; String bucketName; String bucketLocation; boolean isFolder; public String getUrl() { StringBuilder builder = new StringBuilder(); - builder.append(encodeToUrl(bucketName)) + builder.append(bucketName) .append(BlobStorageUtil.PATH_SEPARATOR); if (parentFolders != null) { - String parentPath = parentFolders.stream() - .map(ResourceDescription::encodeToUrl) - .collect(Collectors.joining(BlobStorageUtil.PATH_SEPARATOR)); - builder.append(parentPath) + builder.append(String.join(BlobStorageUtil.PATH_SEPARATOR, parentFolders)) .append(BlobStorageUtil.PATH_SEPARATOR); } if (name != null && !isHomeFolder(name)) { - builder.append(encodeToUrl(name)); + builder.append(name); if (isFolder) { builder.append(BlobStorageUtil.PATH_SEPARATOR); } } - return builder.toString(); + return URLEncoder.encode(builder.toString(), StandardCharsets.UTF_8); } public String getAbsoluteFilePath() { @@ -64,8 +61,9 @@ public String getParentPath() { } public static ResourceDescription from(ResourceType type, String bucketName, String bucketLocation, String relativeFilePath) { + // in case empty path - treat it as a home folder + relativeFilePath = StringUtils.isBlank(relativeFilePath) ? BlobStorageUtil.PATH_SEPARATOR : relativeFilePath; verify(bucketLocation.endsWith(BlobStorageUtil.PATH_SEPARATOR), "Bucket location must end with /"); - verify(!StringUtils.isBlank(relativeFilePath), "Invalid relative path: " + relativeFilePath); String[] elements = relativeFilePath.split(BlobStorageUtil.PATH_SEPARATOR); List parentFolders = null; @@ -80,11 +78,7 @@ public static ResourceDescription from(ResourceType type, String bucketName, Str } } - return new ResourceDescription(type, name, parentFolders, bucketName, bucketLocation, BlobStorageUtil.isFolder(relativeFilePath)); - } - - private static String encodeToUrl(String value) { - return URLEncoder.encode(value, StandardCharsets.UTF_8); + return new ResourceDescription(type, name, parentFolders, relativeFilePath, bucketName, bucketLocation, BlobStorageUtil.isFolder(relativeFilePath)); } private static boolean isHomeFolder(String path) { diff --git a/src/main/java/com/epam/aidial/core/util/Base58.java b/src/main/java/com/epam/aidial/core/util/Base58.java new file mode 100644 index 000000000..805b8b190 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/util/Base58.java @@ -0,0 +1,163 @@ +package com.epam.aidial.core.util; + +/* + * Copyright 2011 Google Inc. + * Copyright 2018 Andreas Schildbach + * + * From https://github.com/multiformats/java-multibase/blob/master/src/main/java/io/ipfs/multibase/Base58.java + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.math.BigInteger; +import java.util.Arrays; + +/** + * Base58 is a way to encode Bitcoin addresses (or arbitrary data) as alphanumeric strings. + * + *

Note that this is not the same base58 as used by Flickr, which you may find referenced around the Internet. + * + *

Satoshi explains: why base-58 instead of standard base-64 encoding? + *

    + *
  • Don't want 0OIl characters that look the same in some fonts and + * could be used to create visually identical looking account numbers.
    • + *
  • A string with non-alphanumeric characters is not as easily accepted as an account number.
    • + *
  • E-mail usually won't line-break if there's no punctuation to break at.
    • + *
  • Doubleclicking selects the whole number as one word if it's all alphanumeric.
    • + *
+ * + *

However, note that the encoding/decoding runs in O(n²) time, so it is not useful for large data. + * + *

The basic idea of the encoding is to treat the data bytes as a large number represented using + * base-256 digits, convert the number to be represented using base-58 digits, preserve the exact + * number of leading zeros (which are otherwise lost during the mathematical operations on the + * numbers), and finally represent the resulting base-58 digits as alphanumeric ASCII characters. + */ +public class Base58 { + public static final char[] ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); + private static final char ENCODED_ZERO = ALPHABET[0]; + private static final int[] INDEXES = new int[128]; + + static { + Arrays.fill(INDEXES, -1); + for (int i = 0; i < ALPHABET.length; i++) { + INDEXES[ALPHABET[i]] = i; + } + } + + /** + * Encodes the given bytes as a base58 string (no checksum is appended). + * + * @param input the bytes to encode + * @return the base58-encoded string + */ + public static String encode(byte[] input) { + if (input.length == 0) { + return ""; + } + // Count leading zeros. + int zeros = 0; + while (zeros < input.length && input[zeros] == 0) { + ++zeros; + } + // Convert base-256 digits to base-58 digits (plus conversion to ASCII characters) + input = Arrays.copyOf(input, input.length); // since we modify it in-place + char[] encoded = new char[input.length * 2]; // upper bound + int outputStart = encoded.length; + for (int inputStart = zeros; inputStart < input.length; ) { + encoded[--outputStart] = ALPHABET[divmod(input, inputStart, 256, 58)]; + if (input[inputStart] == 0) { + ++inputStart; // optimization - skip leading zeros + } + } + // Preserve exactly as many leading encoded zeros in output as there were leading zeros in input. + while (outputStart < encoded.length && encoded[outputStart] == ENCODED_ZERO) { + ++outputStart; + } + while (--zeros >= 0) { + encoded[--outputStart] = ENCODED_ZERO; + } + // Return encoded string (including encoded leading zeros). + return new String(encoded, outputStart, encoded.length - outputStart); + } + + /** + * Decodes the given base58 string into the original data bytes. + * + * @param input the base58-encoded string to decode + * @return the decoded data bytes + */ + public static byte[] decode(String input) { + if (input.length() == 0) { + return new byte[0]; + } + // Convert the base58-encoded ASCII chars to a base58 byte sequence (base58 digits). + byte[] input58 = new byte[input.length()]; + for (int i = 0; i < input.length(); ++i) { + char c = input.charAt(i); + int digit = c < 128 ? INDEXES[c] : -1; + if (digit < 0) { + throw new IllegalArgumentException(String.format("Invalid character in Base58: 0x%04x", (int) c)); + } + input58[i] = (byte) digit; + } + // Count leading zeros. + int zeros = 0; + while (zeros < input58.length && input58[zeros] == 0) { + ++zeros; + } + // Convert base-58 digits to base-256 digits. + byte[] decoded = new byte[input.length()]; + int outputStart = decoded.length; + for (int inputStart = zeros; inputStart < input58.length; ) { + decoded[--outputStart] = divmod(input58, inputStart, 58, 256); + if (input58[inputStart] == 0) { + ++inputStart; // optimization - skip leading zeros + } + } + // Ignore extra leading zeroes that were added during the calculation. + while (outputStart < decoded.length && decoded[outputStart] == 0) { + ++outputStart; + } + // Return decoded data (including original number of leading zeros). + return Arrays.copyOfRange(decoded, outputStart - zeros, decoded.length); + } + + public static BigInteger decodeToBigInteger(String input) { + return new BigInteger(1, decode(input)); + } + + /** + * Divides a number, represented as an array of bytes each containing a single digit + * in the specified base, by the given divisor. The given number is modified in-place + * to contain the quotient, and the return value is the remainder. + * + * @param number the number to divide + * @param firstDigit the index within the array of the first non-zero digit + * (this is used for optimization by skipping the leading zeros) + * @param base the base in which the number's digits are represented (up to 256) + * @param divisor the number to divide by (up to 256) + * @return the remainder of the division operation + */ + private static byte divmod(byte[] number, int firstDigit, int base, int divisor) { + // this is just long division which accounts for the base of the input digits + int remainder = 0; + for (int i = firstDigit; i < number.length; i++) { + int digit = (int) number[i] & 0xFF; + int temp = remainder * base + digit; + number[i] = (byte) (temp / divisor); + remainder = temp % divisor; + } + return (byte) remainder; + } +} diff --git a/src/test/java/com/epam/aidial/core/FileApiTest.java b/src/test/java/com/epam/aidial/core/FileApiTest.java index edf8131fd..6f93b0abe 100644 --- a/src/test/java/com/epam/aidial/core/FileApiTest.java +++ b/src/test/java/com/epam/aidial/core/FileApiTest.java @@ -77,7 +77,7 @@ public void testBucket(Vertx vertx, VertxTestContext context) { .send(context.succeeding(response -> { context.verify(() -> { assertEquals(200, response.statusCode()); - assertEquals(new Bucket("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM="), response.body()); + assertEquals(new Bucket("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt"), response.body()); context.completeNow(); }); })); @@ -87,9 +87,9 @@ public void testBucket(Vertx vertx, VertxTestContext context) { public void testEmptyFilesList(Vertx vertx, VertxTestContext context) { WebClient client = WebClient.create(vertx); - FolderMetadata emptyBucketResponse = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "/", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/", List.of()); - client.get(serverPort, "localhost", "/v1/files/metadata/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=/") + FolderMetadata emptyBucketResponse = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "/", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2F", List.of()); + client.get(serverPort, "localhost", "/v1/files/metadata/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FolderMetadata.class)) .send(context.succeeding(response -> { @@ -104,7 +104,7 @@ public void testEmptyFilesList(Vertx vertx, VertxTestContext context) { @Test public void testFileNotFound(Vertx vertx, VertxTestContext context) { WebClient client = WebClient.create(vertx); - client.get(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") + client.get(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.buffer()) .send(context.succeeding(response -> { @@ -119,7 +119,7 @@ public void testFileNotFound(Vertx vertx, VertxTestContext context) { @Test public void testInvalidFileUploadUrl(Vertx vertx, VertxTestContext context) { WebClient client = WebClient.create(vertx); - client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -155,12 +155,12 @@ public void testDownloadFromAnotherBucket(Vertx vertx, VertxTestContext context) Checkpoint checkpoint = context.checkpoint(2); WebClient client = WebClient.create(vertx); - FileMetadata expectedFileMetadata = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "file.txt", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt", 17, "text/custom"); + FileMetadata expectedFileMetadata = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "file.txt", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffile.txt", 17, "text/custom"); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); - client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffile.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -175,7 +175,7 @@ public void testDownloadFromAnotherBucket(Vertx vertx, VertxTestContext context) ); return promise.future(); }).andThen((mapper) -> { - client.get(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") + client.get(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/file.txt") .putHeader("Api-key", "proxyKey1") .as(BodyCodec.string()) .send(context.succeeding(response -> { @@ -190,7 +190,7 @@ public void testDownloadFromAnotherBucket(Vertx vertx, VertxTestContext context) @Test public void testFileUploadIntoAnotherBucket(Vertx vertx, VertxTestContext context) { WebClient client = WebClient.create(vertx); - client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/file.txt") .putHeader("Api-key", "proxyKey1") .as(BodyCodec.string()) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -208,17 +208,17 @@ public void testFileUpload(Vertx vertx, VertxTestContext context) { Checkpoint checkpoint = context.checkpoint(3); WebClient client = WebClient.create(vertx); - FolderMetadata emptyFolderResponse = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "/", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/", List.of()); - FileMetadata expectedFileMetadata = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "file.txt", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt", 17, "text/custom"); - FolderMetadata expectedFolderMetadata = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "/", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/", List.of(expectedFileMetadata)); + FolderMetadata emptyFolderResponse = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "/", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2F", List.of()); + FileMetadata expectedFileMetadata = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "file.txt", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffile.txt", 17, "text/custom"); + FolderMetadata expectedFolderMetadata = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "/", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2F", List.of(expectedFileMetadata)); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // verify no files - client.get(serverPort, "localhost", "/v1/files/metadata/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/") + client.get(serverPort, "localhost", "/v1/files/metadata/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FolderMetadata.class)) .send(context.succeeding(response -> { @@ -234,7 +234,7 @@ public void testFileUpload(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // upload test file - client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffile.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -251,7 +251,7 @@ public void testFileUpload(Vertx vertx, VertxTestContext context) { return promise.future(); }).andThen((result) -> { // verify uploaded file can be listed - client.get(serverPort, "localhost", "/v1/files/metadata/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/") + client.get(serverPort, "localhost", "/v1/files/metadata/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .send(context.succeeding(response -> { @@ -269,13 +269,13 @@ public void testFileDownload(Vertx vertx, VertxTestContext context) { Checkpoint checkpoint = context.checkpoint(2); WebClient client = WebClient.create(vertx); - FileMetadata expectedFileMetadata = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "file.txt", "folder1", "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/file.txt", 17, "text/plain"); + FileMetadata expectedFileMetadata = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "file.txt", "folder1", "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffolder1%2Ffile.txt", 17, "text/plain"); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // upload test file - client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/file.txt") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffolder1%2Ffile.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT), @@ -291,7 +291,7 @@ public void testFileDownload(Vertx vertx, VertxTestContext context) { return promise.future(); }).andThen((result) -> { - client.get(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/file.txt") + client.get(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/folder1/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .send(context.succeeding(response -> { @@ -309,23 +309,23 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { Checkpoint checkpoint = context.checkpoint(4); WebClient client = WebClient.create(vertx); - FolderMetadata emptyFolderResponse = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "/", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/", List.of()); - - FileMetadata expectedFileMetadata1 = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "file.txt", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt", 17, "text/custom"); - FileMetadata expectedFileMetadata2 = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "file.txt", "folder1", "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/file.txt", 17, "text/custom"); - FolderMetadata expectedFolder1Metadata = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "folder1", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/"); - FolderMetadata expectedRootFolderMetadata = new FolderMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "/", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/", + FolderMetadata emptyFolderResponse = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "/", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2F", List.of()); + + FileMetadata expectedFileMetadata1 = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "file.txt", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffile.txt", 17, "text/custom"); + FileMetadata expectedFileMetadata2 = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "file.txt", "folder1", "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffolder1%2Ffile.txt", 17, "text/custom"); + FolderMetadata expectedFolder1Metadata = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "folder1", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffolder1%2F"); + FolderMetadata expectedRootFolderMetadata = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "/", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2F", List.of(expectedFileMetadata1, expectedFolder1Metadata)); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // verify no files - client.get(serverPort, "localhost", "/v1/files/metadata/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/") + client.get(serverPort, "localhost", "/v1/files/metadata/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FolderMetadata.class)) .send(context.succeeding(response -> { @@ -341,7 +341,7 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // upload test file1 - client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/file.txt") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -359,7 +359,7 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // upload test file2 - client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/folder1/file.txt") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/folder1/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -376,7 +376,7 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { return promise.future(); }).andThen((result) -> { // verify uploaded files can be listed - client.get(serverPort, "localhost", "/v1/files/metadata/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/") + client.get(serverPort, "localhost", "/v1/files/metadata/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .send(context.succeeding(response -> { @@ -394,13 +394,13 @@ public void testFileDelete(Vertx vertx, VertxTestContext context) { Checkpoint checkpoint = context.checkpoint(3); WebClient client = WebClient.create(vertx); - FileMetadata expectedFileMetadata = new FileMetadata("XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM=", - "test_file.txt", null, "XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/test_file.txt", 17, "text/plain"); + FileMetadata expectedFileMetadata = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "test_file.txt", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ftest_file.txt", 17, "text/plain"); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // upload test file - client.put(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/test_file.txt") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/test_file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("test_file.txt", TEST_FILE_CONTENT), @@ -418,7 +418,7 @@ public void testFileDelete(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // delete file - client.delete(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/test_file.txt") + client.delete(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/test_file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .send(context.succeeding(response -> { @@ -432,7 +432,7 @@ public void testFileDelete(Vertx vertx, VertxTestContext context) { return promise.future(); }).andThen((mapper) -> { // try to download deleted file - client.get(serverPort, "localhost", "/v1/files/XQd0n1CwGStyHo2ZTJB4Ygb5AVeNtuKxQssR8AYqqWM%3D/test_file.txt") + client.get(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/test_file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .send(context.succeeding(response -> { diff --git a/src/test/java/com/epam/aidial/core/storage/ResourceDescriptionTest.java b/src/test/java/com/epam/aidial/core/storage/ResourceDescriptionTest.java index fc572d6f2..9763a3b68 100644 --- a/src/test/java/com/epam/aidial/core/storage/ResourceDescriptionTest.java +++ b/src/test/java/com/epam/aidial/core/storage/ResourceDescriptionTest.java @@ -20,8 +20,9 @@ public void testHomeFolderDescription() { assertEquals("aes/bucket/name", resource.getBucketName()); assertEquals("buckets/location/", resource.getBucketLocation()); assertEquals(ResourceType.FILE, resource.getType()); - assertEquals("aes%2Fbucket%2Fname/", resource.getUrl()); + assertEquals("aes%2Fbucket%2Fname%2F", resource.getUrl()); assertEquals("buckets/location/files/", resource.getAbsoluteFilePath()); + assertEquals("/", resource.getRelativePath()); assertTrue(resource.isFolder()); assertNull(resource.getParentPath()); assertNull(resource.getParentFolders()); @@ -34,8 +35,9 @@ public void testUserFolderDescription() { assertEquals("test-bucket-name", resource.getBucketName()); assertEquals("buckets/location/", resource.getBucketLocation()); assertEquals(ResourceType.FILE, resource.getType()); - assertEquals("test-bucket-name/folder1/", resource.getUrl()); + assertEquals("test-bucket-name%2Ffolder1%2F", resource.getUrl()); assertEquals("buckets/location/files/folder1/", resource.getAbsoluteFilePath()); + assertEquals("folder1/", resource.getRelativePath()); assertTrue(resource.isFolder()); assertNull(resource.getParentPath()); assertNull(resource.getParentFolders()); @@ -48,8 +50,9 @@ public void testUserFolderDescription2() { assertEquals("test-bucket-name", resource.getBucketName()); assertEquals("buckets/location/", resource.getBucketLocation()); assertEquals(ResourceType.FILE, resource.getType()); - assertEquals("test-bucket-name/folder1/folder2/", resource.getUrl()); + assertEquals("test-bucket-name%2Ffolder1%2Ffolder2%2F", resource.getUrl()); assertEquals("buckets/location/files/folder1/folder2/", resource.getAbsoluteFilePath()); + assertEquals("folder1/folder2/", resource.getRelativePath()); assertTrue(resource.isFolder()); assertEquals("folder1", resource.getParentPath()); assertIterableEquals(List.of("folder1"), resource.getParentFolders()); @@ -62,8 +65,9 @@ public void testUserFolderDescription3() { assertEquals("test-bucket-name", resource.getBucketName()); assertEquals("buckets/location/", resource.getBucketLocation()); assertEquals(ResourceType.FILE, resource.getType()); - assertEquals("test-bucket-name/folder1/folder2/folder3/", resource.getUrl()); + assertEquals("test-bucket-name%2Ffolder1%2Ffolder2%2Ffolder3%2F", resource.getUrl()); assertEquals("buckets/location/files/folder1/folder2/folder3/", resource.getAbsoluteFilePath()); + assertEquals("folder1/folder2/folder3/", resource.getRelativePath()); assertTrue(resource.isFolder()); assertEquals("folder1/folder2", resource.getParentPath()); assertIterableEquals(List.of("folder1", "folder2"), resource.getParentFolders()); @@ -76,8 +80,9 @@ public void testFileDescription1() { assertEquals("test-bucket-name", resource.getBucketName()); assertEquals("buckets/location/", resource.getBucketLocation()); assertEquals(ResourceType.FILE, resource.getType()); - assertEquals("test-bucket-name/file.txt", resource.getUrl()); + assertEquals("test-bucket-name%2Ffile.txt", resource.getUrl()); assertEquals("buckets/location/files/file.txt", resource.getAbsoluteFilePath()); + assertEquals("file.txt", resource.getRelativePath()); assertFalse(resource.isFolder()); assertNull(resource.getParentPath()); assertNull(resource.getParentFolders()); @@ -90,8 +95,9 @@ public void testFileDescription2() { assertEquals("test-bucket-name", resource.getBucketName()); assertEquals("buckets/location/", resource.getBucketLocation()); assertEquals(ResourceType.FILE, resource.getType()); - assertEquals("test-bucket-name/folder1/file.txt", resource.getUrl()); + assertEquals("test-bucket-name%2Ffolder1%2Ffile.txt", resource.getUrl()); assertEquals("buckets/location/files/folder1/file.txt", resource.getAbsoluteFilePath()); + assertEquals("folder1/file.txt", resource.getRelativePath()); assertFalse(resource.isFolder()); assertEquals("folder1", resource.getParentPath()); assertIterableEquals(List.of("folder1"), resource.getParentFolders()); @@ -104,8 +110,9 @@ public void testFileDescription3() { assertEquals("test-bucket-name", resource.getBucketName()); assertEquals("buckets/location/", resource.getBucketLocation()); assertEquals(ResourceType.FILE, resource.getType()); - assertEquals("test-bucket-name/folder1/folder2/file.txt", resource.getUrl()); + assertEquals("test-bucket-name%2Ffolder1%2Ffolder2%2Ffile.txt", resource.getUrl()); assertEquals("buckets/location/files/folder1/folder2/file.txt", resource.getAbsoluteFilePath()); + assertEquals("folder1/folder2/file.txt", resource.getRelativePath()); assertFalse(resource.isFolder()); assertEquals("folder1/folder2", resource.getParentPath()); assertIterableEquals(List.of("folder1", "folder2"), resource.getParentFolders()); @@ -118,12 +125,18 @@ public void testInvalidBucketLocation() { } @Test - public void testInvalidRelativePath() { - assertThrows(IllegalArgumentException.class, - () -> ResourceDescription.from(ResourceType.FILE, "bucket-name", "buckets/location/", "")); - assertThrows(IllegalArgumentException.class, - () -> ResourceDescription.from(ResourceType.FILE, "bucket-name", "buckets/location/", null)); - assertThrows(IllegalArgumentException.class, - () -> ResourceDescription.from(ResourceType.FILE, "bucket-name", "buckets/location/", " ")); + public void testEmptyRelativePath() { + assertEquals( + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", "/"), + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", "") + ); + assertEquals( + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", "/"), + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", null) + ); + assertEquals( + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", "/"), + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", " ") + ); } } From 7674dfa22f43eaa32d08ecbfb69857f7d11f9d58 Mon Sep 17 00:00:00 2001 From: Maksim_Hadalau Date: Mon, 18 Dec 2023 11:41:28 +0100 Subject: [PATCH 5/5] fix Files API patterns --- .../com/epam/aidial/core/controller/ControllerSelector.java | 4 ++-- src/test/java/com/epam/aidial/core/FileApiTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java index b1c400697..22582e64a 100644 --- a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java +++ b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java @@ -35,9 +35,9 @@ public class ControllerSelector { private static final Pattern PATTERN_BUCKET = Pattern.compile("/v1/bucket"); - private static final Pattern PATTERN_FILES = Pattern.compile("/v1/files/([^/]*)/(.*)"); + private static final Pattern PATTERN_FILES = Pattern.compile("/v1/files/([a-zA-Z0-9]+)/(.*)"); - private static final Pattern PATTERN_FILES_METADATA = Pattern.compile("/v1/files/metadata/([^/]*)/(.*)"); + private static final Pattern PATTERN_FILES_METADATA = Pattern.compile("/v1/files/metadata/([a-zA-Z0-9]+)/(.*)"); private static final Pattern PATTERN_RATE_RESPONSE = Pattern.compile("/+v1/([-.@a-zA-Z0-9]+)/rate"); private static final Pattern PATTERN_TOKENIZE = Pattern.compile("/+v1/deployments/([-.@a-zA-Z0-9]+)/tokenize"); diff --git a/src/test/java/com/epam/aidial/core/FileApiTest.java b/src/test/java/com/epam/aidial/core/FileApiTest.java index 6f93b0abe..ace09b0eb 100644 --- a/src/test/java/com/epam/aidial/core/FileApiTest.java +++ b/src/test/java/com/epam/aidial/core/FileApiTest.java @@ -136,14 +136,14 @@ public void testInvalidFileUploadUrl(Vertx vertx, VertxTestContext context) { @Test public void testInvalidFileUploadUrl2(Vertx vertx, VertxTestContext context) { WebClient client = WebClient.create(vertx); - client.put(serverPort, "localhost", "/v1/files/test-bucket/") + client.put(serverPort, "localhost", "/v1/files/testbucket/") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), context.succeeding(response -> { context.verify(() -> { assertEquals(403, response.statusCode()); - assertEquals("You don't have an access to the bucket test-bucket", response.body()); + assertEquals("You don't have an access to the bucket testbucket", response.body()); context.completeNow(); }); })