diff --git a/README.md b/README.md index 8cc988e9a..06cb06bec 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ Static settings are used on startup and cannot be changed while application is r | storage.credential | - |Blob storage secret key | storage.bucket | - |Blob storage bucket | storage.createBucket | false |Indicates whether bucket should be created on start-up - +| encryption.password | - |Password used for AES encryption +| encryption.salt | - |Salt used for AES encryption ### Dynamic settings Dynamic settings are stored in JSON files, specified via "config.files" static setting, and reloaded at interval, specified via "config.reload" static setting. Dynamic settings include: diff --git a/src/main/java/com/epam/aidial/core/AiDial.java b/src/main/java/com/epam/aidial/core/AiDial.java index 11706950d..35bceffed 100644 --- a/src/main/java/com/epam/aidial/core/AiDial.java +++ b/src/main/java/com/epam/aidial/core/AiDial.java @@ -1,13 +1,14 @@ package com.epam.aidial.core; -import com.auth0.jwk.UrlJwkProvider; import com.epam.aidial.core.config.ConfigStore; +import com.epam.aidial.core.config.Encryption; import com.epam.aidial.core.config.FileConfigStore; import com.epam.aidial.core.config.Storage; import com.epam.aidial.core.limiter.RateLimiter; import com.epam.aidial.core.log.GfLogStore; import com.epam.aidial.core.log.LogStore; import com.epam.aidial.core.security.AccessTokenValidator; +import com.epam.aidial.core.security.EncryptionService; import com.epam.aidial.core.storage.BlobStorage; import com.epam.aidial.core.upstream.UpstreamBalancer; import com.epam.deltix.gflog.core.LogConfigurator; @@ -68,7 +69,8 @@ void start() throws Exception { Storage storageConfig = Json.decodeValue(settings("storage").toBuffer(), Storage.class); storage = new BlobStorage(storageConfig); } - Proxy proxy = new Proxy(vertx, client, configStore, logStore, rateLimiter, upstreamBalancer, accessTokenValidator, storage); + EncryptionService encryptionService = new EncryptionService(Json.decodeValue(settings("encryption").toBuffer(), Encryption.class)); + Proxy proxy = new Proxy(vertx, client, configStore, logStore, rateLimiter, upstreamBalancer, accessTokenValidator, storage, encryptionService); server = vertx.createHttpServer(new HttpServerOptions(settings("server"))).requestHandler(proxy); open(server, HttpServer::listen); diff --git a/src/main/java/com/epam/aidial/core/Proxy.java b/src/main/java/com/epam/aidial/core/Proxy.java index 609f2efb3..bff0c45bc 100644 --- a/src/main/java/com/epam/aidial/core/Proxy.java +++ b/src/main/java/com/epam/aidial/core/Proxy.java @@ -8,6 +8,7 @@ import com.epam.aidial.core.limiter.RateLimiter; import com.epam.aidial.core.log.LogStore; import com.epam.aidial.core.security.AccessTokenValidator; +import com.epam.aidial.core.security.EncryptionService; import com.epam.aidial.core.security.ExtractedClaims; import com.epam.aidial.core.storage.BlobStorage; import com.epam.aidial.core.upstream.UpstreamBalancer; @@ -27,6 +28,7 @@ import java.net.URLDecoder; import java.nio.charset.StandardCharsets; +import java.util.Set; import static com.epam.aidial.core.security.AccessTokenValidator.extractTokenFromHeader; @@ -48,6 +50,8 @@ public class Proxy implements Handler { public static final int REQUEST_BODY_MAX_SIZE_BYTES = 16 * 1024 * 1024; public static final int FILES_REQUEST_BODY_MAX_SIZE_BYTES = 512 * 1024 * 1024; + private static final Set ALLOWED_HTTP_METHODS = Set.of(HttpMethod.GET, HttpMethod.POST, HttpMethod.PUT, HttpMethod.DELETE); + private final Vertx vertx; private final HttpClient client; private final ConfigStore configStore; @@ -56,6 +60,7 @@ public class Proxy implements Handler { private final UpstreamBalancer upstreamBalancer; private final AccessTokenValidator tokenValidator; private final BlobStorage storage; + private final EncryptionService encryptionService; @Override public void handle(HttpServerRequest request) { @@ -81,7 +86,7 @@ private void handleRequest(HttpServerRequest request) { } HttpMethod requestMethod = request.method(); - if (requestMethod != HttpMethod.GET && requestMethod != HttpMethod.POST && requestMethod != HttpMethod.DELETE) { + if (!ALLOWED_HTTP_METHODS.contains(requestMethod)) { respond(request, HttpStatus.METHOD_NOT_ALLOWED); return; } diff --git a/src/main/java/com/epam/aidial/core/config/Encryption.java b/src/main/java/com/epam/aidial/core/config/Encryption.java new file mode 100644 index 000000000..81107ddc5 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/config/Encryption.java @@ -0,0 +1,9 @@ +package com.epam.aidial.core.config; + +import lombok.Data; + +@Data +public class Encryption { + String password; + String salt; +} diff --git a/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java b/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java new file mode 100644 index 000000000..fd5d9195e --- /dev/null +++ b/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java @@ -0,0 +1,39 @@ +package com.epam.aidial.core.controller; + +import com.epam.aidial.core.Proxy; +import com.epam.aidial.core.ProxyContext; +import com.epam.aidial.core.storage.BlobStorageUtil; +import com.epam.aidial.core.storage.ResourceDescription; +import com.epam.aidial.core.storage.ResourceType; +import com.epam.aidial.core.util.HttpStatus; +import io.vertx.core.Future; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public abstract class AccessControlBaseController { + + final Proxy proxy; + final ProxyContext context; + + + public Future handle(String bucket, String filePath) { + String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); + String decryptedBucket = proxy.getEncryptionService().decrypt(bucket); + + if (!expectedUserBucket.equals(decryptedBucket)) { + return context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the bucket " + bucket); + } + + ResourceDescription resource; + try { + resource = ResourceDescription.from(ResourceType.FILE, bucket, decryptedBucket, filePath); + } catch (Exception ex) { + return context.respond(HttpStatus.BAD_REQUEST, "Invalid file url provided"); + } + + return handle(resource); + } + + protected abstract Future handle(ResourceDescription resource); + +} diff --git a/src/main/java/com/epam/aidial/core/controller/BucketController.java b/src/main/java/com/epam/aidial/core/controller/BucketController.java new file mode 100644 index 000000000..2f46be9e9 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/controller/BucketController.java @@ -0,0 +1,25 @@ +package com.epam.aidial.core.controller; + +import com.epam.aidial.core.Proxy; +import com.epam.aidial.core.ProxyContext; +import com.epam.aidial.core.data.Bucket; +import com.epam.aidial.core.security.EncryptionService; +import com.epam.aidial.core.storage.BlobStorageUtil; +import com.epam.aidial.core.util.HttpStatus; +import io.vertx.core.Future; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class BucketController { + + private final Proxy proxy; + private final ProxyContext context; + + public Future getBucket() { + EncryptionService encryptionService = proxy.getEncryptionService(); + String bucketLocation = BlobStorageUtil.buildUserBucket(context); + String encryptedBucket = encryptionService.encrypt(bucketLocation); + + return context.respond(HttpStatus.OK, new Bucket(encryptedBucket)); + } +} diff --git a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java index 8fbd5b696..22582e64a 100644 --- a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java +++ b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java @@ -33,7 +33,11 @@ public class ControllerSelector { private static final Pattern PATTERN_APPLICATIONS = Pattern.compile("/+openai/applications"); - private static final Pattern PATTERN_FILES = Pattern.compile("/v1/files(.*)"); + private static final Pattern PATTERN_BUCKET = Pattern.compile("/v1/bucket"); + + private static final Pattern PATTERN_FILES = Pattern.compile("/v1/files/([a-zA-Z0-9]+)/(.*)"); + + private static final Pattern PATTERN_FILES_METADATA = Pattern.compile("/v1/files/metadata/([a-zA-Z0-9]+)/(.*)"); private static final Pattern PATTERN_RATE_RESPONSE = Pattern.compile("/+v1/([-.@a-zA-Z0-9]+)/rate"); private static final Pattern PATTERN_TOKENIZE = Pattern.compile("/+v1/deployments/([-.@a-zA-Z0-9]+)/tokenize"); @@ -50,6 +54,8 @@ public Controller select(Proxy proxy, ProxyContext context) { controller = selectPost(proxy, context, path); } else if (method == HttpMethod.DELETE) { controller = selectDelete(proxy, context, path); + } else if (method == HttpMethod.PUT) { + controller = selectPut(proxy, context, path); } return (controller == null) ? new RouteController(proxy, context) : controller; @@ -123,17 +129,26 @@ private static Controller selectGet(Proxy proxy, ProxyContext context, String pa return controller::getApplications; } + match = match(PATTERN_FILES_METADATA, path); + if (match != null) { + String bucket = match.group(1); + String filePath = match.group(2); + FileMetadataController controller = new FileMetadataController(proxy, context); + return () -> controller.handle(bucket, filePath); + } + match = match(PATTERN_FILES, path); if (match != null) { - String filePath = match.group(1); - String purpose = context.getRequest().params().get(DownloadFileController.PURPOSE_FILE_QUERY_PARAMETER); - if (DownloadFileController.QUERY_METADATA_QUERY_PARAMETER_VALUE.equals(purpose)) { - FileMetadataController controller = new FileMetadataController(proxy, context); - return () -> controller.list(filePath); - } else { - DownloadFileController controller = new DownloadFileController(proxy, context); - return () -> controller.download(filePath); - } + String bucket = match.group(1); + String filePath = match.group(2); + DownloadFileController controller = new DownloadFileController(proxy, context); + return () -> controller.handle(bucket, filePath); + } + + match = match(PATTERN_BUCKET, path); + if (match != null) { + BucketController controller = new BucketController(proxy, context); + return controller::getBucket; } return null; @@ -148,22 +163,15 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return () -> controller.handle(deploymentId, deploymentApi); } - match = match(PATTERN_FILES, path); - if (match != null) { - String relativeFilePath = match.group(1); - UploadFileController controller = new UploadFileController(proxy, context); - return () -> controller.upload(relativeFilePath); - } - match = match(PATTERN_RATE_RESPONSE, path); if (match != null) { String deploymentId = match.group(1); Function getter = (model) -> { return Optional.ofNullable(model) - .map(d -> d.getFeatures()) - .map(t -> t.getRateEndpoint()) - .orElse(null); + .map(d -> d.getFeatures()) + .map(t -> t.getRateEndpoint()) + .orElse(null); }; DeploymentFeatureController controller = new DeploymentFeatureController(proxy, context); @@ -176,9 +184,9 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p Function getter = (model) -> { return Optional.ofNullable(model) - .map(d -> d.getFeatures()) - .map(t -> t.getTokenizeEndpoint()) - .orElse(null); + .map(d -> d.getFeatures()) + .map(t -> t.getTokenizeEndpoint()) + .orElse(null); }; DeploymentFeatureController controller = new DeploymentFeatureController(proxy, context); @@ -191,9 +199,9 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p Function getter = (model) -> { return Optional.ofNullable(model) - .map(d -> d.getFeatures()) - .map(t -> t.getTruncatePromptEndpoint()) - .orElse(null); + .map(d -> d.getFeatures()) + .map(t -> t.getTruncatePromptEndpoint()) + .orElse(null); }; DeploymentFeatureController controller = new DeploymentFeatureController(proxy, context); @@ -206,9 +214,22 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p private static Controller selectDelete(Proxy proxy, ProxyContext context, String path) { Matcher match = match(PATTERN_FILES, path); if (match != null) { - String relativeFilePath = match.group(1); + String bucket = match.group(1); + String filePath = match.group(2); DeleteFileController controller = new DeleteFileController(proxy, context); - return () -> controller.delete(relativeFilePath); + return () -> controller.handle(bucket, filePath); + } + + return null; + } + + private static Controller selectPut(Proxy proxy, ProxyContext context, String path) { + Matcher match = match(PATTERN_FILES, path); + if (match != null) { + String bucket = match.group(1); + String filePath = match.group(2); + UploadFileController controller = new UploadFileController(proxy, context); + return () -> controller.handle(bucket, filePath); } return null; diff --git a/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java b/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java index ae9945cf6..1fa329c5c 100644 --- a/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/DeleteFileController.java @@ -3,26 +3,26 @@ import com.epam.aidial.core.Proxy; import com.epam.aidial.core.ProxyContext; import com.epam.aidial.core.storage.BlobStorage; -import com.epam.aidial.core.storage.BlobStorageUtil; +import com.epam.aidial.core.storage.ResourceDescription; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j -@AllArgsConstructor -public class DeleteFileController { - private final Proxy proxy; - private final ProxyContext context; +public class DeleteFileController extends AccessControlBaseController { + + public DeleteFileController(Proxy proxy, ProxyContext context) { + super(proxy, context); + } + + @Override + protected Future handle(ResourceDescription resource) { + if (resource.isFolder()) { + return context.respond(HttpStatus.BAD_REQUEST, "Can't delete a folder"); + } + + String absoluteFilePath = resource.getAbsoluteFilePath(); - /** - * Deletes file from storage. - * Current API implementation requires a relative path, absolute path will be calculated based on authentication context - * - * @param path relative path, for example: /inputs/data.csv - */ - public Future delete(String path) { - String absoluteFilePath = BlobStorageUtil.buildAbsoluteFilePath(context, path); BlobStorage storage = proxy.getStorage(); Future result = proxy.getVertx().executeBlocking(() -> { try { diff --git a/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java b/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java index 801af823f..d6752951e 100644 --- a/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/DownloadFileController.java @@ -2,14 +2,13 @@ import com.epam.aidial.core.Proxy; import com.epam.aidial.core.ProxyContext; -import com.epam.aidial.core.storage.BlobStorageUtil; import com.epam.aidial.core.storage.InputStreamReader; +import com.epam.aidial.core.storage.ResourceDescription; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; import io.vertx.core.Promise; import io.vertx.core.http.HttpHeaders; import io.vertx.core.http.HttpServerResponse; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jclouds.blobstore.domain.Blob; import org.jclouds.io.MutableContentMetadata; @@ -18,36 +17,20 @@ import java.io.IOException; @Slf4j -@AllArgsConstructor -public class DownloadFileController { +public class DownloadFileController extends AccessControlBaseController { - private static final String PATH_TYPE_QUERY_PARAMETER = "path"; - private static final String RELATIVE_PATH_TYPE = "relative"; - - static final String PURPOSE_FILE_QUERY_PARAMETER = "purpose"; - - static final String QUERY_METADATA_QUERY_PARAMETER_VALUE = "metadata"; - - private final Proxy proxy; - private final ProxyContext context; + public DownloadFileController(Proxy proxy, ProxyContext context) { + super(proxy, context); + } - /** - * Downloads file content from provided path. - * Path can be either absolute or relative. - * Path type determined by "path" query parameter which can be "absolute" or "relative"(default value) - * - * @param path file path; absolute or relative - */ - public Future download(String path) { - String pathType = context.getRequest().params().get(PATH_TYPE_QUERY_PARAMETER); - String absoluteFilePath; - if (RELATIVE_PATH_TYPE.equals(pathType)) { - absoluteFilePath = BlobStorageUtil.buildAbsoluteFilePath(context, path); - } else { - absoluteFilePath = path; + @Override + protected Future handle(ResourceDescription resource) { + if (resource.isFolder()) { + return context.respond(HttpStatus.BAD_REQUEST, "Can't download a folder"); } + Future blobFuture = proxy.getVertx().executeBlocking(() -> - proxy.getStorage().load(BlobStorageUtil.removeLeadingPathSeparator(absoluteFilePath))); + proxy.getStorage().load(resource.getAbsoluteFilePath())); Promise result = Promise.promise(); blobFuture.onSuccess(blob -> { @@ -78,7 +61,7 @@ public Future download(String path) { result.fail(e); } }).onFailure(error -> context.respond(HttpStatus.INTERNAL_SERVER_ERROR, - "Failed to fetch file with path " + path)); + "Failed to fetch file with path %s/%s".formatted(resource.getBucketName(), resource.getRelativePath()))); return result.future(); } diff --git a/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java b/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java index aabb1fb0e..ed0035bda 100644 --- a/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java +++ b/src/main/java/com/epam/aidial/core/controller/FileMetadataController.java @@ -4,36 +4,33 @@ import com.epam.aidial.core.ProxyContext; import com.epam.aidial.core.data.FileMetadataBase; import com.epam.aidial.core.storage.BlobStorage; -import com.epam.aidial.core.storage.BlobStorageUtil; +import com.epam.aidial.core.storage.ResourceDescription; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import java.util.List; - -@AllArgsConstructor @Slf4j -public class FileMetadataController { - private final Proxy proxy; - private final ProxyContext context; +public class FileMetadataController extends AccessControlBaseController { + + public FileMetadataController(Proxy proxy, ProxyContext context) { + super(proxy, context); + } - /** - * Lists all files and folders that belong to the provided path. - * Current API implementation requires a relative path, absolute path will be calculated based on authentication context - * - * @param path relative path, for example: /inputs - */ - public Future list(String path) { + @Override + protected Future handle(ResourceDescription resource) { BlobStorage storage = proxy.getStorage(); return proxy.getVertx().executeBlocking(() -> { try { - String absolutePath = BlobStorageUtil.buildAbsoluteFilePath(context, path); - List metadata = storage.listMetadata(absolutePath); - context.respond(HttpStatus.OK, metadata); + FileMetadataBase metadata = storage.listMetadata(resource); + if (metadata != null) { + context.respond(HttpStatus.OK, metadata); + } else { + context.respond(HttpStatus.NOT_FOUND); + } } catch (Exception ex) { log.error("Failed to list files", ex); - context.respond(HttpStatus.INTERNAL_SERVER_ERROR, "Failed to list files by path %s".formatted(path)); + context.respond(HttpStatus.INTERNAL_SERVER_ERROR, + "Failed to list files by path %s/%s".formatted(resource.getBucketName(), resource.getRelativePath())); } return null; diff --git a/src/main/java/com/epam/aidial/core/controller/UploadFileController.java b/src/main/java/com/epam/aidial/core/controller/UploadFileController.java index 9486f72d0..3ea3e0ac1 100644 --- a/src/main/java/com/epam/aidial/core/controller/UploadFileController.java +++ b/src/main/java/com/epam/aidial/core/controller/UploadFileController.java @@ -2,45 +2,39 @@ import com.epam.aidial.core.Proxy; import com.epam.aidial.core.ProxyContext; -import com.epam.aidial.core.storage.BlobStorageUtil; import com.epam.aidial.core.storage.BlobWriteStream; +import com.epam.aidial.core.storage.ResourceDescription; import com.epam.aidial.core.util.HttpStatus; import io.vertx.core.Future; import io.vertx.core.Promise; import io.vertx.core.buffer.Buffer; import io.vertx.core.streams.Pipe; import io.vertx.core.streams.impl.PipeImpl; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j -@RequiredArgsConstructor -public class UploadFileController { +public class UploadFileController extends AccessControlBaseController { - private final Proxy proxy; - private final ProxyContext context; + public UploadFileController(Proxy proxy, ProxyContext context) { + super(proxy, context); + } + + @Override + protected Future handle(ResourceDescription resource) { + if (resource.isFolder()) { + return context.respond(HttpStatus.BAD_REQUEST, "File name is missing"); + } - /** - * Uploads file to the storage. - * Current API implementation requires a relative path, absolute path will be calculated based on authentication context. - * File name defined in multipart/form-data context - * - * @param path relative path, for example: /inputs - */ - public Future upload(String path) { - String absoluteFilePath = BlobStorageUtil.buildAbsoluteFilePath(context, path); Promise result = Promise.promise(); context.getRequest() .setExpectMultipart(true) .uploadHandler(upload -> { String contentType = upload.contentType(); - String filename = upload.filename(); Pipe pipe = new PipeImpl<>(upload).endOnFailure(false); BlobWriteStream writeStream = new BlobWriteStream( proxy.getVertx(), proxy.getStorage(), - filename, - absoluteFilePath, + resource, contentType); pipe.to(writeStream, result); @@ -49,7 +43,7 @@ public Future upload(String path) { .onFailure(error -> { writeStream.abortUpload(error); context.respond(HttpStatus.INTERNAL_SERVER_ERROR, - "Failed to upload file by path " + path); + "Failed to upload file by path %s/%s".formatted(resource.getBucketName(), resource.getRelativePath())); }); }); diff --git a/src/main/java/com/epam/aidial/core/data/Bucket.java b/src/main/java/com/epam/aidial/core/data/Bucket.java new file mode 100644 index 000000000..21ef09f57 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/data/Bucket.java @@ -0,0 +1,4 @@ +package com.epam.aidial.core.data; + +public record Bucket(String bucket) { +} diff --git a/src/main/java/com/epam/aidial/core/data/FileMetadata.java b/src/main/java/com/epam/aidial/core/data/FileMetadata.java index 1a1fe6403..6e70c5805 100644 --- a/src/main/java/com/epam/aidial/core/data/FileMetadata.java +++ b/src/main/java/com/epam/aidial/core/data/FileMetadata.java @@ -1,5 +1,6 @@ package com.epam.aidial.core.data; +import com.epam.aidial.core.storage.ResourceDescription; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -11,9 +12,13 @@ public class FileMetadata extends FileMetadataBase { long contentLength; String contentType; - public FileMetadata(String name, String path, long contentLength, String contentType) { - super(name, path, FileType.FILE); + public FileMetadata(String bucket, String name, String path, String url, long contentLength, String contentType) { + super(name, path, bucket, url, FileType.FILE); this.contentLength = contentLength; this.contentType = contentType; } + + public FileMetadata(ResourceDescription resource, long contentLength, String contentType) { + this(resource.getBucketName(), resource.getName(), resource.getParentPath(), resource.getUrl(), contentLength, contentType); + } } diff --git a/src/main/java/com/epam/aidial/core/data/FileMetadataBase.java b/src/main/java/com/epam/aidial/core/data/FileMetadataBase.java index d75dc6b8d..700feacda 100644 --- a/src/main/java/com/epam/aidial/core/data/FileMetadataBase.java +++ b/src/main/java/com/epam/aidial/core/data/FileMetadataBase.java @@ -9,6 +9,8 @@ @NoArgsConstructor public abstract class FileMetadataBase { private String name; - private String path; + private String parentPath; + private String bucket; + private String url; private FileType type; } diff --git a/src/main/java/com/epam/aidial/core/data/FolderMetadata.java b/src/main/java/com/epam/aidial/core/data/FolderMetadata.java index 16d060d51..1ea16174d 100644 --- a/src/main/java/com/epam/aidial/core/data/FolderMetadata.java +++ b/src/main/java/com/epam/aidial/core/data/FolderMetadata.java @@ -1,8 +1,33 @@ package com.epam.aidial.core.data; +import com.epam.aidial.core.storage.ResourceDescription; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.List; + +@Getter +@Setter +@NoArgsConstructor public class FolderMetadata extends FileMetadataBase { - public FolderMetadata(String name, String path) { - super(name, path, FileType.FOLDER); + private List files; + + public FolderMetadata(String bucket, String name, String path, String url, List files) { + super(name, path, bucket, url, FileType.FOLDER); + this.files = files; + } + + public FolderMetadata(String bucket, String name, String path, String url) { + this(bucket, name, path, url, null); + } + + public FolderMetadata(ResourceDescription resource) { + this(resource, null); + } + + public FolderMetadata(ResourceDescription resource, List files) { + this(resource.getBucketName(), resource.getName(), resource.getParentPath(), resource.getUrl(), files); } } diff --git a/src/main/java/com/epam/aidial/core/security/EncryptionService.java b/src/main/java/com/epam/aidial/core/security/EncryptionService.java new file mode 100644 index 000000000..b2f9ce8fd --- /dev/null +++ b/src/main/java/com/epam/aidial/core/security/EncryptionService.java @@ -0,0 +1,63 @@ +package com.epam.aidial.core.security; + +import com.epam.aidial.core.config.Encryption; +import com.epam.aidial.core.util.Base58; +import lombok.extern.slf4j.Slf4j; + +import java.security.spec.KeySpec; +import java.util.Objects; +import javax.annotation.Nullable; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; + +@Slf4j +public class EncryptionService { + + private static final String TRANSFORMATION = "AES/CBC/PKCS5Padding"; + + private final SecretKey key; + private final IvParameterSpec iv = new IvParameterSpec( + new byte[]{25, -13, -25, -119, -42, 117, -118, -128, -101, 20, -103, -81, -48, -23, -54, -113}); + + public EncryptionService(Encryption config) { + this(config.getPassword(), config.getSalt()); + } + + EncryptionService(String password, String salt) { + Objects.requireNonNull(password, "Encryption password is not set"); + Objects.requireNonNull(salt, "Encryption salt is not set"); + try { + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256"); + KeySpec spec = new PBEKeySpec(password.toCharArray(), salt.getBytes(), 3000, 256); + key = new SecretKeySpec(secretKeyFactory.generateSecret(spec).getEncoded(), "AES"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public String encrypt(String value) { + try { + Cipher cipher = Cipher.getInstance(TRANSFORMATION); + cipher.init(Cipher.ENCRYPT_MODE, key, iv); + return Base58.encode(cipher.doFinal(value.getBytes())); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Nullable + public String decrypt(String value) { + try { + Cipher cipher = Cipher.getInstance(TRANSFORMATION); + cipher.init(Cipher.DECRYPT_MODE, key, iv); + return new String(cipher.doFinal(Base58.decode(value))); + } catch (Exception e) { + log.error("Failed to decrypt value " + value, e); + return null; + } + } +} diff --git a/src/main/java/com/epam/aidial/core/storage/BlobStorage.java b/src/main/java/com/epam/aidial/core/storage/BlobStorage.java index 440f7e7da..9e9015fd0 100644 --- a/src/main/java/com/epam/aidial/core/storage/BlobStorage.java +++ b/src/main/java/com/epam/aidial/core/storage/BlobStorage.java @@ -59,13 +59,12 @@ public BlobStorage(Storage config, Properties overrides) { /** * Initialize multipart upload * - * @param fileName name of the file, for example: data.csv - * @param path absolute path according to the bucket, for example: Users/user1/files/input data - * @param contentType MIME type of the content, for example: text/csv + * @param absoluteFilePath absolute path according to the bucket, for example: Users/user1/files/input/file.txt + * @param contentType MIME type of the content, for example: text/csv */ @SuppressWarnings("UnstableApiUsage") // multipart upload uses beta API - public MultipartUpload initMultipartUpload(String fileName, String path, String contentType) { - BlobMetadata metadata = buildBlobMetadata(fileName, path, contentType, bucketName); + public MultipartUpload initMultipartUpload(String absoluteFilePath, String contentType) { + BlobMetadata metadata = buildBlobMetadata(absoluteFilePath, contentType, bucketName); return blobStore.initiateMultipartUpload(bucketName, metadata, PutOptions.NONE); } @@ -102,14 +101,12 @@ public void abortMultipartUpload(MultipartUpload multipart) { /** * Upload file in a single request * - * @param fileName name of the file, for example: data.csv - * @param path absolute path according to the bucket, for example: Users/user1/files/input data - * @param contentType MIME type of the content, for example: text/csv - * @param data whole content data + * @param absoluteFilePath absolute path according to the bucket, for example: Users/user1/files/input/file.txt + * @param contentType MIME type of the content, for example: text/csv + * @param data whole content data */ - public void store(String fileName, String path, String contentType, Buffer data) { - String filePath = BlobStorageUtil.buildFilePath(fileName, path); - Blob blob = blobStore.blobBuilder(filePath) + public void store(String absoluteFilePath, String contentType, Buffer data) { + Blob blob = blobStore.blobBuilder(absoluteFilePath) .payload(new BufferPayload(data)) .contentLength(data.length()) .contentType(contentType) @@ -138,14 +135,23 @@ public void delete(String filePath) { } /** - * List all files/folder metadata for a given path - * - * @param path absolute path for a folder, for example: Users/user1/files + * List all files/folder metadata for a given resource */ - public List listMetadata(String path) { - ListContainerOptions options = buildListContainerOptions(BlobStorageUtil.normalizePathForQuery(path)); - PageSet list = blobStore.list(bucketName, options); - return list.stream().map(BlobStorage::buildFileMetadata).toList(); + public FileMetadataBase listMetadata(ResourceDescription resource) { + ListContainerOptions options = buildListContainerOptions(resource.getAbsoluteFilePath()); + PageSet list = blobStore.list(this.bucketName, options); + List filesMetadata = list.stream().map(meta -> buildFileMetadata(resource, meta)).toList(); + + // listing folder + if (resource.isFolder()) { + return new FolderMetadata(resource, filesMetadata); + } else { + // listing file + if (filesMetadata.size() == 1) { + return filesMetadata.get(0); + } + return null; + } } @Override @@ -159,27 +165,27 @@ private static ListContainerOptions buildListContainerOptions(String prefix) { .delimiter(BlobStorageUtil.PATH_SEPARATOR); } - private static FileMetadataBase buildFileMetadata(StorageMetadata metadata) { - String absoluteFilePath = metadata.getName(); - String[] elements = absoluteFilePath.split(BlobStorageUtil.PATH_SEPARATOR); - String lastElement = elements[elements.length - 1]; - String path = absoluteFilePath.substring(0, absoluteFilePath.length() - lastElement.length() - 1); - String normalizedPath = BlobStorageUtil.normalizeParentPath(path); + private static FileMetadataBase buildFileMetadata(ResourceDescription resource, StorageMetadata metadata) { + String bucketName = resource.getBucketName(); + ResourceDescription resultResource = getResourceDescription(resource.getType(), bucketName, + resource.getBucketLocation(), metadata.getName()); return switch (metadata.getType()) { - case BLOB -> - new FileMetadata(lastElement, normalizedPath, metadata.getSize(), - ((BlobMetadata) metadata).getContentMetadata().getContentType()); - case FOLDER, RELATIVE_PATH -> - new FolderMetadata(lastElement, normalizedPath); + case BLOB -> new FileMetadata(resultResource, metadata.getSize(), + ((BlobMetadata) metadata).getContentMetadata().getContentType()); + case FOLDER, RELATIVE_PATH -> new FolderMetadata(resultResource); case CONTAINER -> throw new IllegalArgumentException("Can't list container"); }; } - private static BlobMetadata buildBlobMetadata(String fileName, String path, String contentType, String bucketName) { - String filePath = BlobStorageUtil.buildFilePath(fileName, path); + private static ResourceDescription getResourceDescription(ResourceType resourceType, String bucketName, String bucketLocation, String absoluteFilePath) { + String relativeFilePath = absoluteFilePath.substring(bucketLocation.length() + resourceType.getFolder().length() + 1); + return ResourceDescription.from(resourceType, bucketName, bucketLocation, relativeFilePath); + } + + private static BlobMetadata buildBlobMetadata(String absoluteFilePath, String contentType, String bucketName) { ContentMetadata contentMetadata = buildContentMetadata(contentType); - return new BlobMetadataImpl(null, filePath, null, null, null, null, null, Map.of(), null, bucketName, contentMetadata, null, Tier.STANDARD); + return new BlobMetadataImpl(null, absoluteFilePath, null, null, null, null, null, Map.of(), null, bucketName, contentMetadata, null, Tier.STANDARD); } private static ContentMetadata buildContentMetadata(String contentType) { diff --git a/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java b/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java index c6d2a17f7..b32fce61b 100644 --- a/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java +++ b/src/main/java/com/epam/aidial/core/storage/BlobStorageUtil.java @@ -7,92 +7,37 @@ @UtilityClass public class BlobStorageUtil { - private static final String USER_ROOT_DIR_PATTERN = "Users/%s/files/%s"; + private static final String USER_BUCKET_PATTERN = "Users/%s/"; - private static final String API_KEY_ROOT_DIR_PATTERN = "Keys/%s/files/%s"; + private static final String API_KEY_BUCKET_PATTERN = "Keys/%s/"; public static final String PATH_SEPARATOR = "/"; - private static final char DELIMITER = PATH_SEPARATOR.charAt(0); - - /** - * Normalize provided path for listing files query by removing leading and adding trailing path separator. - * For example, path /Users/User1/files/folders will be transformed to Users/User1/files/folders/ - * - * @return normalized path - */ - public String normalizePathForQuery(String path) { - if (path == null || path.isBlank()) { - return null; - } - - if (path.equals(PATH_SEPARATOR)) { - return path; - } - - // remove leading separator - if (path.charAt(0) == DELIMITER) { - path = path.substring(1); - } - - // add trailing separator if needed - return path.charAt(path.length() - 1) == DELIMITER ? path : path + PATH_SEPARATOR; - } - - /** - * Normalize parent path for file metadata by adding leading path separator and removing trailing one. - * For example, path Users/User1/files/folder1/ will be transformed to /Users/User1/files/folder1 - * - * @return normalized path - */ - public String normalizeParentPath(String path) { - path = removeTrailingPathSeparator(path); - - return path.charAt(0) == DELIMITER ? path : PATH_SEPARATOR + path; - } - - public String removeLeadingPathSeparator(String path) { - if (path == null || path.isBlank()) { - return null; - } - return path.charAt(0) == DELIMITER ? path.substring(1) : path; - } - - public String buildFilePath(String fileName, String path) { - path = removeTrailingPathSeparator(path); - return path + PATH_SEPARATOR + fileName; - } public String getContentType(String fileName) { String mimeType = MimeMapping.getMimeTypeForFilename(fileName); return mimeType == null ? "application/octet-stream" : mimeType; } - public String buildAbsoluteFilePath(ProxyContext context, String path) { - return buildAbsoluteFilePath(context.getUserSub(), context.getProject(), path); - } + public String buildUserBucket(ProxyContext context) { + String userSub = context.getUserSub(); + String apiKeyId = context.getProject(); - private String buildAbsoluteFilePath(String userSub, String apiKeyId, String path) { - path = removeLeadingAndTrailingPathSeparators(path); if (userSub != null) { - return USER_ROOT_DIR_PATTERN.formatted(userSub, path); - } else { - return API_KEY_ROOT_DIR_PATTERN.formatted(apiKeyId, path); + return USER_BUCKET_PATTERN.formatted(userSub); } - } - String removeTrailingPathSeparator(String path) { - if (path == null || path.isBlank()) { - return null; + if (apiKeyId != null) { + return API_KEY_BUCKET_PATTERN.formatted(apiKeyId); } - int length = path.length(); - return path.charAt(length - 1) == DELIMITER ? path.substring(0, length - 1) : path; + + throw new IllegalArgumentException("Can't find user bucket. Either user sub or api-key project must be provided"); } - String removeLeadingAndTrailingPathSeparators(String path) { - if (path == null || path.isBlank() || path.equals(PATH_SEPARATOR)) { - return ""; - } - path = removeLeadingPathSeparator(path); - return removeTrailingPathSeparator(path); + public String buildAbsoluteFilePath(ResourceType resource, String bucket, String path) { + return bucket + resource.getFolder() + PATH_SEPARATOR + path; + } + + public boolean isFolder(String path) { + return path.endsWith(PATH_SEPARATOR); } } diff --git a/src/main/java/com/epam/aidial/core/storage/BlobWriteStream.java b/src/main/java/com/epam/aidial/core/storage/BlobWriteStream.java index 089589175..725dc45b2 100644 --- a/src/main/java/com/epam/aidial/core/storage/BlobWriteStream.java +++ b/src/main/java/com/epam/aidial/core/storage/BlobWriteStream.java @@ -28,8 +28,7 @@ public class BlobWriteStream implements WriteStream { private final Vertx vertx; private final BlobStorage storage; - private final String parentPath; - private final String fileName; + private final ResourceDescription resource; private final String contentType; private final Buffer chunkBuffer = Buffer.buffer(); @@ -51,14 +50,12 @@ public class BlobWriteStream implements WriteStream { public BlobWriteStream(Vertx vertx, BlobStorage storage, - String fileName, - String parentPath, + ResourceDescription resource, String contentType) { this.vertx = vertx; this.storage = storage; - this.fileName = fileName; - this.parentPath = parentPath; - this.contentType = contentType != null ? contentType : BlobStorageUtil.getContentType(fileName); + this.resource = resource; + this.contentType = contentType != null ? contentType : BlobStorageUtil.getContentType(resource.getName()); } @Override @@ -103,11 +100,10 @@ public void end(Handler> handler) { } Buffer lastChunk = chunkBuffer.slice(0, position); - metadata = new FileMetadata(fileName, BlobStorageUtil.normalizeParentPath(parentPath), - bytesHandled, contentType); + metadata = new FileMetadata(resource, bytesHandled, contentType); if (mpu == null) { log.info("Resource is too small for multipart upload, sending as a regular blob"); - storage.store(fileName, parentPath, contentType, lastChunk); + storage.store(resource.getAbsoluteFilePath(), contentType, lastChunk); } else { if (position != 0) { MultipartPart part = storage.storeMultipartPart(mpu, ++chunkNumber, lastChunk); @@ -150,7 +146,7 @@ public WriteStream drainHandler(Handler handler) { synchronized (BlobWriteStream.this) { try { if (mpu == null) { - mpu = storage.initMultipartUpload(fileName, parentPath, contentType); + mpu = storage.initMultipartUpload(resource.getAbsoluteFilePath(), contentType); } MultipartPart part = storage.storeMultipartPart(mpu, ++chunkNumber, chunkBuffer.slice(0, position)); parts.add(part); diff --git a/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java b/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java new file mode 100644 index 000000000..8376030de --- /dev/null +++ b/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java @@ -0,0 +1,93 @@ +package com.epam.aidial.core.storage; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.List; + +@Data +@AllArgsConstructor(access = AccessLevel.PRIVATE) +public class ResourceDescription { + ResourceType type; + String name; + List parentFolders; + String relativePath; + String bucketName; + String bucketLocation; + boolean isFolder; + + public String getUrl() { + StringBuilder builder = new StringBuilder(); + builder.append(bucketName) + .append(BlobStorageUtil.PATH_SEPARATOR); + if (parentFolders != null) { + builder.append(String.join(BlobStorageUtil.PATH_SEPARATOR, parentFolders)) + .append(BlobStorageUtil.PATH_SEPARATOR); + } + if (name != null && !isHomeFolder(name)) { + builder.append(name); + + if (isFolder) { + builder.append(BlobStorageUtil.PATH_SEPARATOR); + } + } + + return URLEncoder.encode(builder.toString(), StandardCharsets.UTF_8); + } + + public String getAbsoluteFilePath() { + StringBuilder builder = new StringBuilder(); + if (parentFolders != null) { + builder.append(getParentPath()) + .append(BlobStorageUtil.PATH_SEPARATOR); + } + if (name != null && !isHomeFolder(name)) { + builder.append(name); + + if (isFolder) { + builder.append(BlobStorageUtil.PATH_SEPARATOR); + } + } + + return BlobStorageUtil.buildAbsoluteFilePath(type, bucketLocation, builder.toString()); + } + + public String getParentPath() { + return parentFolders == null ? null : String.join(BlobStorageUtil.PATH_SEPARATOR, parentFolders); + } + + public static ResourceDescription from(ResourceType type, String bucketName, String bucketLocation, String relativeFilePath) { + // in case empty path - treat it as a home folder + relativeFilePath = StringUtils.isBlank(relativeFilePath) ? BlobStorageUtil.PATH_SEPARATOR : relativeFilePath; + verify(bucketLocation.endsWith(BlobStorageUtil.PATH_SEPARATOR), "Bucket location must end with /"); + + String[] elements = relativeFilePath.split(BlobStorageUtil.PATH_SEPARATOR); + List parentFolders = null; + String name = "/"; + if (elements.length > 0) { + name = elements[elements.length - 1]; + } + if (elements.length > 1) { + String parentPath = relativeFilePath.substring(0, relativeFilePath.length() - name.length() - 1); + if (!parentPath.isEmpty() && !parentPath.equals(BlobStorageUtil.PATH_SEPARATOR)) { + parentFolders = List.of(parentPath.split(BlobStorageUtil.PATH_SEPARATOR)); + } + } + + return new ResourceDescription(type, name, parentFolders, relativeFilePath, bucketName, bucketLocation, BlobStorageUtil.isFolder(relativeFilePath)); + } + + private static boolean isHomeFolder(String path) { + return path.equals(BlobStorageUtil.PATH_SEPARATOR); + } + + private static void verify(boolean condition, String message) { + if (!condition) { + throw new IllegalArgumentException(message); + } + } +} diff --git a/src/main/java/com/epam/aidial/core/storage/ResourceType.java b/src/main/java/com/epam/aidial/core/storage/ResourceType.java new file mode 100644 index 000000000..9b8285b82 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/storage/ResourceType.java @@ -0,0 +1,15 @@ +package com.epam.aidial.core.storage; + +public enum ResourceType { + FILE("files"); + + private final String folder; + + ResourceType(String folder) { + this.folder = folder; + } + + public String getFolder() { + return folder; + } +} diff --git a/src/main/java/com/epam/aidial/core/util/Base58.java b/src/main/java/com/epam/aidial/core/util/Base58.java new file mode 100644 index 000000000..805b8b190 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/util/Base58.java @@ -0,0 +1,163 @@ +package com.epam.aidial.core.util; + +/* + * Copyright 2011 Google Inc. + * Copyright 2018 Andreas Schildbach + * + * From https://github.com/multiformats/java-multibase/blob/master/src/main/java/io/ipfs/multibase/Base58.java + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.math.BigInteger; +import java.util.Arrays; + +/** + * Base58 is a way to encode Bitcoin addresses (or arbitrary data) as alphanumeric strings. + * + *

Note that this is not the same base58 as used by Flickr, which you may find referenced around the Internet. + * + *

Satoshi explains: why base-58 instead of standard base-64 encoding? + *

    + *
  • Don't want 0OIl characters that look the same in some fonts and + * could be used to create visually identical looking account numbers.
    • + *
  • A string with non-alphanumeric characters is not as easily accepted as an account number.
    • + *
  • E-mail usually won't line-break if there's no punctuation to break at.
    • + *
  • Doubleclicking selects the whole number as one word if it's all alphanumeric.
    • + *
+ * + *

However, note that the encoding/decoding runs in O(n²) time, so it is not useful for large data. + * + *

The basic idea of the encoding is to treat the data bytes as a large number represented using + * base-256 digits, convert the number to be represented using base-58 digits, preserve the exact + * number of leading zeros (which are otherwise lost during the mathematical operations on the + * numbers), and finally represent the resulting base-58 digits as alphanumeric ASCII characters. + */ +public class Base58 { + public static final char[] ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); + private static final char ENCODED_ZERO = ALPHABET[0]; + private static final int[] INDEXES = new int[128]; + + static { + Arrays.fill(INDEXES, -1); + for (int i = 0; i < ALPHABET.length; i++) { + INDEXES[ALPHABET[i]] = i; + } + } + + /** + * Encodes the given bytes as a base58 string (no checksum is appended). + * + * @param input the bytes to encode + * @return the base58-encoded string + */ + public static String encode(byte[] input) { + if (input.length == 0) { + return ""; + } + // Count leading zeros. + int zeros = 0; + while (zeros < input.length && input[zeros] == 0) { + ++zeros; + } + // Convert base-256 digits to base-58 digits (plus conversion to ASCII characters) + input = Arrays.copyOf(input, input.length); // since we modify it in-place + char[] encoded = new char[input.length * 2]; // upper bound + int outputStart = encoded.length; + for (int inputStart = zeros; inputStart < input.length; ) { + encoded[--outputStart] = ALPHABET[divmod(input, inputStart, 256, 58)]; + if (input[inputStart] == 0) { + ++inputStart; // optimization - skip leading zeros + } + } + // Preserve exactly as many leading encoded zeros in output as there were leading zeros in input. + while (outputStart < encoded.length && encoded[outputStart] == ENCODED_ZERO) { + ++outputStart; + } + while (--zeros >= 0) { + encoded[--outputStart] = ENCODED_ZERO; + } + // Return encoded string (including encoded leading zeros). + return new String(encoded, outputStart, encoded.length - outputStart); + } + + /** + * Decodes the given base58 string into the original data bytes. + * + * @param input the base58-encoded string to decode + * @return the decoded data bytes + */ + public static byte[] decode(String input) { + if (input.length() == 0) { + return new byte[0]; + } + // Convert the base58-encoded ASCII chars to a base58 byte sequence (base58 digits). + byte[] input58 = new byte[input.length()]; + for (int i = 0; i < input.length(); ++i) { + char c = input.charAt(i); + int digit = c < 128 ? INDEXES[c] : -1; + if (digit < 0) { + throw new IllegalArgumentException(String.format("Invalid character in Base58: 0x%04x", (int) c)); + } + input58[i] = (byte) digit; + } + // Count leading zeros. + int zeros = 0; + while (zeros < input58.length && input58[zeros] == 0) { + ++zeros; + } + // Convert base-58 digits to base-256 digits. + byte[] decoded = new byte[input.length()]; + int outputStart = decoded.length; + for (int inputStart = zeros; inputStart < input58.length; ) { + decoded[--outputStart] = divmod(input58, inputStart, 58, 256); + if (input58[inputStart] == 0) { + ++inputStart; // optimization - skip leading zeros + } + } + // Ignore extra leading zeroes that were added during the calculation. + while (outputStart < decoded.length && decoded[outputStart] == 0) { + ++outputStart; + } + // Return decoded data (including original number of leading zeros). + return Arrays.copyOfRange(decoded, outputStart - zeros, decoded.length); + } + + public static BigInteger decodeToBigInteger(String input) { + return new BigInteger(1, decode(input)); + } + + /** + * Divides a number, represented as an array of bytes each containing a single digit + * in the specified base, by the given divisor. The given number is modified in-place + * to contain the quotient, and the return value is the remainder. + * + * @param number the number to divide + * @param firstDigit the index within the array of the first non-zero digit + * (this is used for optimization by skipping the leading zeros) + * @param base the base in which the number's digits are represented (up to 256) + * @param divisor the number to divide by (up to 256) + * @return the remainder of the division operation + */ + private static byte divmod(byte[] number, int firstDigit, int base, int divisor) { + // this is just long division which accounts for the base of the input digits + int remainder = 0; + for (int i = firstDigit; i < number.length; i++) { + int digit = (int) number[i] & 0xFF; + int temp = remainder * base + digit; + number[i] = (byte) (temp / divisor); + remainder = temp % divisor; + } + return (byte) remainder; + } +} diff --git a/src/test/java/com/epam/aidial/core/FileApiTest.java b/src/test/java/com/epam/aidial/core/FileApiTest.java index 05bd0e082..ace09b0eb 100644 --- a/src/test/java/com/epam/aidial/core/FileApiTest.java +++ b/src/test/java/com/epam/aidial/core/FileApiTest.java @@ -2,14 +2,14 @@ import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; +import com.epam.aidial.core.data.Bucket; import com.epam.aidial.core.data.FileMetadata; import com.epam.aidial.core.data.FolderMetadata; +import com.epam.aidial.core.util.ProxyUtil; import io.vertx.core.Future; import io.vertx.core.Promise; import io.vertx.core.Vertx; import io.vertx.core.buffer.Buffer; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; import io.vertx.ext.web.client.WebClient; import io.vertx.ext.web.codec.BodyCodec; import io.vertx.ext.web.multipart.MultipartForm; @@ -25,9 +25,9 @@ import org.junit.jupiter.api.extension.ExtendWith; import java.nio.file.Path; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertIterableEquals; import static org.junit.jupiter.api.Assertions.assertNull; @ExtendWith(VertxExtension.class) @@ -68,17 +68,34 @@ public static void destroy() { dial.stop(); } + @Test + public void testBucket(Vertx vertx, VertxTestContext context) { + WebClient client = WebClient.create(vertx); + client.get(serverPort, "localhost", "/v1/bucket") + .putHeader("Api-key", "proxyKey2") + .as(BodyCodec.json(Bucket.class)) + .send(context.succeeding(response -> { + context.verify(() -> { + assertEquals(200, response.statusCode()); + assertEquals(new Bucket("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt"), response.body()); + context.completeNow(); + }); + })); + } + @Test public void testEmptyFilesList(Vertx vertx, VertxTestContext context) { WebClient client = WebClient.create(vertx); - client.get(serverPort, "localhost", "/v1/files") + + FolderMetadata emptyBucketResponse = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "/", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2F", List.of()); + client.get(serverPort, "localhost", "/v1/files/metadata/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") .putHeader("Api-key", "proxyKey2") - .addQueryParam("purpose", "metadata") - .as(BodyCodec.jsonArray()) + .as(BodyCodec.json(FolderMetadata.class)) .send(context.succeeding(response -> { context.verify(() -> { assertEquals(200, response.statusCode()); - assertEquals(JsonArray.of(), response.body()); + assertEquals(emptyBucketResponse, response.body()); context.completeNow(); }); })); @@ -87,7 +104,7 @@ public void testEmptyFilesList(Vertx vertx, VertxTestContext context) { @Test public void testFileNotFound(Vertx vertx, VertxTestContext context) { WebClient client = WebClient.create(vertx); - client.get(serverPort, "localhost", "/v1/files/test_file.txt") + client.get(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.buffer()) .send(context.succeeding(response -> { @@ -99,24 +116,115 @@ public void testFileNotFound(Vertx vertx, VertxTestContext context) { })); } + @Test + public void testInvalidFileUploadUrl(Vertx vertx, VertxTestContext context) { + WebClient client = WebClient.create(vertx); + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") + .putHeader("Api-key", "proxyKey2") + .as(BodyCodec.string()) + .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), + context.succeeding(response -> { + context.verify(() -> { + assertEquals(400, response.statusCode()); + assertEquals("File name is missing", response.body()); + context.completeNow(); + }); + }) + ); + } + + @Test + public void testInvalidFileUploadUrl2(Vertx vertx, VertxTestContext context) { + WebClient client = WebClient.create(vertx); + client.put(serverPort, "localhost", "/v1/files/testbucket/") + .putHeader("Api-key", "proxyKey2") + .as(BodyCodec.string()) + .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), + context.succeeding(response -> { + context.verify(() -> { + assertEquals(403, response.statusCode()); + assertEquals("You don't have an access to the bucket testbucket", response.body()); + context.completeNow(); + }); + }) + ); + } + + @Test + public void testDownloadFromAnotherBucket(Vertx vertx, VertxTestContext context) { + Checkpoint checkpoint = context.checkpoint(2); + WebClient client = WebClient.create(vertx); + + FileMetadata expectedFileMetadata = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "file.txt", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffile.txt", 17, "text/custom"); + + Future.succeededFuture().compose((mapper) -> { + Promise promise = Promise.promise(); + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffile.txt") + .putHeader("Api-key", "proxyKey2") + .as(BodyCodec.json(FileMetadata.class)) + .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), + context.succeeding(response -> { + context.verify(() -> { + assertEquals(200, response.statusCode()); + assertEquals(expectedFileMetadata, response.body()); + checkpoint.flag(); + promise.complete(); + }); + }) + ); + return promise.future(); + }).andThen((mapper) -> { + client.get(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/file.txt") + .putHeader("Api-key", "proxyKey1") + .as(BodyCodec.string()) + .send(context.succeeding(response -> { + context.verify(() -> { + assertEquals(403, response.statusCode()); + checkpoint.flag(); + }); + })); + }); + } + + @Test + public void testFileUploadIntoAnotherBucket(Vertx vertx, VertxTestContext context) { + WebClient client = WebClient.create(vertx); + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/file.txt") + .putHeader("Api-key", "proxyKey1") + .as(BodyCodec.string()) + .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), + context.succeeding(response -> { + context.verify(() -> { + assertEquals(403, response.statusCode()); + context.completeNow(); + }); + }) + ); + } + @Test public void testFileUpload(Vertx vertx, VertxTestContext context) { Checkpoint checkpoint = context.checkpoint(3); WebClient client = WebClient.create(vertx); - FileMetadata expectedFileMetadata = new FileMetadata("file.txt", "/Keys/EPM-RTC-RAIL/files", 17, "text/custom"); + FolderMetadata emptyFolderResponse = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "/", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2F", List.of()); + FileMetadata expectedFileMetadata = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "file.txt", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffile.txt", 17, "text/custom"); + FolderMetadata expectedFolderMetadata = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "/", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2F", List.of(expectedFileMetadata)); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // verify no files - client.get(serverPort, "localhost", "/v1/files") + client.get(serverPort, "localhost", "/v1/files/metadata/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") .putHeader("Api-key", "proxyKey2") - .addQueryParam("purpose", "metadata") - .as(BodyCodec.jsonArray()) + .as(BodyCodec.json(FolderMetadata.class)) .send(context.succeeding(response -> { context.verify(() -> { assertEquals(200, response.statusCode()); - assertEquals(JsonArray.of(), response.body()); + assertEquals(emptyFolderResponse, response.body()); checkpoint.flag(); promise.complete(); }); @@ -126,7 +234,7 @@ public void testFileUpload(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // upload test file - client.post(serverPort, "localhost", "/v1/files") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffile.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -143,14 +251,13 @@ public void testFileUpload(Vertx vertx, VertxTestContext context) { return promise.future(); }).andThen((result) -> { // verify uploaded file can be listed - client.get(serverPort, "localhost", "/v1/files") + client.get(serverPort, "localhost", "/v1/files/metadata/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") .putHeader("Api-key", "proxyKey2") - .addQueryParam("purpose", "metadata") - .as(BodyCodec.jsonArray()) + .as(BodyCodec.string()) .send(context.succeeding(response -> { context.verify(() -> { assertEquals(200, response.statusCode()); - assertIterableEquals(JsonArray.of(JsonObject.mapFrom(expectedFileMetadata)), response.body()); + assertEquals(ProxyUtil.MAPPER.writeValueAsString(expectedFolderMetadata), response.body()); checkpoint.flag(); }); })); @@ -159,15 +266,16 @@ public void testFileUpload(Vertx vertx, VertxTestContext context) { @Test public void testFileDownload(Vertx vertx, VertxTestContext context) { - Checkpoint checkpoint = context.checkpoint(3); + Checkpoint checkpoint = context.checkpoint(2); WebClient client = WebClient.create(vertx); - FileMetadata expectedFileMetadata = new FileMetadata("file.txt", "/Keys/EPM-RTC-RAIL/files/folder1", 17, "text/plain"); + FileMetadata expectedFileMetadata = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "file.txt", "folder1", "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffolder1%2Ffile.txt", 17, "text/plain"); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // upload test file - client.post(serverPort, "localhost", "/v1/files/folder1") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffolder1%2Ffile.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT), @@ -181,27 +289,9 @@ public void testFileDownload(Vertx vertx, VertxTestContext context) { }) ); - return promise.future(); - }).compose((mapper) -> { - Promise promise = Promise.promise(); - // download by relative path - client.get(serverPort, "localhost", "/v1/files/folder1/file.txt") - .putHeader("Api-key", "proxyKey2") - .addQueryParam("path", "relative") - .as(BodyCodec.string()) - .send(context.succeeding(response -> { - context.verify(() -> { - assertEquals(200, response.statusCode()); - assertEquals(TEST_FILE_CONTENT, response.body()); - checkpoint.flag(); - promise.complete(); - }); - })); - return promise.future(); }).andThen((result) -> { - // download by absolute path - client.get(serverPort, "localhost", "/v1/files/Keys/EPM-RTC-RAIL/files/folder1/file.txt") + client.get(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/folder1/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .send(context.succeeding(response -> { @@ -219,21 +309,29 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { Checkpoint checkpoint = context.checkpoint(4); WebClient client = WebClient.create(vertx); - FileMetadata expectedFileMetadata1 = new FileMetadata("file.txt", "/Keys/EPM-RTC-RAIL/files", 17, "text/custom"); - FileMetadata expectedFileMetadata2 = new FileMetadata("file.txt", "/Keys/EPM-RTC-RAIL/files/folder1", 17, "text/custom"); - FolderMetadata expectedFolderMetadata = new FolderMetadata("folder1", "/Keys/EPM-RTC-RAIL/files"); + FolderMetadata emptyFolderResponse = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "/", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2F", List.of()); + + FileMetadata expectedFileMetadata1 = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "file.txt", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffile.txt", 17, "text/custom"); + FileMetadata expectedFileMetadata2 = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "file.txt", "folder1", "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffolder1%2Ffile.txt", 17, "text/custom"); + FolderMetadata expectedFolder1Metadata = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "folder1", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ffolder1%2F"); + FolderMetadata expectedRootFolderMetadata = new FolderMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "/", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2F", + List.of(expectedFileMetadata1, expectedFolder1Metadata)); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // verify no files - client.get(serverPort, "localhost", "/v1/files") + client.get(serverPort, "localhost", "/v1/files/metadata/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") .putHeader("Api-key", "proxyKey2") - .addQueryParam("purpose", "metadata") - .as(BodyCodec.jsonArray()) + .as(BodyCodec.json(FolderMetadata.class)) .send(context.succeeding(response -> { context.verify(() -> { assertEquals(200, response.statusCode()); - assertEquals(JsonArray.of(), response.body()); + assertEquals(emptyFolderResponse, response.body()); checkpoint.flag(); promise.complete(); }); @@ -243,7 +341,7 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // upload test file1 - client.post(serverPort, "localhost", "/v1/files") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -261,7 +359,7 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // upload test file2 - client.post(serverPort, "localhost", "/v1/files/folder1") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/folder1/file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("file.txt", TEST_FILE_CONTENT, "text/custom"), @@ -278,17 +376,13 @@ public void testListFileWithFolder(Vertx vertx, VertxTestContext context) { return promise.future(); }).andThen((result) -> { // verify uploaded files can be listed - client.get(serverPort, "localhost", "/v1/files") + client.get(serverPort, "localhost", "/v1/files/metadata/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/") .putHeader("Api-key", "proxyKey2") - .addQueryParam("purpose", "metadata") - .as(BodyCodec.jsonArray()) + .as(BodyCodec.string()) .send(context.succeeding(response -> { context.verify(() -> { assertEquals(200, response.statusCode()); - assertIterableEquals( - JsonArray.of(JsonObject.mapFrom(expectedFileMetadata1), - JsonObject.mapFrom(expectedFolderMetadata)), - response.body()); + assertEquals(ProxyUtil.MAPPER.writeValueAsString(expectedRootFolderMetadata), response.body()); checkpoint.flag(); }); })); @@ -300,12 +394,13 @@ public void testFileDelete(Vertx vertx, VertxTestContext context) { Checkpoint checkpoint = context.checkpoint(3); WebClient client = WebClient.create(vertx); - FileMetadata expectedFileMetadata = new FileMetadata("test_file.txt", "/Keys/EPM-RTC-RAIL/files", 17, "text/plain"); + FileMetadata expectedFileMetadata = new FileMetadata("7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt", + "test_file.txt", null, "7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt%2Ftest_file.txt", 17, "text/plain"); Future.succeededFuture().compose((mapper) -> { Promise promise = Promise.promise(); // upload test file - client.post(serverPort, "localhost", "/v1/files") + client.put(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/test_file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.json(FileMetadata.class)) .sendMultipartForm(generateMultipartForm("test_file.txt", TEST_FILE_CONTENT), @@ -323,7 +418,7 @@ public void testFileDelete(Vertx vertx, VertxTestContext context) { }).compose((mapper) -> { Promise promise = Promise.promise(); // delete file - client.delete(serverPort, "localhost", "/v1/files/test_file.txt") + client.delete(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/test_file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .send(context.succeeding(response -> { @@ -337,7 +432,7 @@ public void testFileDelete(Vertx vertx, VertxTestContext context) { return promise.future(); }).andThen((mapper) -> { // try to download deleted file - client.get(serverPort, "localhost", "/v1/files/test_file.txt") + client.get(serverPort, "localhost", "/v1/files/7G9WZNcoY26Vy9D7bEgbv6zqbJGfyDp9KZyEbJR4XMZt/test_file.txt") .putHeader("Api-key", "proxyKey2") .as(BodyCodec.string()) .send(context.succeeding(response -> { diff --git a/src/test/java/com/epam/aidial/core/controller/ControllerSelectorTest.java b/src/test/java/com/epam/aidial/core/controller/ControllerSelectorTest.java index b87f3ab0e..1a51d008b 100644 --- a/src/test/java/com/epam/aidial/core/controller/ControllerSelectorTest.java +++ b/src/test/java/com/epam/aidial/core/controller/ControllerSelectorTest.java @@ -2,7 +2,6 @@ import com.epam.aidial.core.Proxy; import com.epam.aidial.core.ProxyContext; -import io.vertx.core.MultiMap; import io.vertx.core.http.HttpMethod; import io.vertx.core.http.HttpServerRequest; import org.junit.jupiter.api.BeforeEach; @@ -20,7 +19,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -181,37 +179,36 @@ public void testSelectGetApplicationsController() { @Test public void testSelectListMetadataFileController() { - when(request.path()).thenReturn("/v1/files/folder1/file1?purpose=metadata"); + when(request.path()).thenReturn("/v1/files/metadata/bucket/file1"); when(request.method()).thenReturn(HttpMethod.GET); - MultiMap params = mock(MultiMap.class); - when(request.params()).thenReturn(params); - when(params.get("purpose")).thenReturn("metadata"); Controller controller = ControllerSelector.select(proxy, context); assertNotNull(controller); SerializedLambda lambda = getSerializedLambda(controller); assertNotNull(lambda); - assertEquals(2, lambda.getCapturedArgCount()); + assertEquals(3, lambda.getCapturedArgCount()); Object arg1 = lambda.getCapturedArg(0); Object arg2 = lambda.getCapturedArg(1); + Object arg3 = lambda.getCapturedArg(2); assertInstanceOf(FileMetadataController.class, arg1); - assertEquals("/folder1/file1?purpose=metadata", arg2); + assertEquals("bucket", arg2); + assertEquals("file1", arg3); } @Test public void testSelectDownloadFileController() { - when(request.path()).thenReturn("/v1/files/folder1/file1"); + when(request.path()).thenReturn("/v1/files/bucket/folder1/file1.txt"); when(request.method()).thenReturn(HttpMethod.GET); - MultiMap params = mock(MultiMap.class); - when(request.params()).thenReturn(params); Controller controller = ControllerSelector.select(proxy, context); assertNotNull(controller); SerializedLambda lambda = getSerializedLambda(controller); assertNotNull(lambda); - assertEquals(2, lambda.getCapturedArgCount()); + assertEquals(3, lambda.getCapturedArgCount()); Object arg1 = lambda.getCapturedArg(0); Object arg2 = lambda.getCapturedArg(1); + Object arg3 = lambda.getCapturedArg(2); assertInstanceOf(DownloadFileController.class, arg1); - assertEquals("/folder1/file1", arg2); + assertEquals("bucket", arg2); + assertEquals("folder1/file1.txt", arg3); } @Test @@ -233,32 +230,36 @@ public void testSelectPostDeploymentController() { @Test public void testSelectUploadFileController() { - when(request.path()).thenReturn("/v1/files/folder1/file1"); - when(request.method()).thenReturn(HttpMethod.POST); + when(request.path()).thenReturn("/v1/files/bucket/folder1/file1.txt"); + when(request.method()).thenReturn(HttpMethod.PUT); Controller controller = ControllerSelector.select(proxy, context); assertNotNull(controller); SerializedLambda lambda = getSerializedLambda(controller); assertNotNull(lambda); - assertEquals(2, lambda.getCapturedArgCount()); + assertEquals(3, lambda.getCapturedArgCount()); Object arg1 = lambda.getCapturedArg(0); Object arg2 = lambda.getCapturedArg(1); + Object arg3 = lambda.getCapturedArg(2); assertInstanceOf(UploadFileController.class, arg1); - assertEquals("/folder1/file1", arg2); + assertEquals("bucket", arg2); + assertEquals("folder1/file1.txt", arg3); } @Test public void testSelectDeleteFileController() { - when(request.path()).thenReturn("/v1/files/folder1/file1"); + when(request.path()).thenReturn("/v1/files/bucket/folder1/file1.txt"); when(request.method()).thenReturn(HttpMethod.DELETE); Controller controller = ControllerSelector.select(proxy, context); assertNotNull(controller); SerializedLambda lambda = getSerializedLambda(controller); assertNotNull(lambda); - assertEquals(2, lambda.getCapturedArgCount()); + assertEquals(3, lambda.getCapturedArgCount()); Object arg1 = lambda.getCapturedArg(0); Object arg2 = lambda.getCapturedArg(1); + Object arg3 = lambda.getCapturedArg(2); assertInstanceOf(DeleteFileController.class, arg1); - assertEquals("/folder1/file1", arg2); + assertEquals("bucket", arg2); + assertEquals("folder1/file1.txt", arg3); } @Test diff --git a/src/test/java/com/epam/aidial/core/storage/BlobStorageUtilTest.java b/src/test/java/com/epam/aidial/core/storage/BlobStorageUtilTest.java deleted file mode 100644 index eccdf6818..000000000 --- a/src/test/java/com/epam/aidial/core/storage/BlobStorageUtilTest.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.epam.aidial.core.storage; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import static com.epam.aidial.core.storage.BlobStorageUtil.normalizePathForQuery; -import static com.epam.aidial.core.storage.BlobStorageUtil.removeLeadingAndTrailingPathSeparators; -import static com.epam.aidial.core.storage.BlobStorageUtil.removeLeadingPathSeparator; -import static com.epam.aidial.core.storage.BlobStorageUtil.removeTrailingPathSeparator; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - -public class BlobStorageUtilTest { - - @Test - public void testNormalizePathForQuery() { - assertEquals("Users/User/files/", normalizePathForQuery("/Users/User/files")); - assertEquals("Users/User/files/", normalizePathForQuery("Users/User/files")); - assertEquals("folder/", normalizePathForQuery("folder")); - assertEquals("/", normalizePathForQuery("/")); - assertNull(normalizePathForQuery("")); - } - - @Test - public void testRemoveLeadingPathSeparator() { - assertEquals("Users/User/files/", removeLeadingPathSeparator("/Users/User/files/")); - assertEquals("Users/User/files", removeLeadingPathSeparator("Users/User/files")); - assertEquals("folder/", removeLeadingPathSeparator("/folder/")); - assertEquals("", removeLeadingPathSeparator("/")); - assertNull(removeLeadingPathSeparator("")); - } - - @Test - public void testRemoveTrailingPathSeparator() { - assertEquals("/Users/User/files", removeTrailingPathSeparator("/Users/User/files/")); - assertEquals("Users/User/files", removeTrailingPathSeparator("Users/User/files")); - assertEquals("/folder", removeTrailingPathSeparator("/folder/")); - assertEquals("", removeTrailingPathSeparator("/")); - assertNull(removeTrailingPathSeparator("")); - } - - @Test - public void testRemoveLeadingAndTrailingPathSeparators() { - assertEquals("Users/User/files", removeLeadingAndTrailingPathSeparators("/Users/User/files/")); - assertEquals("Users/User/files", removeLeadingAndTrailingPathSeparators("Users/User/files")); - assertEquals("folder", removeLeadingAndTrailingPathSeparators("/folder/")); - assertEquals("", removeLeadingAndTrailingPathSeparators(null)); - assertEquals("", removeLeadingAndTrailingPathSeparators("/")); - assertEquals("", removeLeadingAndTrailingPathSeparators("")); - } -} diff --git a/src/test/java/com/epam/aidial/core/storage/ResourceDescriptionTest.java b/src/test/java/com/epam/aidial/core/storage/ResourceDescriptionTest.java new file mode 100644 index 000000000..9763a3b68 --- /dev/null +++ b/src/test/java/com/epam/aidial/core/storage/ResourceDescriptionTest.java @@ -0,0 +1,142 @@ +package com.epam.aidial.core.storage; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ResourceDescriptionTest { + + @Test + public void testHomeFolderDescription() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "aes/bucket/name", "buckets/location/", "/"); + assertEquals("/", resource.getName()); + assertEquals("aes/bucket/name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("aes%2Fbucket%2Fname%2F", resource.getUrl()); + assertEquals("buckets/location/files/", resource.getAbsoluteFilePath()); + assertEquals("/", resource.getRelativePath()); + assertTrue(resource.isFolder()); + assertNull(resource.getParentPath()); + assertNull(resource.getParentFolders()); + } + + @Test + public void testUserFolderDescription() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "folder1/"); + assertEquals("folder1", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name%2Ffolder1%2F", resource.getUrl()); + assertEquals("buckets/location/files/folder1/", resource.getAbsoluteFilePath()); + assertEquals("folder1/", resource.getRelativePath()); + assertTrue(resource.isFolder()); + assertNull(resource.getParentPath()); + assertNull(resource.getParentFolders()); + } + + @Test + public void testUserFolderDescription2() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "folder1/folder2/"); + assertEquals("folder2", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name%2Ffolder1%2Ffolder2%2F", resource.getUrl()); + assertEquals("buckets/location/files/folder1/folder2/", resource.getAbsoluteFilePath()); + assertEquals("folder1/folder2/", resource.getRelativePath()); + assertTrue(resource.isFolder()); + assertEquals("folder1", resource.getParentPath()); + assertIterableEquals(List.of("folder1"), resource.getParentFolders()); + } + + @Test + public void testUserFolderDescription3() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "folder1/folder2/folder3/"); + assertEquals("folder3", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name%2Ffolder1%2Ffolder2%2Ffolder3%2F", resource.getUrl()); + assertEquals("buckets/location/files/folder1/folder2/folder3/", resource.getAbsoluteFilePath()); + assertEquals("folder1/folder2/folder3/", resource.getRelativePath()); + assertTrue(resource.isFolder()); + assertEquals("folder1/folder2", resource.getParentPath()); + assertIterableEquals(List.of("folder1", "folder2"), resource.getParentFolders()); + } + + @Test + public void testFileDescription1() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "file.txt"); + assertEquals("file.txt", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name%2Ffile.txt", resource.getUrl()); + assertEquals("buckets/location/files/file.txt", resource.getAbsoluteFilePath()); + assertEquals("file.txt", resource.getRelativePath()); + assertFalse(resource.isFolder()); + assertNull(resource.getParentPath()); + assertNull(resource.getParentFolders()); + } + + @Test + public void testFileDescription2() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "folder1/file.txt"); + assertEquals("file.txt", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name%2Ffolder1%2Ffile.txt", resource.getUrl()); + assertEquals("buckets/location/files/folder1/file.txt", resource.getAbsoluteFilePath()); + assertEquals("folder1/file.txt", resource.getRelativePath()); + assertFalse(resource.isFolder()); + assertEquals("folder1", resource.getParentPath()); + assertIterableEquals(List.of("folder1"), resource.getParentFolders()); + } + + @Test + public void testFileDescription3() { + ResourceDescription resource = ResourceDescription.from(ResourceType.FILE, "test-bucket-name", "buckets/location/", "folder1/folder2/file.txt"); + assertEquals("file.txt", resource.getName()); + assertEquals("test-bucket-name", resource.getBucketName()); + assertEquals("buckets/location/", resource.getBucketLocation()); + assertEquals(ResourceType.FILE, resource.getType()); + assertEquals("test-bucket-name%2Ffolder1%2Ffolder2%2Ffile.txt", resource.getUrl()); + assertEquals("buckets/location/files/folder1/folder2/file.txt", resource.getAbsoluteFilePath()); + assertEquals("folder1/folder2/file.txt", resource.getRelativePath()); + assertFalse(resource.isFolder()); + assertEquals("folder1/folder2", resource.getParentPath()); + assertIterableEquals(List.of("folder1", "folder2"), resource.getParentFolders()); + } + + @Test + public void testInvalidBucketLocation() { + assertThrows(IllegalArgumentException.class, + () -> ResourceDescription.from(ResourceType.FILE, "bucket-name", "buckets/location", "file.txt")); + } + + @Test + public void testEmptyRelativePath() { + assertEquals( + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", "/"), + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", "") + ); + assertEquals( + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", "/"), + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", null) + ); + assertEquals( + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", "/"), + ResourceDescription.from(ResourceType.FILE, "bucket", "location/", " ") + ); + } +} diff --git a/src/test/resources/aidial.settings.json b/src/test/resources/aidial.settings.json index 681cdecd6..c9b1cf14e 100644 --- a/src/test/resources/aidial.settings.json +++ b/src/test/resources/aidial.settings.json @@ -44,5 +44,9 @@ "rolePath": "roles", "issuerPattern": "issuer" } + }, + "encryption": { + "password": "password", + "salt": "salt" } }