From fb8afbdb0a4f193d70fba8b762402b92a1d66568 Mon Sep 17 00:00:00 2001 From: Maxim Date: Thu, 4 Apr 2024 13:32:16 +0200 Subject: [PATCH] feat: implement published resource listing (#299)(#294) --- .../core/controller/ControllerSelector.java | 7 +++ .../controller/PublicationController.java | 16 ++++++ .../data/ListPublishedResourcesRequest.java | 10 ++++ .../core/service/PublicationService.java | 43 ++++++++++++++- .../epam/aidial/core/PublicationApiTest.java | 55 +++++++++++++++++++ 5 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/epam/aidial/core/data/ListPublishedResourcesRequest.java diff --git a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java index fa2696c32..68f390286 100644 --- a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java +++ b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java @@ -51,6 +51,7 @@ public class ControllerSelector { private static final Pattern INVITATIONS = Pattern.compile("^/v1/invitations$"); private static final Pattern INVITATION = Pattern.compile("^/v1/invitations/([a-zA-Z0-9]+)$"); private static final Pattern PUBLICATIONS = Pattern.compile("^/v1/ops/publication/(list|get|create|delete|approve|reject)$"); + private static final Pattern PUBLISHED_RESOURCES = Pattern.compile("^/v1/ops/publication/resource/list$"); private static final Pattern PUBLICATION_RULES = Pattern.compile("^/v1/ops/publication/rule/list$"); private static final Pattern RESOURCE_OPERATIONS = Pattern.compile("^/v1/ops/resource/(move)$"); @@ -293,6 +294,12 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return controller::move; } + match = match(PUBLISHED_RESOURCES, path); + if (match != null) { + PublicationController controller = new PublicationController(proxy, context); + return controller::listPublishedResources; + } + return null; } diff --git a/src/main/java/com/epam/aidial/core/controller/PublicationController.java b/src/main/java/com/epam/aidial/core/controller/PublicationController.java index 73a7c5550..5d3f816ad 100644 --- a/src/main/java/com/epam/aidial/core/controller/PublicationController.java +++ b/src/main/java/com/epam/aidial/core/controller/PublicationController.java @@ -2,6 +2,7 @@ import com.epam.aidial.core.Proxy; import com.epam.aidial.core.ProxyContext; +import com.epam.aidial.core.data.ListPublishedResourcesRequest; import com.epam.aidial.core.data.Publication; import com.epam.aidial.core.data.Publications; import com.epam.aidial.core.data.ResourceLink; @@ -149,6 +150,21 @@ public Future listRules() { return Future.succeededFuture(); } + public Future listPublishedResources() { + context.getRequest() + .body() + .compose(body -> { + ListPublishedResourcesRequest request = ProxyUtil.convertToObject(body, ListPublishedResourcesRequest.class); + String bucketLocation = BlobStorageUtil.buildInitiatorBucket(context); + String bucket = encryptService.encrypt(bucketLocation); + return vertx.executeBlocking(() -> publicationService.listPublishedResources(request, bucket, bucketLocation)); + }) + .onSuccess(metadata -> context.respond(HttpStatus.OK, metadata)) + .onFailure(error -> respondError("Can't list published resources", error)); + + return Future.succeededFuture(); + } + private void respondError(String message, Throwable error) { HttpStatus status = HttpStatus.INTERNAL_SERVER_ERROR; String body = null; diff --git a/src/main/java/com/epam/aidial/core/data/ListPublishedResourcesRequest.java b/src/main/java/com/epam/aidial/core/data/ListPublishedResourcesRequest.java new file mode 100644 index 000000000..fc2cfc454 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/data/ListPublishedResourcesRequest.java @@ -0,0 +1,10 @@ +package com.epam.aidial.core.data; + +import lombok.Data; + +import java.util.Set; + +@Data +public class ListPublishedResourcesRequest { + Set resourceTypes; +} diff --git a/src/main/java/com/epam/aidial/core/service/PublicationService.java b/src/main/java/com/epam/aidial/core/service/PublicationService.java index 2c7a8e628..6643ae2d1 100644 --- a/src/main/java/com/epam/aidial/core/service/PublicationService.java +++ b/src/main/java/com/epam/aidial/core/service/PublicationService.java @@ -1,6 +1,7 @@ package com.epam.aidial.core.service; import com.epam.aidial.core.ProxyContext; +import com.epam.aidial.core.data.ListPublishedResourcesRequest; import com.epam.aidial.core.data.MetadataBase; import com.epam.aidial.core.data.Publication; import com.epam.aidial.core.data.ResourceFolderMetadata; @@ -31,6 +32,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; import java.util.function.Supplier; +import java.util.stream.Collectors; import javax.annotation.Nullable; import static com.epam.aidial.core.storage.BlobStorageUtil.PATH_SEPARATOR; @@ -145,6 +147,39 @@ public Collection listPublications(ResourceDescription resource) { return publications.values(); } + public Collection listPublishedResources(ListPublishedResourcesRequest request, String bucket, String location) { + ResourceDescription publicationResource = publications(bucket, location); + Map publications = decodePublications(resources.getResource(publicationResource)); + + // get approved publications only + List approvedPublications = publications.values() + .stream() + .filter(publication -> Publication.Status.APPROVED.equals(publication.getStatus())) + .toList(); + + Set resourceSet = approvedPublications.stream() + .flatMap(publication -> publication.getResources().stream()) + .collect(Collectors.toSet()); + Set requestedResourceTypes = request.getResourceTypes(); + + Set metadata = new HashSet<>(); + for (Publication.Resource resource : resourceSet) { + ResourceDescription resourceDescription = ResourceDescription.fromPrivateUrl(resource.getSourceUrl(), encryption); + // check if published resource match requested criteria + if (!requestedResourceTypes.contains(resourceDescription.getType())) { + continue; + } + + if (resourceDescription.isFolder()) { + metadata.add(new ResourceFolderMetadata(resourceDescription)); + } else { + metadata.add(new ResourceItemMetadata(resourceDescription)); + } + } + + return metadata; + } + public Publication getPublication(ResourceDescription resource) { if (resource.getType() != ResourceType.PUBLICATION || resource.isPublic() || resource.isFolder() || resource.getParentPath() != null) { throw new IllegalArgumentException("Bad publication url: " + resource.getUrl()); @@ -572,10 +607,12 @@ private static Publication newMetadata(Publication publication) { } private static ResourceDescription publications(ResourceDescription resource) { + return publications(resource.getBucketName(), resource.getBucketLocation()); + } + + private static ResourceDescription publications(String bucket, String location) { return ResourceDescription.fromDecoded(ResourceType.PUBLICATION, - resource.getBucketName(), - resource.getBucketLocation(), - PUBLICATIONS_NAME); + bucket, location, PUBLICATIONS_NAME); } private static Map decodePublications(String json) { diff --git a/src/test/java/com/epam/aidial/core/PublicationApiTest.java b/src/test/java/com/epam/aidial/core/PublicationApiTest.java index 7f3fdb47d..4499f600a 100644 --- a/src/test/java/com/epam/aidial/core/PublicationApiTest.java +++ b/src/test/java/com/epam/aidial/core/PublicationApiTest.java @@ -603,4 +603,59 @@ void listRules() { """); verify(response, 403); } + + @Test + void testPublishedResourceList() { + // verify no published resource + Response response = operationRequest("/v1/ops/publication/resource/list", """ + {"resourceTypes": ["CONVERSATION"]} + """); + verify(response, 200, "[]"); + + response = resourceRequest(HttpMethod.PUT, "/my/folder/conversation", CONVERSATION_BODY_1); + verify(response, 200); + + // create publication request + response = operationRequest("/v1/ops/publication/create", PUBLICATION_REQUEST.formatted(bucket, bucket)); + verify(response, 200); + + // verify admin can view publication request + response = operationRequest("/v1/ops/publication/list", """ + {"url": "publications/public/"} + """, "authorization", "admin"); + verifyJson(response, 200, """ + { + "publications" : [ { + "url" : "publications/3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/0123", + "targetUrl" : "public/folder/", + "status" : "PENDING", + "createdAt" : 0 + } ] + } + """); + + // verify no published resources (due to PENDING publication request) + response = operationRequest("/v1/ops/publication/resource/list", """ + {"resourceTypes": ["CONVERSATION"]} + """); + verify(response, 200, "[]"); + + response = operationRequest("/v1/ops/publication/approve", PUBLICATION_URL, "authorization", "admin"); + verify(response, 200); + + // verify published resource can be listed + response = operationRequest("/v1/ops/publication/resource/list", """ + {"resourceTypes": ["CONVERSATION"]} + """); + verifyJson(response, 200, """ + [ { + "name" : "conversation", + "parentPath" : "my/folder", + "bucket" : "3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST", + "url" : "conversations/3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/my/folder/conversation", + "nodeType" : "ITEM", + "resourceType" : "CONVERSATION" + } ] + """); + } } \ No newline at end of file