From f2a4e1707783ea4a34a2ec77beb720832356f8a3 Mon Sep 17 00:00:00 2001 From: Artsiom Korzun Date: Thu, 14 Mar 2024 12:35:17 +0100 Subject: [PATCH 1/2] cache public rules --- .../core/service/PublicationService.java | 26 ++++++++++++-- .../aidial/core/service/ResourceService.java | 36 +++++++++++++++---- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/epam/aidial/core/service/PublicationService.java b/src/main/java/com/epam/aidial/core/service/PublicationService.java index 438a189da..f1b78ee54 100644 --- a/src/main/java/com/epam/aidial/core/service/PublicationService.java +++ b/src/main/java/com/epam/aidial/core/service/PublicationService.java @@ -4,6 +4,7 @@ import com.epam.aidial.core.data.MetadataBase; import com.epam.aidial.core.data.Publication; import com.epam.aidial.core.data.ResourceFolderMetadata; +import com.epam.aidial.core.data.ResourceItemMetadata; import com.epam.aidial.core.data.ResourceType; import com.epam.aidial.core.data.ResourceUrl; import com.epam.aidial.core.data.Rule; @@ -17,6 +18,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.mutable.MutableObject; +import org.apache.commons.lang3.tuple.Pair; import java.util.Collection; import java.util.HashMap; @@ -25,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; import java.util.function.Supplier; import javax.annotation.Nullable; @@ -53,6 +56,7 @@ public class PublicationService { private static final Set ALLOWED_RESOURCES = Set.of(ResourceType.FILE, ResourceType.CONVERSATION, ResourceType.PROMPT); + private final AtomicReference>>> cachedRules = new AtomicReference<>(); private final EncryptionService encryption; private final ResourceService resources; private final BlobStorage files; @@ -78,7 +82,7 @@ public boolean hasPublicAccess(ProxyContext context, ResourceDescription resourc return false; } - Map> rules = decodeRules(resources.getResource(PUBLIC_RULES)); + Map> rules = getCachedRules(); Map cache = new HashMap<>(); return evaluate(context, resource, rules, cache); @@ -89,7 +93,7 @@ public void filterForbidden(ProxyContext context, ResourceDescription folder, Re return; } - Map> rules = decodeRules(resources.getResource(PUBLIC_RULES)); + Map> rules = getCachedRules(); Map cache = new HashMap<>(); cache.put(ruleUrl(folder), true); @@ -473,6 +477,24 @@ private String encodeReviewBucket(ResourceDescription bucket, String id) { return encryption.encrypt(path); } + private Map> getCachedRules() { + ResourceItemMetadata meta = resources.getResourceMetadata(PUBLIC_RULES); + long key = (meta == null) ? Long.MIN_VALUE : meta.getUpdatedAt(); + Pair>> current = cachedRules.get(); + + if (current == null || current.getKey() != key) { + Pair resource = resources.getResourceWithMetadata(PUBLIC_RULES); + Pair>> next = (resource == null) + ? Pair.of(Long.MIN_VALUE, decodeRules(null)) + : Pair.of(resource.getKey().getUpdatedAt(), decodeRules(resource.getValue())); + + cachedRules.compareAndSet(current, next); + current = next; + } + + return current.getValue(); + } + private static boolean evaluate(ProxyContext context, ResourceDescription resource, Map> rules, diff --git a/src/main/java/com/epam/aidial/core/service/ResourceService.java b/src/main/java/com/epam/aidial/core/service/ResourceService.java index c548e361c..3861b6c8b 100644 --- a/src/main/java/com/epam/aidial/core/service/ResourceService.java +++ b/src/main/java/com/epam/aidial/core/service/ResourceService.java @@ -12,6 +12,7 @@ import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.PageSet; @@ -114,7 +115,7 @@ public void close() { public MetadataBase getMetadata(ResourceDescription descriptor, String token, int limit, boolean recursive) { return descriptor.isFolder() ? getFolderMetadata(descriptor, token, limit, recursive) - : getItemMetadata(descriptor); + : getResourceMetadata(descriptor); } private ResourceFolderMetadata getFolderMetadata(ResourceDescription descriptor, String token, int limit, boolean recursive) { @@ -156,7 +157,11 @@ private ResourceFolderMetadata getFolderMetadata(ResourceDescription descriptor, return new ResourceFolderMetadata(descriptor, resources).setNextToken(set.getNextMarker()); } - private ResourceItemMetadata getItemMetadata(ResourceDescription descriptor) { + public ResourceItemMetadata getResourceMetadata(ResourceDescription descriptor) { + if (descriptor.isFolder()) { + throw new IllegalArgumentException("Resource folder: " + descriptor.getUrl()); + } + String redisKey = redisKey(descriptor); String blobKey = blobKey(descriptor); Result result = redisGet(redisKey, false); @@ -187,12 +192,12 @@ public boolean hasResource(ResourceDescription descriptor) { } @Nullable - public String getResource(ResourceDescription descriptor) { - return getResource(descriptor, true); + public Pair getResourceWithMetadata(ResourceDescription descriptor) { + return getResourceWithMetadata(descriptor, true); } @Nullable - public String getResource(ResourceDescription descriptor, boolean lock) { + public Pair getResourceWithMetadata(ResourceDescription descriptor, boolean lock) { String redisKey = redisKey(descriptor); Result result = redisGet(redisKey, true); @@ -208,7 +213,26 @@ public String getResource(ResourceDescription descriptor, boolean lock) { } } - return result.exists ? result.body : null; + if (result.exists) { + ResourceItemMetadata metadata = new ResourceItemMetadata(descriptor) + .setCreatedAt(result.createdAt) + .setUpdatedAt(result.updatedAt); + + return Pair.of(metadata, result.body); + } + + return null; + } + + @Nullable + public String getResource(ResourceDescription descriptor) { + return getResource(descriptor, true); + } + + @Nullable + public String getResource(ResourceDescription descriptor, boolean lock) { + Pair result = getResourceWithMetadata(descriptor, lock); + return (result == null) ? null : result.getRight(); } public ResourceItemMetadata putResource(ResourceDescription descriptor, String body, boolean overwrite) { From c468b49e62b527a9357984e3e2ff278f5ba8121e Mon Sep 17 00:00:00 2001 From: Artsiom Korzun Date: Thu, 14 Mar 2024 14:16:11 +0100 Subject: [PATCH 2/2] add api for listing rules --- .../core/controller/ControllerSelector.java | 9 +- .../controller/PublicationController.java | 41 +++++++++ .../java/com/epam/aidial/core/data/Rules.java | 7 ++ .../core/service/PublicationService.java | 37 +++++--- .../epam/aidial/core/PublicationApiTest.java | 85 +++++++++++++++++++ 5 files changed, 167 insertions(+), 12 deletions(-) create mode 100644 src/main/java/com/epam/aidial/core/data/Rules.java diff --git a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java index 74cba54c6..c04700034 100644 --- a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java +++ b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java @@ -49,6 +49,7 @@ public class ControllerSelector { private static final Pattern INVITATIONS = Pattern.compile("^/v1/invitations$"); private static final Pattern INVITATION = Pattern.compile("^/v1/invitations/([a-zA-Z0-9]+)$"); private static final Pattern PUBLICATIONS = Pattern.compile("^/v1/ops/publications/(list|get|create|delete|approve|reject)$"); + private static final Pattern PUBLICATION_RULES = Pattern.compile("^/v1/ops/publications/rules/list$"); private static final Pattern RESOURCE_OPERATIONS = Pattern.compile("^/v1/ops/resources/(move)$"); @@ -261,11 +262,17 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p case "create" -> controller::createPublication; case "delete" -> controller::deletePublication; case "approve" -> controller::approvePublication; - case "reject" -> controller:: rejectPublication; + case "reject" -> controller::rejectPublication; default -> null; }; } + match = match(PUBLICATION_RULES, path); + if (match != null) { + PublicationController controller = new PublicationController(proxy, context); + return controller::listRules; + } + match = match(RESOURCE_OPERATIONS, path); if (match != null) { ResourceOperationController controller = new ResourceOperationController(proxy, context); diff --git a/src/main/java/com/epam/aidial/core/controller/PublicationController.java b/src/main/java/com/epam/aidial/core/controller/PublicationController.java index 7907e19b2..73a7c5550 100644 --- a/src/main/java/com/epam/aidial/core/controller/PublicationController.java +++ b/src/main/java/com/epam/aidial/core/controller/PublicationController.java @@ -6,6 +6,7 @@ import com.epam.aidial.core.data.Publications; import com.epam.aidial.core.data.ResourceLink; import com.epam.aidial.core.data.ResourceType; +import com.epam.aidial.core.data.Rules; import com.epam.aidial.core.security.AccessService; import com.epam.aidial.core.security.EncryptionService; import com.epam.aidial.core.service.LockService; @@ -133,6 +134,21 @@ public Future rejectPublication() { return Future.succeededFuture(); } + public Future listRules() { + context.getRequest() + .body() + .compose(body -> { + String url = ProxyUtil.convertToObject(body, ResourceLink.class).url(); + ResourceDescription rule = decodeRule(url); + checkRuleAccess(rule); + return vertx.executeBlocking(() -> publicationService.listRules(rule)); + }) + .onSuccess(rules -> context.respond(HttpStatus.OK, new Rules(rules))) + .onFailure(error -> respondError("Can't list rules", error)); + + return Future.succeededFuture(); + } + private void respondError(String message, Throwable error) { HttpStatus status = HttpStatus.INTERNAL_SERVER_ERROR; String body = null; @@ -172,6 +188,25 @@ private ResourceDescription decodePublication(String path, boolean allowPublic) return resource; } + private ResourceDescription decodeRule(String path) { + try { + if (!path.startsWith(BlobStorageUtil.PUBLIC_LOCATION)) { + throw new IllegalArgumentException(); + } + + String folder = path.substring(BlobStorageUtil.PUBLIC_LOCATION.length()); + ResourceDescription resource = ResourceDescription.fromEncoded(ResourceType.RULES, BlobStorageUtil.PUBLIC_BUCKET, BlobStorageUtil.PUBLIC_LOCATION, folder); + + if (!resource.isFolder()) { + throw new IllegalArgumentException(); + } + + return resource; + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid resource: " + path, e); + } + } + private void checkAccess(ResourceDescription resource, boolean allowUser) { boolean hasAccess = accessService.hasAdminAccess(context); @@ -184,4 +219,10 @@ private void checkAccess(ResourceDescription resource, boolean allowUser) { throw new HttpException(HttpStatus.FORBIDDEN, "Forbidden resource: " + resource.getUrl()); } } + + private void checkRuleAccess(ResourceDescription rule) { + if (!accessService.hasReadAccess(rule, context)) { + throw new HttpException(HttpStatus.FORBIDDEN, "Forbidden resource: " + rule.getUrl()); + } + } } \ No newline at end of file diff --git a/src/main/java/com/epam/aidial/core/data/Rules.java b/src/main/java/com/epam/aidial/core/data/Rules.java new file mode 100644 index 000000000..2ff6a9698 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/data/Rules.java @@ -0,0 +1,7 @@ +package com.epam.aidial.core.data; + +import java.util.List; +import java.util.Map; + +public record Rules(Map> rules) { +} \ No newline at end of file diff --git a/src/main/java/com/epam/aidial/core/service/PublicationService.java b/src/main/java/com/epam/aidial/core/service/PublicationService.java index f1b78ee54..2c7a8e628 100644 --- a/src/main/java/com/epam/aidial/core/service/PublicationService.java +++ b/src/main/java/com/epam/aidial/core/service/PublicationService.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -56,6 +57,9 @@ public class PublicationService { private static final Set ALLOWED_RESOURCES = Set.of(ResourceType.FILE, ResourceType.CONVERSATION, ResourceType.PROMPT); + /** + * Key is updated time from the metadata. Value is decoded map (folder path, list of rules). + */ private final AtomicReference>>> cachedRules = new AtomicReference<>(); private final EncryptionService encryption; private final ResourceService resources; @@ -105,6 +109,27 @@ public void filterForbidden(ProxyContext context, ResourceDescription folder, Re metadata.setItems(filtered); } + public Map> listRules(ResourceDescription resource) { + if (!resource.isFolder() || !resource.isPublic()) { + throw new IllegalArgumentException("Bad rule url: " + resource.getUrl()); + } + + Map> rules = getCachedRules(); + Map> result = new TreeMap<>(); + + while (resource != null) { + String url = ruleUrl(resource);; + List list = rules.get(url); + resource = resource.getParent(); + + if (list != null) { + result.put(url, list); + } + } + + return result; + } + public Collection listPublications(ResourceDescription resource) { if (resource.getType() != ResourceType.PUBLICATION || !resource.isRootFolder()) { throw new IllegalArgumentException("Bad publication url: " + resource.getUrl()); @@ -564,17 +589,7 @@ private static String encodePublications(Map publications) private static Map> decodeRules(String json) { Map> rules = ProxyUtil.convertToObject(json, RULES_TYPE); - - if (rules == null) { - Rule rule = new Rule(); - rule.setSource("roles"); - rule.setFunction(Rule.Function.TRUE); - rule.setTargets(List.of()); - rules = new LinkedHashMap<>(); - rules.put(PUBLIC_LOCATION, List.of(rule)); - } - - return rules; + return (rules == null) ? new LinkedHashMap<>() : rules; } private static String encodeRules(Map> rules) { diff --git a/src/test/java/com/epam/aidial/core/PublicationApiTest.java b/src/test/java/com/epam/aidial/core/PublicationApiTest.java index b6692c545..0ecc26e8c 100644 --- a/src/test/java/com/epam/aidial/core/PublicationApiTest.java +++ b/src/test/java/com/epam/aidial/core/PublicationApiTest.java @@ -518,4 +518,89 @@ void testPublicationList() { } """); } + + @Test + void listRules() { + Response response = operationRequest("/v1/ops/publications/rules/list", """ + {"url": ""} + """); + verify(response, 400); + + response = operationRequest("/v1/ops/publications/rules/list", """ + {"url": "public"} + """); + verify(response, 400); + + response = operationRequest("/v1/ops/publications/rules/list", """ + {"url": "public/"} + """); + verifyJson(response, 200, """ + { + "rules" : { } + } + """); + + response = operationRequest("/v1/ops/publications/rules/list", """ + {"url": "public/"} + """, "authorization", "user"); + verifyJson(response, 200, """ + { + "rules" : { } + } + """); + + response = operationRequest("/v1/ops/publications/rules/list", """ + {"url": "public/"} + """, "authorization", "admin"); + verifyJson(response, 200, """ + { + "rules" : { } + } + """); + + response = resourceRequest(HttpMethod.PUT, "/my/folder/conversation", CONVERSATION_BODY_1); + verify(response, 200); + + response = operationRequest("/v1/ops/publications/create", PUBLICATION_REQUEST.formatted(bucket, bucket)); + verify(response, 200); + + response = operationRequest("/v1/ops/publications/approve", PUBLICATION_URL, "authorization", "admin"); + verify(response, 200); + + + response = operationRequest("/v1/ops/publications/rules/list", """ + {"url": "public/folder/"} + """, "authorization", "user"); + verifyJson(response, 200, """ + { + "rules" : { + "public/folder/" : [ { + "function" : "EQUAL", + "source" : "roles", + "targets" : [ "user" ] + } ] + } + } + """); + + response = operationRequest("/v1/ops/publications/rules/list", """ + {"url": "public/folder/"} + """, "authorization", "admin"); + verifyJson(response, 200, """ + { + "rules" : { + "public/folder/" : [ { + "function" : "EQUAL", + "source" : "roles", + "targets" : [ "user" ] + } ] + } + } + """); + + response = operationRequest("/v1/ops/publications/rules/list", """ + {"url": "public/folder/"} + """); + verify(response, 403); + } } \ No newline at end of file