diff --git a/src/main/java/com/epam/aidial/core/AiDial.java b/src/main/java/com/epam/aidial/core/AiDial.java index 7a900eaa0..3babd0cdc 100644 --- a/src/main/java/com/epam/aidial/core/AiDial.java +++ b/src/main/java/com/epam/aidial/core/AiDial.java @@ -13,6 +13,7 @@ import com.epam.aidial.core.security.EncryptionService; import com.epam.aidial.core.service.InvitationService; import com.epam.aidial.core.service.LockService; +import com.epam.aidial.core.service.PublicationService; import com.epam.aidial.core.service.ResourceService; import com.epam.aidial.core.service.ShareService; import com.epam.aidial.core.storage.BlobStorage; @@ -51,8 +52,11 @@ import java.util.Map; import java.util.Objects; import java.util.Properties; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; +import java.util.function.Supplier; @Slf4j @Setter @@ -71,6 +75,10 @@ public class AiDial { private ResourceService resourceService; private InvitationService invitationService; private ShareService shareService; + private PublicationService publicationService; + + private LongSupplier clock = System::currentTimeMillis; + private Supplier generator = () -> UUID.randomUUID().toString().replace("-", ""); @VisibleForTesting void start() throws Exception { @@ -103,18 +111,18 @@ void start() throws Exception { resourceService = new ResourceService(vertx, redis, storage, lockService, settings("resources"), storage.getPrefix()); invitationService = new InvitationService(resourceService, encryptionService, settings("invitations")); shareService = new ShareService(resourceService, invitationService, encryptionService); + publicationService = new PublicationService(encryptionService, resourceService, storage, generator, clock); } else { log.warn("Redis config is not found, some features may be unavailable"); } - AccessService accessService = new AccessService(encryptionService, shareService); - + AccessService accessService = new AccessService(encryptionService, shareService, publicationService); RateLimiter rateLimiter = new RateLimiter(vertx, resourceService); proxy = new Proxy(vertx, client, configStore, logStore, rateLimiter, upstreamBalancer, accessTokenValidator, storage, encryptionService, apiKeyStore, tokenStatsTracker, resourceService, invitationService, - shareService, accessService); + shareService, publicationService, accessService); server = vertx.createHttpServer(new HttpServerOptions(settings("server"))).requestHandler(proxy); open(server, HttpServer::listen); diff --git a/src/main/java/com/epam/aidial/core/Proxy.java b/src/main/java/com/epam/aidial/core/Proxy.java index c24afb5f7..03d0b9939 100644 --- a/src/main/java/com/epam/aidial/core/Proxy.java +++ b/src/main/java/com/epam/aidial/core/Proxy.java @@ -13,6 +13,7 @@ import com.epam.aidial.core.security.EncryptionService; import com.epam.aidial.core.security.ExtractedClaims; import com.epam.aidial.core.service.InvitationService; +import com.epam.aidial.core.service.PublicationService; import com.epam.aidial.core.service.ResourceService; import com.epam.aidial.core.service.ShareService; import com.epam.aidial.core.storage.BlobStorage; @@ -75,6 +76,7 @@ public class Proxy implements Handler { private final ResourceService resourceService; private final InvitationService invitationService; private final ShareService shareService; + private final PublicationService publicationService; private final AccessService accessService; @Override diff --git a/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java b/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java index d64b2e710..17776607c 100644 --- a/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java +++ b/src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java @@ -54,12 +54,16 @@ public Future handle(String resourceType, String bucket, String path) { return true; } + if (proxy.getAccessService().isReviewResource(resource, context)) { + return true; + } + return proxy.getAccessService().isSharedResource(resource, context); } return false; }) - .map(hasAccess -> { + .map(hasAccess -> { if (hasAccess) { handle(resource); } else { diff --git a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java index c01eede66..ae4784f59 100644 --- a/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java +++ b/src/main/java/com/epam/aidial/core/controller/ControllerSelector.java @@ -48,6 +48,7 @@ public class ControllerSelector { private static final Pattern SHARE_RESOURCE_OPERATIONS = Pattern.compile("^/v1/ops/resource/share/(create|list|discard|revoke)$"); private static final Pattern INVITATIONS = Pattern.compile("^/v1/invitations$"); private static final Pattern INVITATION = Pattern.compile("^/v1/invitations/([a-zA-Z0-9]+)$"); + private static final Pattern PUBLICATIONS = Pattern.compile("^/v1/ops/publications/(list|get|create|delete)$"); private static final Pattern DEPLOYMENT_LIMITS = Pattern.compile("^/v1/deployments/([^/]+)/limits$"); @@ -257,6 +258,20 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p return () -> controller.handle(op); } + match = match(PUBLICATIONS, path); + if (match != null) { + String operation = match.group(1); + PublicationController controller = new PublicationController(proxy, context); + + return switch (operation) { + case "list" -> controller::listPublications; + case "get"-> controller::getPublication; + case "create"-> controller::createPublication; + case "delete"-> controller::deletePublication; + default -> null; + }; + } + return null; } @@ -280,7 +295,7 @@ private static Controller selectDelete(Proxy proxy, ProxyContext context, String match = match(INVITATION, path); if (match != null) { - String invitationId = UrlUtil.decodePath(match.group(1)); + String invitationId = UrlUtil.decodePath(match.group(1)); InvitationController controller = new InvitationController(proxy, context); return () -> controller.deleteInvitation(invitationId); } diff --git a/src/main/java/com/epam/aidial/core/controller/PublicationController.java b/src/main/java/com/epam/aidial/core/controller/PublicationController.java new file mode 100644 index 000000000..70e013763 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/controller/PublicationController.java @@ -0,0 +1,142 @@ +package com.epam.aidial.core.controller; + +import com.epam.aidial.core.Proxy; +import com.epam.aidial.core.ProxyContext; +import com.epam.aidial.core.data.Publication; +import com.epam.aidial.core.data.Publications; +import com.epam.aidial.core.data.ResourceLink; +import com.epam.aidial.core.data.ResourceType; +import com.epam.aidial.core.security.EncryptionService; +import com.epam.aidial.core.service.PublicationService; +import com.epam.aidial.core.storage.BlobStorageUtil; +import com.epam.aidial.core.storage.ResourceDescription; +import com.epam.aidial.core.util.HttpException; +import com.epam.aidial.core.util.HttpStatus; +import com.epam.aidial.core.util.ProxyUtil; +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class PublicationController { + + private final Vertx vertx; + private final EncryptionService encryptService; + private final PublicationService publicationService; + private final ProxyContext context; + + public PublicationController(Proxy proxy, ProxyContext context) { + this.vertx = proxy.getVertx(); + this.encryptService = proxy.getEncryptionService(); + this.publicationService = proxy.getPublicationService(); + this.context = context; + } + + public Future listPublications() { + context.getRequest() + .body() + .compose(body -> { + String url = ProxyUtil.convertToObject(body, ResourceLink.class).url(); + ResourceDescription resource = decodePublication(url); + checkAccess(resource); + return vertx.executeBlocking(() -> publicationService.listPublications(resource)); + }) + .onSuccess(publications -> context.respond(HttpStatus.OK, new Publications(publications))) + .onFailure(error -> respond("Can't list publications", error)); + + return Future.succeededFuture(); + } + + public Future getPublication() { + context.getRequest() + .body() + .compose(body -> { + String url = ProxyUtil.convertToObject(body, ResourceLink.class).url(); + ResourceDescription resource = decodePublication(url); + checkAccess(resource); + return vertx.executeBlocking(() -> publicationService.getPublication(resource)); + }) + .onSuccess(publication -> { + if (publication == null) { + context.respond(HttpStatus.NOT_FOUND); + } else { + context.respond(HttpStatus.OK, publication); + } + }) + .onFailure(error -> respond("Can't get publication", error)); + + return Future.succeededFuture(); + } + + public Future createPublication() { + context.getRequest() + .body() + .compose(body -> { + Publication publication = ProxyUtil.convertToObject(body, Publication.class); + ResourceDescription resource = decodePublication(publication.getUrl()); + checkAccess(resource); + return vertx.executeBlocking(() -> publicationService.createPublication(resource, publication)); + }) + .onSuccess(publication -> context.respond(HttpStatus.OK, publication)) + .onFailure(error -> respond("Can't create publication", error)); + + return Future.succeededFuture(); + } + + public Future deletePublication() { + context.getRequest() + .body() + .compose(body -> { + String url = ProxyUtil.convertToObject(body, ResourceLink.class).url(); + ResourceDescription resource = decodePublication(url); + checkAccess(resource); + return vertx.executeBlocking(() -> publicationService.deletePublication(resource)); + }) + .onSuccess(deleted -> context.respond(deleted ? HttpStatus.OK : HttpStatus.NOT_FOUND)) + .onFailure(error -> respond("Can't delete publication", error)); + + return Future.succeededFuture(); + } + + private void respond(String message, Throwable error) { + HttpStatus status = HttpStatus.INTERNAL_SERVER_ERROR; + String body = null; + + if (error instanceof HttpException e) { + status = e.getStatus(); + body = e.getMessage(); + } else if (error instanceof IllegalArgumentException e) { + status = HttpStatus.BAD_REQUEST; + body = e.getMessage(); + } else { + log.warn(message, error); + } + + context.respond(status, body == null ? "" : body); + } + + private ResourceDescription decodePublication(String path) { + ResourceDescription resource; + try { + resource = ResourceDescription.fromLink(path, encryptService); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid resource: " + path, e); + } + + if (resource.getType() != ResourceType.PUBLICATION) { + throw new IllegalArgumentException("Invalid resource: " + path); + } + + return resource; + } + + private void checkAccess(ResourceDescription resource) { + String bucket = BlobStorageUtil.buildInitiatorBucket(context); + + if (!resource.getBucketLocation().equals(bucket)) { + throw new HttpException(HttpStatus.FORBIDDEN, "Forbidden resource: " + resource.getUrl()); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/epam/aidial/core/data/Publication.java b/src/main/java/com/epam/aidial/core/data/Publication.java new file mode 100644 index 000000000..ce073c6c7 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/data/Publication.java @@ -0,0 +1,43 @@ +package com.epam.aidial.core.data; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.util.List; + +@Data +@Accessors(chain = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Publication { + String url; + String sourceUrl; + String targetUrl; + Status status; + Long createdAt; + List resources; + List rules; + + public enum Status { + PENDING, APPROVED, REJECTED + } + + @Data + public static class Resource { + String sourceUrl; + String targetUrl; + String reviewUrl; + String version; + } + + @Data + public static class Rule { + Function function; + String source; + List targets; + + public enum Function { + EQUAL, CONTAIN, REGEX, + } + } +} \ No newline at end of file diff --git a/src/main/java/com/epam/aidial/core/data/Publications.java b/src/main/java/com/epam/aidial/core/data/Publications.java new file mode 100644 index 000000000..5121b93d7 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/data/Publications.java @@ -0,0 +1,6 @@ +package com.epam.aidial.core.data; + +import java.util.Collection; + +public record Publications(Collection publications) { +} \ No newline at end of file diff --git a/src/main/java/com/epam/aidial/core/data/ResourceType.java b/src/main/java/com/epam/aidial/core/data/ResourceType.java index ade1d718f..68525d313 100644 --- a/src/main/java/com/epam/aidial/core/data/ResourceType.java +++ b/src/main/java/com/epam/aidial/core/data/ResourceType.java @@ -5,7 +5,8 @@ @Getter public enum ResourceType { FILE("files"), CONVERSATION("conversations"), PROMPT("prompts"), LIMIT("limits"), - SHARED_WITH_ME("shared_with_me"), SHARED_BY_ME("shared_by_me"), INVITATION("invitations"); + SHARED_WITH_ME("shared_with_me"), SHARED_BY_ME("shared_by_me"), INVITATION("invitations"), + PUBLICATION("publications"); private final String group; @@ -23,6 +24,7 @@ public static ResourceType of(String group) { case "conversations" -> CONVERSATION; case "prompts" -> PROMPT; case "invitations" -> INVITATION; + case "publications" -> PUBLICATION; default -> throw new IllegalArgumentException("Unsupported group: " + group); }; } diff --git a/src/main/java/com/epam/aidial/core/data/ResourceUrl.java b/src/main/java/com/epam/aidial/core/data/ResourceUrl.java new file mode 100644 index 000000000..3ef1c6bc9 --- /dev/null +++ b/src/main/java/com/epam/aidial/core/data/ResourceUrl.java @@ -0,0 +1,67 @@ +package com.epam.aidial.core.data; + +import com.epam.aidial.core.storage.BlobStorageUtil; +import com.epam.aidial.core.util.UrlUtil; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class ResourceUrl { + + @Getter + private final String rawUrl; + private final String[] segments; + @Getter + private final boolean folder; + + public boolean startsWith(String segment) { + return segments.length > 0 && segments[0].equals(segment); + } + + public String getUrl() { + StringBuilder builder = new StringBuilder(rawUrl.length()); + + for (int i = 0; i < segments.length; i++) { + if (i > 0) { + builder.append(BlobStorageUtil.PATH_SEPARATOR); + } + + builder.append(UrlUtil.encodePath(segments[i])); + } + + if (folder) { + builder.append(BlobStorageUtil.PATH_SEPARATOR); + } + + return builder.toString(); + } + + @Override + public String toString() { + return getUrl(); + } + + public static ResourceUrl parse(String url) { + if (url == null) { + throw new IllegalArgumentException("url is missing"); + } + + try { + String[] segments = url.split(BlobStorageUtil.PATH_SEPARATOR); + + for (int i = 0; i < segments.length; i++) { + String segment = UrlUtil.decodePath(segments[i]); + + if (segment == null || segment.isEmpty() || segment.contains(BlobStorageUtil.PATH_SEPARATOR)) { + throw new IllegalArgumentException("Bad segment: " + segment + " in url: " + url); + } + + segments[i] = segment; + } + + return new ResourceUrl(url, segments, url.endsWith(BlobStorageUtil.PATH_SEPARATOR)); + } catch (Throwable e) { + throw new IllegalArgumentException("Bad resource url: " + url); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/epam/aidial/core/security/AccessService.java b/src/main/java/com/epam/aidial/core/security/AccessService.java index f4c441df7..3989a73c1 100644 --- a/src/main/java/com/epam/aidial/core/security/AccessService.java +++ b/src/main/java/com/epam/aidial/core/security/AccessService.java @@ -1,6 +1,7 @@ package com.epam.aidial.core.security; import com.epam.aidial.core.ProxyContext; +import com.epam.aidial.core.service.PublicationService; import com.epam.aidial.core.service.ShareService; import com.epam.aidial.core.storage.BlobStorageUtil; import com.epam.aidial.core.storage.ResourceDescription; @@ -11,8 +12,8 @@ public class AccessService { private final EncryptionService encryptionService; - private final ShareService shareService; + private final PublicationService publicationService; public boolean hasWriteAccess(String filePath, String decryptedBucket, ProxyContext context) { String expectedUserBucket = BlobStorageUtil.buildUserBucket(context); @@ -43,4 +44,9 @@ public boolean isSharedResource(ResourceDescription resource, ProxyContext conte return shareService != null && shareService.hasReadAccess(actualUserBucket, actualUserLocation, resource); } + public boolean isReviewResource(ResourceDescription resource, ProxyContext context) { + String actualUserLocation = BlobStorageUtil.buildInitiatorBucket(context); + String actualUserBucket = encryptionService.encrypt(actualUserLocation); + return publicationService != null && publicationService.hasReviewAccess(resource, actualUserBucket, actualUserLocation); + } } diff --git a/src/main/java/com/epam/aidial/core/service/PublicationService.java b/src/main/java/com/epam/aidial/core/service/PublicationService.java new file mode 100644 index 000000000..083a5e4db --- /dev/null +++ b/src/main/java/com/epam/aidial/core/service/PublicationService.java @@ -0,0 +1,358 @@ +package com.epam.aidial.core.service; + +import com.epam.aidial.core.data.Publication; +import com.epam.aidial.core.data.ResourceType; +import com.epam.aidial.core.data.ResourceUrl; +import com.epam.aidial.core.security.EncryptionService; +import com.epam.aidial.core.storage.BlobStorage; +import com.epam.aidial.core.storage.ResourceDescription; +import com.epam.aidial.core.util.ProxyUtil; +import com.epam.aidial.core.util.UrlUtil; +import com.fasterxml.jackson.core.type.TypeReference; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.mutable.MutableObject; + +import javax.annotation.Nullable; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.LongSupplier; +import java.util.function.Supplier; + +import static com.epam.aidial.core.storage.BlobStorageUtil.PATH_SEPARATOR; +import static com.epam.aidial.core.storage.ResourceDescription.PUBLIC_BUCKET; + +@RequiredArgsConstructor +public class PublicationService { + + private static final String PUBLICATIONS_NAME = "publications"; + private static final TypeReference> PUBLICATIONS_TYPE = new TypeReference<>() { + }; + + private static final ResourceDescription PUBLIC_PUBLICATIONS = ResourceDescription.fromDecoded( + ResourceType.PUBLICATION, PUBLIC_BUCKET, ResourceDescription.PUBLIC_LOCATION, + PUBLICATIONS_NAME); + + + private final EncryptionService encryption; + private final ResourceService resources; + private final BlobStorage files; + private final Supplier ids; + private final LongSupplier clock; + + public boolean hasReviewAccess(ResourceDescription resource, String userBucket, String userLocation) { + String reviewLocation = userLocation + PUBLICATIONS_NAME + PATH_SEPARATOR; + return resource.getBucketLocation().startsWith(reviewLocation); + } + + public Collection listPublications(ResourceDescription resource) { + if (resource.getType() != ResourceType.PUBLICATION || !resource.isRootFolder()) { + throw new IllegalArgumentException("Bad publication url: " + resource.getUrl()); + } + + ResourceDescription key = localPublications(resource); + Map publications = decodePublications(resources.getResource(key)); + + for (Publication publication : publications.values()) { + leaveDescription(publication); + } + + return publications.values(); + } + + @Nullable + public Publication getPublication(ResourceDescription resource) { + if (resource.getType() != ResourceType.PUBLICATION || resource.isFolder() || resource.getParentPath() != null) { + throw new IllegalArgumentException("Bad publication url: " + resource.getUrl()); + } + + ResourceDescription key = localPublications(resource); + Map publications = decodePublications(resources.getResource(key)); + + return publications.get(resource.getUrl()); + } + + public Publication createPublication(ResourceDescription bucket, Publication publication) { + if (bucket.getType() != ResourceType.PUBLICATION || !bucket.isRootFolder()) { + throw new IllegalArgumentException("Bad publication bucket: " + bucket.getUrl()); + } + + preparePublication(bucket, publication); + + checkSourceResources(publication); + checkTargetResources(publication); + + copySourceToReviewResources(publication); + + resources.computeResource(localPublications(bucket), body -> { + Map publications = decodePublications(body); + + if (publications.put(publication.getUrl(), publication) != null) { + throw new IllegalStateException("Publication with such url already exists: " + publication.getUrl()); + } + + return encodePublications(publications); + }); + + leaveDescription(publication); + resources.computeResource(PUBLIC_PUBLICATIONS, body -> { + Map publications = decodePublications(body); + + if (publications.put(publication.getUrl(), publication) != null) { + throw new IllegalStateException("Publication with such url already exists: " + publication.getUrl()); + } + + return encodePublications(publications); + }); + + return publication; + } + + public boolean deletePublication(ResourceDescription resource) { + if (resource.isFolder() || resource.getParentPath() != null) { + throw new IllegalArgumentException("Bad publication url: " + resource.getUrl()); + } + + MutableObject deleted = new MutableObject<>(); + + resources.computeResource(PUBLIC_PUBLICATIONS, body -> { + Map publications = decodePublications(body); + Publication publication = publications.remove(resource.getUrl()); + return (publication == null) ? body : encodePublications(publications); + }); + + resources.computeResource(localPublications(resource), body -> { + Map publications = decodePublications(body); + Publication publication = publications.remove(resource.getUrl()); + + if (publication == null) { + return body; + } + + deleted.setValue(publication); + return encodePublications(publications); + }); + + Publication publication = deleted.getValue(); + + if (publication == null) { + return false; + } + + if (publication.getStatus() == Publication.Status.PENDING) { + deleteReviewResources(publication); + } + + return true; + } + + private void preparePublication(ResourceDescription bucket, Publication publication) { + if (publication.getSourceUrl() == null) { + throw new IllegalArgumentException("Publication \"sourceUrl\" is missing"); + } + + if (publication.getTargetUrl() == null) { + throw new IllegalArgumentException("Publication \"targetUrl\" is missing"); + } + + if (publication.getResources() == null) { + publication.setResources(List.of()); + } + + if (publication.getResources().isEmpty() && publication.getRules() == null) { + throw new IllegalArgumentException("No resources and no rules in publication"); + } + + ResourceUrl sourceFolder = ResourceUrl.parse(publication.getSourceUrl()); + ResourceUrl targetFolder = ResourceUrl.parse(publication.getTargetUrl()); + + if (!sourceFolder.startsWith(bucket.getBucketName()) || !sourceFolder.isFolder()) { + throw new IllegalArgumentException("Publication \"sourceUrl\" must start with: %s and ends with: %s" + .formatted(bucket.getBucketName(), PATH_SEPARATOR)); + } + + if (!targetFolder.startsWith(PUBLIC_BUCKET) || !targetFolder.isFolder()) { + throw new IllegalArgumentException("Publication \"targetUrl\" must start with: %s and ends with: %s" + .formatted(PUBLIC_BUCKET, PATH_SEPARATOR)); + } + + String id = UrlUtil.encodePath(ids.get()); + String reviewBucket = encodeReviewBucket(bucket, id); + + publication.setUrl(bucket.getUrl() + id); + publication.setSourceUrl(sourceFolder.getUrl()); + publication.setTargetUrl(targetFolder.getUrl()); + publication.setCreatedAt(clock.getAsLong()); + publication.setStatus(Publication.Status.PENDING); + + Set urls = new HashSet<>(); + + for (Publication.Resource resource : publication.getResources()) { + ResourceDescription source = ResourceDescription.fromBucketLink(resource.getSourceUrl(), bucket); + String sourceUrl = source.getUrl(); + + if (source.isFolder()) { + throw new IllegalArgumentException("Resource folder is not allowed: " + sourceUrl); + } + + if (source.getType() != ResourceType.FILE && source.getType() != ResourceType.PROMPT && source.getType() != ResourceType.CONVERSATION) { + throw new IllegalArgumentException("Resource type is not supported: " + sourceUrl); + } + + String sourceSuffix = sourceUrl.substring(source.getType().getGroup().length() + 1); + + if (!sourceSuffix.startsWith(publication.getSourceUrl())) { + throw new IllegalArgumentException("Resource folder does not match with source folder: " + publication.getSourceUrl()); + } + + String version = resource.getVersion(); + + if (version == null) { + throw new IllegalArgumentException("Resource version is missing: " + sourceUrl); + } + + if (!version.equals(UrlUtil.decodePath(version))) { + throw new IllegalArgumentException("Resource version contains not allowed characters: " + version); + } + + sourceSuffix = sourceSuffix.substring(publication.getSourceUrl().length()); + + String targetUrl = source.getType().getGroup() + PATH_SEPARATOR + + publication.getTargetUrl() + sourceSuffix + "." + version; + + String reviewUrl = source.getType().getGroup() + PATH_SEPARATOR + + reviewBucket + PATH_SEPARATOR + sourceSuffix; + + if (!urls.add(sourceUrl)) { + throw new IllegalArgumentException("Source resources have duplicate urls: " + sourceUrl); + } + + if (!urls.add(targetUrl)) { + throw new IllegalArgumentException("Target resources have duplicate urls: " + targetUrl); + } + + if (!urls.add(reviewUrl)) { + throw new IllegalArgumentException("Review resources have duplicate urls: " + reviewUrl); + } + + resource.setSourceUrl(sourceUrl); + resource.setTargetUrl(targetUrl); + resource.setReviewUrl(reviewUrl); + } + + if (publication.getRules() != null) { + for (Publication.Rule rule : publication.getRules()) { + if (rule.getSource() == null) { + throw new IllegalArgumentException("Rule does not have source"); + } + + if (rule.getTargets() == null || rule.getTargets().isEmpty()) { + throw new IllegalArgumentException("Rule does not have targets"); + } + + if (rule.getFunction() == null) { + throw new IllegalArgumentException("Rule does not have function"); + } + } + } + } + + private void checkSourceResources(Publication publication) { + for (Publication.Resource resource : publication.getResources()) { + String url = resource.getSourceUrl(); + ResourceDescription descriptor = ResourceDescription.fromLink(url, encryption); + + if (!checkResource(descriptor)) { + throw new IllegalArgumentException("Source resource does not exist: " + descriptor.getUrl()); + } + } + } + + private void checkTargetResources(Publication publication) { + for (Publication.Resource resource : publication.getResources()) { + String url = resource.getTargetUrl(); + ResourceDescription descriptor = ResourceDescription.fromPublicLink(url); + + if (checkResource(descriptor)) { + throw new IllegalArgumentException("Target resource already exists: " + descriptor.getUrl()); + } + } + } + + private void copySourceToReviewResources(Publication publication) { + for (Publication.Resource resource : publication.getResources()) { + String sourceUrl = resource.getSourceUrl(); + String reviewUrl = resource.getReviewUrl(); + + ResourceDescription from = ResourceDescription.fromLink(sourceUrl, encryption); + ResourceDescription to = ResourceDescription.fromLink(reviewUrl, encryption); + + if (!copyResource(from, to)) { + throw new IllegalStateException("Can't copy source resource from: " + from.getUrl() + " to review: " + to.getUrl()); + } + } + } + + private void deleteReviewResources(Publication publication) { + for (Publication.Resource resource : publication.getResources()) { + String url = resource.getReviewUrl(); + ResourceDescription descriptor = ResourceDescription.fromLink(url, encryption); + deleteResource(descriptor); + } + } + + private boolean checkResource(ResourceDescription descriptor) { + return switch (descriptor.getType()) { + case FILE -> files.exists(descriptor.getAbsoluteFilePath()); + case PROMPT, CONVERSATION -> resources.hasResource(descriptor); + default -> throw new IllegalStateException("Unsupported type: " + descriptor.getType()); + }; + } + + private boolean copyResource(ResourceDescription from, ResourceDescription to) { + return switch (from.getType()) { + case FILE -> files.copy(from.getAbsoluteFilePath(), to.getAbsoluteFilePath()); + case PROMPT, CONVERSATION -> resources.copyResource(from, to); + default -> throw new IllegalStateException("Unsupported type: " + from.getType()); + }; + } + + private void deleteResource(ResourceDescription descriptor) { + switch (descriptor.getType()) { + case FILE -> files.delete(descriptor.getAbsoluteFilePath()); + case PROMPT, CONVERSATION -> resources.deleteResource(descriptor); + default -> throw new IllegalStateException("Unsupported type: " + descriptor.getType()); + } + } + + private String encodeReviewBucket(ResourceDescription bucket, String id) { + String path = bucket.getBucketLocation() + + PUBLICATIONS_NAME + PATH_SEPARATOR + + id + PATH_SEPARATOR; + + return encryption.encrypt(path); + } + + private static void leaveDescription(Publication publication) { + publication.setResources(null).setResources(null); + } + + private static ResourceDescription localPublications(ResourceDescription resource) { + return ResourceDescription.fromDecoded(ResourceType.PUBLICATION, + resource.getBucketName(), + resource.getBucketLocation(), + PUBLICATIONS_NAME); + } + + private static Map decodePublications(String resources) { + Map publications = ProxyUtil.convertToObject(resources, PUBLICATIONS_TYPE); + return (publications == null) ? new LinkedHashMap<>() : publications; + } + + private static String encodePublications(Map publications) { + return ProxyUtil.convertToString(publications); + } +} diff --git a/src/main/java/com/epam/aidial/core/service/ResourceService.java b/src/main/java/com/epam/aidial/core/service/ResourceService.java index 141ccb044..14bdaa5e2 100644 --- a/src/main/java/com/epam/aidial/core/service/ResourceService.java +++ b/src/main/java/com/epam/aidial/core/service/ResourceService.java @@ -175,6 +175,18 @@ private ResourceItemMetadata getItemMetadata(ResourceDescription descriptor) { .setUpdatedAt(result.updatedAt); } + public boolean hasResource(ResourceDescription descriptor) { + String redisKey = redisKey(descriptor); + Result result = redisGet(redisKey, false); + + if (result == null) { + String blobKey = blobKey(descriptor); + return blobExists(blobKey); + } + + return result.exists; + } + @Nullable public String getResource(ResourceDescription descriptor) { return getResource(descriptor, true); @@ -266,6 +278,17 @@ public boolean deleteResource(ResourceDescription descriptor) { } } + public boolean copyResource(ResourceDescription from, ResourceDescription to) { + String body = getResource(from); + + if (body == null) { + return false; + } + + putResource(to, body, true); + return true; + } + private Void sync() { log.debug("Syncing"); try { diff --git a/src/main/java/com/epam/aidial/core/storage/BlobStorage.java b/src/main/java/com/epam/aidial/core/storage/BlobStorage.java index 1c90c9428..b08b83b3d 100644 --- a/src/main/java/com/epam/aidial/core/storage/BlobStorage.java +++ b/src/main/java/com/epam/aidial/core/storage/BlobStorage.java @@ -23,6 +23,7 @@ import org.jclouds.blobstore.domain.internal.BlobMetadataImpl; import org.jclouds.blobstore.domain.internal.MutableStorageMetadataImpl; import org.jclouds.blobstore.domain.internal.PageSetImpl; +import org.jclouds.blobstore.options.CopyOptions; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.options.PutOptions; import org.jclouds.io.ContentMetadata; @@ -191,6 +192,11 @@ public void delete(String filePath) { blobStore.removeBlob(bucketName, storageLocation); } + public boolean copy(String fromPath, String toPath) { + blobStore.copyBlob(bucketName, getStorageLocation(fromPath), bucketName, getStorageLocation(toPath), CopyOptions.NONE); + return true; + } + /** * List all files/folder metadata for a given resource */ diff --git a/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java b/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java index f47b35821..233dd2251 100644 --- a/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java +++ b/src/main/java/com/epam/aidial/core/storage/ResourceDescription.java @@ -18,6 +18,9 @@ @AllArgsConstructor(access = AccessLevel.PRIVATE) public class ResourceDescription { + public static final String PUBLIC_BUCKET = "public"; + public static final String PUBLIC_LOCATION = PUBLIC_BUCKET + BlobStorageUtil.PATH_SEPARATOR; + private static final int MAX_PATH_SIZE = 900; ResourceType type; @@ -162,6 +165,30 @@ public static ResourceDescription fromLink(String link, EncryptionService encryp return fromEncoded(resourceType, bucket, location, resourcePath); } + public static ResourceDescription fromBucketLink(String link, ResourceDescription bucket) { + return fromLink(link, bucket.getBucketName(), bucket.getBucketLocation()); + } + + public static ResourceDescription fromPublicLink(String link) { + return fromLink(link, PUBLIC_BUCKET, PUBLIC_LOCATION); + } + + private static ResourceDescription fromLink(String link, String bucketEncoded, String bucketDecoded) { + String[] parts = link.split(BlobStorageUtil.PATH_SEPARATOR); + if (parts.length < 2) { + throw new IllegalArgumentException("Invalid resource link provided " + link); + } + + ResourceType resourceType = ResourceType.of(UrlUtil.decodePath(parts[0])); + String bucket = UrlUtil.decodePath(parts[1]); + if (!bucket.equals(bucketEncoded)) { + throw new IllegalArgumentException("Bucket does not match: " + bucket); + } + + String relativePath = link.substring(parts[0].length() + parts[1].length() + 2); + return fromEncoded(resourceType, bucketEncoded, bucketDecoded, relativePath); + } + private static ResourceDescription from(ResourceType type, String bucketName, String bucketLocation, String originalPath, List paths, boolean isFolder) { boolean isEmptyElements = paths.isEmpty(); diff --git a/src/main/java/com/epam/aidial/core/util/ProxyUtil.java b/src/main/java/com/epam/aidial/core/util/ProxyUtil.java index 539d170cb..a118eefd3 100644 --- a/src/main/java/com/epam/aidial/core/util/ProxyUtil.java +++ b/src/main/java/com/epam/aidial/core/util/ProxyUtil.java @@ -5,18 +5,21 @@ import com.epam.aidial.core.security.EncryptionService; import com.epam.aidial.core.storage.ResourceDescription; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.vertx.core.MultiMap; +import io.vertx.core.buffer.Buffer; import io.vertx.core.http.HttpClientResponse; import io.vertx.core.http.HttpHeaders; import io.vertx.core.http.HttpServerRequest; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.function.Consumer; import javax.annotation.Nullable; @@ -113,6 +116,27 @@ public static void collectAttachedFiles(ObjectNode tree, Consumer consum } } + public static T convertToObject(Buffer json, Class clazz) { + try { + String text = json.toString(StandardCharsets.UTF_8); + return MAPPER.readValue(text, clazz); + } catch (Throwable e) { + throw new IllegalArgumentException("Failed to parse json: " + e.getMessage()); + } + } + + @Nullable + public static T convertToObject(String payload, TypeReference type) { + if (payload == null) { + return null; + } + try { + return MAPPER.readValue(payload, type); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException(e); + } + } + @Nullable public static T convertToObject(String payload, Class clazz) { if (payload == null || payload.isEmpty()) { diff --git a/src/test/java/com/epam/aidial/core/PublicationApiTest.java b/src/test/java/com/epam/aidial/core/PublicationApiTest.java new file mode 100644 index 000000000..506856d0c --- /dev/null +++ b/src/test/java/com/epam/aidial/core/PublicationApiTest.java @@ -0,0 +1,126 @@ +package com.epam.aidial.core; + +import io.vertx.core.http.HttpMethod; +import org.junit.jupiter.api.Test; + +class PublicationApiTest extends ResourceBaseTest { + + @Test + void testPublicationCreation() { + Response response = resourceRequest(HttpMethod.PUT, "/my/folder/conversation", "12345"); + verify(response, 200); + + response = operationRequest("/v1/ops/publications/create", """ + { + "url": "publications/%s/", + "sourceUrl": "%s/my/folder/", + "targetUrl": "public/folder/", + "resources": [ + { + "sourceUrl": "conversations/%s/my/folder/conversation", + "version": "1" + } + ] + } + """.formatted(bucket, bucket, bucket)); + + verifyPretty(response, 200, """ + { + "url" : "publications/3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/0123", + "sourceUrl" : "3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/my/folder/", + "targetUrl" : "public/folder/", + "status" : "PENDING", + "createdAt" : 0 + } + """); + + response = operationRequest("/v1/ops/publications/get", """ + { + "url": "publications/3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/0123" + } + """); + + verifyPretty(response, 200, """ + { + "url" : "publications/3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/0123", + "sourceUrl" : "3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/my/folder/", + "targetUrl" : "public/folder/", + "status" : "PENDING", + "createdAt" : 0, + "resources" : [ { + "sourceUrl" : "conversations/3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/my/folder/conversation", + "targetUrl" : "conversations/public/folder/conversation.1", + "reviewUrl" : "conversations/2CZ9i2bcBACFts8JbBu3MdTHfU5imDZBmDVomBuDCkbhEstv1KXNzCiw693js8BLmo/conversation", + "version" : "1" + } ] + } + """); + + response = operationRequest("/v1/ops/publications/list", """ + { + "url": "publications/3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/" + } + """); + + verifyPretty(response, 200, """ + { + "publications" : [ { + "url" : "publications/3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/0123", + "sourceUrl" : "3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/my/folder/", + "targetUrl" : "public/folder/", + "status" : "PENDING", + "createdAt" : 0 + } ] + } + """); + + response = send(HttpMethod.GET, "/v1/conversations/2CZ9i2bcBACFts8JbBu3MdTHfU5imDZBmDVomBuDCkbhEstv1KXNzCiw693js8BLmo/conversation"); + verify(response, 200, "12345"); + + response = send(HttpMethod.PUT, "/v1/conversations/2CZ9i2bcBACFts8JbBu3MdTHfU5imDZBmDVomBuDCkbhEstv1KXNzCiw693js8BLmo/conversation"); + verify(response, 403); + + response = send(HttpMethod.DELETE, "/v1/conversations/2CZ9i2bcBACFts8JbBu3MdTHfU5imDZBmDVomBuDCkbhEstv1KXNzCiw693js8BLmo/conversation"); + verify(response, 403); + } + + @Test + void testPublicationDeletion() { + Response response = resourceRequest(HttpMethod.PUT, "/my/folder/conversation", "12345"); + verify(response, 200); + + response = operationRequest("/v1/ops/publications/create", """ + { + "url": "publications/%s/", + "sourceUrl": "%s/my/folder/", + "targetUrl": "public/folder/", + "resources": [ + { + "sourceUrl": "conversations/%s/my/folder/conversation", + "version": "1" + } + ] + } + """.formatted(bucket, bucket, bucket)); + verify(response, 200); + + response = send(HttpMethod.GET, "/v1/conversations/2CZ9i2bcBACFts8JbBu3MdTHfU5imDZBmDVomBuDCkbhEstv1KXNzCiw693js8BLmo/conversation"); + verify(response, 200); + + response = operationRequest("/v1/ops/publications/delete", """ + { + "url": "publications/3CcedGxCx23EwiVbVmscVktScRyf46KypuBQ65miviST/0123" + } + """); + verify(response, 200); + + response = send(HttpMethod.GET, "/v1/conversations/2CZ9i2bcBACFts8JbBu3MdTHfU5imDZBmDVomBuDCkbhEstv1KXNzCiw693js8BLmo/conversation"); + verify(response, 404); + + response = send(HttpMethod.PUT, "/v1/conversations/2CZ9i2bcBACFts8JbBu3MdTHfU5imDZBmDVomBuDCkbhEstv1KXNzCiw693js8BLmo/conversation"); + verify(response, 403); + + response = send(HttpMethod.DELETE, "/v1/conversations/2CZ9i2bcBACFts8JbBu3MdTHfU5imDZBmDVomBuDCkbhEstv1KXNzCiw693js8BLmo/conversation"); + verify(response, 403); + } +} \ No newline at end of file diff --git a/src/test/java/com/epam/aidial/core/ResourceBaseTest.java b/src/test/java/com/epam/aidial/core/ResourceBaseTest.java index c511b54f9..996a22273 100644 --- a/src/test/java/com/epam/aidial/core/ResourceBaseTest.java +++ b/src/test/java/com/epam/aidial/core/ResourceBaseTest.java @@ -2,6 +2,8 @@ import com.epam.aidial.core.util.ProxyUtil; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import io.vertx.core.http.HttpMethod; import io.vertx.core.json.JsonObject; import lombok.SneakyThrows; @@ -32,6 +34,8 @@ public class ResourceBaseTest { Path testDir; CloseableHttpClient client; String bucket; + long time = 0; + String id = "0123"; @BeforeEach void init() throws Exception { @@ -82,6 +86,8 @@ void init() throws Exception { dial = new AiDial(); dial.setSettings(settings); + dial.setGenerator(() -> id); + dial.setClock(() -> time); dial.start(); Response response = send(HttpMethod.GET, "/v1/bucket", null, ""); @@ -117,6 +123,11 @@ static void verify(Response response, int status) { assertEquals(status, response.status()); } + static void verifyPretty(Response response, int status, String body) { + assertEquals(status, response.status()); + assertEquals(body.trim(), pretty(response.body())); + } + static void verify(Response response, int status, String body) { assertEquals(status, response.status()); assertEquals(body, response.body()); @@ -154,6 +165,10 @@ Response metadata(String resource) { return send(HttpMethod.GET, "/v1/metadata/conversations/" + bucket + resource, null, ""); } + Response send(HttpMethod method, String path) { + return send(method, path, null, ""); + } + @SneakyThrows Response send(HttpMethod method, String path, String queryParams, String body, String... headers) { String uri = "http://127.0.0.1:" + dial.getServer().actualPort() + path + (queryParams != null ? "?" + queryParams : ""); @@ -190,6 +205,13 @@ Response send(HttpMethod method, String path, String queryParams, String body, S } } + @SneakyThrows + private static String pretty(String json) { + ObjectMapper mapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); + Object jsonObject = mapper.readValue(json, Object.class); + return mapper.writeValueAsString(jsonObject); + } + record Response(int status, String body) { public boolean ok() { return status() == 200;