Merge branch 'development' into issue-278

README.md

@@ -69,7 +69,7 @@ Static settings are used on startup and cannot be changed while application is r
 | redis.singleServerConfig.address           | -                  | Redis single server addresses, e.g. "redis://host:port"
 | redis.clusterServersConfig.nodeAddresses   | -                  | Json array with Redis cluster server addresses, e.g. ["redis://host1:port1","redis://host2:port2"]
 | invitations.ttlInSeconds                   | 259200             | Invitation time to live in seconds
-
+| access.admin.rules                         | -                  | Matches claims from identity providers with the rules to figure out whether a user is allowed to perform admin actions, like deleting any resource or approving a publication. Example: [{"source": "roles", "function": "EQUAL", "targets": ["admin"]}]. If roles contain "admin, the actions are allowed.
 ### Google Cloud Storage
 
 There are two types of credential providers supported:

src/main/java/com/epam/aidial/core/AiDial.java

@@ -72,6 +72,8 @@ public class AiDial {
     private RedissonClient redis;
     private Proxy proxy;
 
+    private AccessTokenValidator accessTokenValidator;
+
     private BlobStorage storage;
     private ResourceService resourceService;
 
@@ -94,7 +96,11 @@ void start() throws Exception {
             ConfigStore configStore = new FileConfigStore(vertx, settings("config"), apiKeyStore);
             LogStore logStore = new GfLogStore(vertx);
             UpstreamBalancer upstreamBalancer = new UpstreamBalancer();
-            AccessTokenValidator accessTokenValidator = new AccessTokenValidator(settings("identityProviders"), vertx);
+
+            if (accessTokenValidator == null) {
+                accessTokenValidator = new AccessTokenValidator(settings("identityProviders"), vertx);
+            }
+
             if (storage == null) {
                 Storage storageConfig = Json.decodeValue(settings("storage").toBuffer(), Storage.class);
                 storage = new BlobStorage(storageConfig);
@@ -104,14 +110,14 @@ void start() throws Exception {
 
             redis = openRedis();
 
-            LockService lockService = new LockService(redis);
+            LockService lockService = new LockService(redis, storage.getPrefix());
             resourceService = new ResourceService(vertx, redis, storage, lockService, settings("resources"), storage.getPrefix());
             InvitationService invitationService = new InvitationService(resourceService, encryptionService, settings("invitations"));
             ShareService shareService = new ShareService(resourceService, invitationService, encryptionService);
             PublicationService publicationService = new PublicationService(encryptionService, resourceService, storage, generator, clock);
             ResourceOperationService resourceOperationService = new ResourceOperationService(resourceService, storage, invitationService, shareService);
 
-            AccessService accessService = new AccessService(encryptionService, shareService, publicationService);
+            AccessService accessService = new AccessService(encryptionService, shareService, publicationService, settings("access"));
             RateLimiter rateLimiter = new RateLimiter(vertx, resourceService);
 
             proxy = new Proxy(vertx, client, configStore, logStore,

src/main/java/com/epam/aidial/core/controller/AccessControlBaseController.java

@@ -2,71 +2,40 @@
 
 import com.epam.aidial.core.Proxy;
 import com.epam.aidial.core.ProxyContext;
-import com.epam.aidial.core.data.ResourceType;
+import com.epam.aidial.core.security.AccessService;
 import com.epam.aidial.core.storage.ResourceDescription;
 import com.epam.aidial.core.util.HttpStatus;
-import com.epam.aidial.core.util.UrlUtil;
 import io.vertx.core.Future;
 import lombok.AllArgsConstructor;
 
 @AllArgsConstructor
 public abstract class AccessControlBaseController {
 
-    private static final String DEFAULT_RESOURCE_ERROR_MESSAGE = "Invalid resource url provided %s";
-
     final Proxy proxy;
     final ProxyContext context;
-    final boolean checkFullAccess;
-
-    /**
-     * @param bucket url encoded bucket name
-     * @param path   url encoded resource path
-     */
-    public Future<?> handle(String resourceType, String bucket, String path) {
-        ResourceType type = ResourceType.of(resourceType);
-        String urlDecodedBucket = UrlUtil.decodePath(bucket);
-        String decryptedBucket = proxy.getEncryptionService().decrypt(urlDecodedBucket);
-        if (decryptedBucket == null) {
-            context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the %s %s/%s".formatted(type, bucket, path));
-            return Future.succeededFuture();
-        }
+    final boolean isWriteAccess;
 
+    public Future<?> handle(String resourceUrl) {
         ResourceDescription resource;
+
         try {
-            resource = ResourceDescription.fromEncoded(type, urlDecodedBucket, decryptedBucket, path);
-        } catch (Exception ex) {
-            String errorMessage = ex.getMessage() != null ? ex.getMessage() : DEFAULT_RESOURCE_ERROR_MESSAGE.formatted(path);
+            resource = ResourceDescription.fromAnyUrl(resourceUrl, proxy.getEncryptionService());
+        } catch (IllegalArgumentException e) {
+            String errorMessage = e.getMessage() != null ? e.getMessage() : ("Invalid resource url provided: " + resourceUrl);
             context.respond(HttpStatus.BAD_REQUEST, errorMessage);
             return Future.succeededFuture();
         }
 
         return proxy.getVertx()
                 .executeBlocking(() -> {
-                    boolean hasWriteAccess = proxy.getAccessService().hasWriteAccess(path, decryptedBucket, context);
-                    if (hasWriteAccess) {
-                        return true;
-                    }
-
-                    if (!checkFullAccess) {
-                        // some per-request API-keys may have access to the resources implicitly
-                        if (proxy.getAccessService().isAutoSharedResource(resource, context)) {
-                            return true;
-                        }
-
-                        if (proxy.getAccessService().isReviewResource(resource, context)) {
-                            return true;
-                        }
-
-                        return proxy.getAccessService().isSharedResource(resource, context);
-                    }
-
-                    return false;
+                    AccessService service = proxy.getAccessService();
+                    return isWriteAccess ? service.hasWriteAccess(resource, context) : service.hasReadAccess(resource, context);
                 })
                 .map(hasAccess -> {
                     if (hasAccess) {
                         handle(resource);
                     } else {
-                        context.respond(HttpStatus.FORBIDDEN, "You don't have an access to the %s %s/%s".formatted(type, bucket, path));
+                        context.respond(HttpStatus.FORBIDDEN, "You don't have an access to: " + resourceUrl);
                     }
                     return null;
                 });

src/main/java/com/epam/aidial/core/controller/ControllerSelector.java

@@ -35,11 +35,11 @@ public class ControllerSelector {
 
     private static final Pattern PATTERN_BUCKET = Pattern.compile("^/v1/bucket$");
 
-    private static final Pattern PATTERN_FILES = Pattern.compile("^/v1/files/([a-zA-Z0-9]+)/(.*)");
-    private static final Pattern PATTERN_FILES_METADATA = Pattern.compile("^/v1/metadata/files/([a-zA-Z0-9]+)/(.*)");
+    private static final Pattern PATTERN_FILES = Pattern.compile("^/v1/files/[a-zA-Z0-9]+/.*");
+    private static final Pattern PATTERN_FILES_METADATA = Pattern.compile("^/v1/metadata/files/[a-zA-Z0-9]+/.*");
 
-    private static final Pattern PATTERN_RESOURCE = Pattern.compile("^/v1/(conversations|prompts)/([a-zA-Z0-9]+)/(.*)");
-    private static final Pattern PATTERN_RESOURCE_METADATA = Pattern.compile("^/v1/metadata/(conversations|prompts)/([a-zA-Z0-9]+)/(.*)");
+    private static final Pattern PATTERN_RESOURCE = Pattern.compile("^/v1/(conversations|prompts)/[a-zA-Z0-9]+/.*");
+    private static final Pattern PATTERN_RESOURCE_METADATA = Pattern.compile("^/v1/metadata/(conversations|prompts)/[a-zA-Z0-9]+/.*");
 
     private static final Pattern PATTERN_RATE_RESPONSE = Pattern.compile("^/+v1/([^/]+)/rate$");
     private static final Pattern PATTERN_TOKENIZE = Pattern.compile("^/+v1/deployments/([^/]+)/tokenize$");
@@ -48,7 +48,7 @@ public class ControllerSelector {
     private static final Pattern SHARE_RESOURCE_OPERATIONS = Pattern.compile("^/v1/ops/resource/share/(create|list|discard|revoke)$");
     private static final Pattern INVITATIONS = Pattern.compile("^/v1/invitations$");
     private static final Pattern INVITATION = Pattern.compile("^/v1/invitations/([a-zA-Z0-9]+)$");
-    private static final Pattern PUBLICATIONS = Pattern.compile("^/v1/ops/publications/(list|get|create|delete)$");
+    private static final Pattern PUBLICATIONS = Pattern.compile("^/v1/ops/publications/(list|get|create|delete|approve|reject)$");
 
     private static final Pattern RESOURCE_OPERATIONS = Pattern.compile("^/v1/ops/resources/(move)$");
 
@@ -142,36 +142,26 @@ private static Controller selectGet(Proxy proxy, ProxyContext context, String pa
 
         match = match(PATTERN_FILES_METADATA, path);
         if (match != null) {
-            String bucket = match.group(1);
-            String filePath = match.group(2);
             FileMetadataController controller = new FileMetadataController(proxy, context);
-            return () -> controller.handle("files", bucket, filePath);
+            return () -> controller.handle(resourcePath(path));
         }
 
         match = match(PATTERN_FILES, path);
         if (match != null) {
-            String bucket = match.group(1);
-            String filePath = match.group(2);
             DownloadFileController controller = new DownloadFileController(proxy, context);
-            return () -> controller.handle("files", bucket, filePath);
+            return () -> controller.handle(resourcePath(path));
         }
 
         match = match(PATTERN_RESOURCE, path);
         if (match != null) {
-            String resource = match.group(1);
-            String bucket = match.group(2);
-            String relativePath = match.group(3);
             ResourceController controller = new ResourceController(proxy, context, false);
-            return () -> controller.handle(resource, bucket, relativePath);
+            return () -> controller.handle(resourcePath(path));
         }
 
         match = match(PATTERN_RESOURCE_METADATA, path);
         if (match != null) {
-            String resource = match.group(1);
-            String bucket = match.group(2);
-            String relativePath = match.group(3);
             ResourceController controller = new ResourceController(proxy, context, true);
-            return () -> controller.handle(resource, bucket, relativePath);
+            return () -> controller.handle(resourcePath(path));
         }
 
         match = match(PATTERN_BUCKET, path);
@@ -270,6 +260,8 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p
                 case "get" -> controller::getPublication;
                 case "create" -> controller::createPublication;
                 case "delete" -> controller::deletePublication;
+                case "approve" -> controller::approvePublication;
+                case "reject" -> controller:: rejectPublication;
                 default -> null;
             };
         }
@@ -286,19 +278,14 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p
     private static Controller selectDelete(Proxy proxy, ProxyContext context, String path) {
         Matcher match = match(PATTERN_FILES, path);
         if (match != null) {
-            String bucket = match.group(1);
-            String filePath = match.group(2);
             DeleteFileController controller = new DeleteFileController(proxy, context);
-            return () -> controller.handle("files", bucket, filePath);
+            return () -> controller.handle(resourcePath(path));
         }
 
         match = match(PATTERN_RESOURCE, path);
         if (match != null) {
-            String resource = match.group(1);
-            String bucket = match.group(2);
-            String relativePath = match.group(3);
             ResourceController controller = new ResourceController(proxy, context, false);
-            return () -> controller.handle(resource, bucket, relativePath);
+            return () -> controller.handle(resourcePath(path));
         }
 
         match = match(INVITATION, path);
@@ -314,19 +301,14 @@ private static Controller selectDelete(Proxy proxy, ProxyContext context, String
     private static Controller selectPut(Proxy proxy, ProxyContext context, String path) {
         Matcher match = match(PATTERN_FILES, path);
         if (match != null) {
-            String bucket = match.group(1);
-            String filePath = match.group(2);
             UploadFileController controller = new UploadFileController(proxy, context);
-            return () -> controller.handle("files", bucket, filePath);
+            return () -> controller.handle(resourcePath(path));
         }
 
         match = match(PATTERN_RESOURCE, path);
         if (match != null) {
-            String resource = match.group(1);
-            String bucket = match.group(2);
-            String relativePath = match.group(3);
             ResourceController controller = new ResourceController(proxy, context, false);
-            return () -> controller.handle(resource, bucket, relativePath);
+            return () -> controller.handle(resourcePath(path));
         }
 
         return null;
@@ -336,4 +318,18 @@ private Matcher match(Pattern pattern, String path) {
         Matcher matcher = pattern.matcher(path);
         return matcher.find() ? matcher : null;
     }
+
+    private String resourcePath(String url) {
+        String prefix = "/v1/";
+
+        if (!url.startsWith(prefix)) {
+            throw new IllegalArgumentException("Resource url must start with /v1/: " + url);
+        }
+
+        if (url.startsWith("/v1/metadata/")) {
+            prefix = "/v1/metadata/";
+        }
+
+        return url.substring(prefix.length());
+    }
 }

src/main/java/com/epam/aidial/core/controller/DeleteFileController.java

@@ -45,7 +45,7 @@ protected Future<?> handle(ResourceDescription resource) {
             try {
                 Set<ResourceLink> resourceLinks = new HashSet<>();
                 resourceLinks.add(new ResourceLink(resource.getUrl()));
-                return lockService.underBucketLock(proxy, bucketLocation, () -> {
+                return lockService.underBucketLock(bucketLocation, () -> {
                     invitationService.cleanUpResourceLinks(bucketName, bucketLocation, resourceLinks);
                     shareService.revokeSharedAccess(bucketName, bucketLocation, new ResourceLinkCollection(resourceLinks));
                     storage.delete(absoluteFilePath);

src/main/java/com/epam/aidial/core/controller/DeploymentPostController.java

@@ -261,9 +261,7 @@ private void processAttachedFile(String url) {
         ApiKeyData sourceApiKeyData = context.getApiKeyData();
         ApiKeyData destApiKeyData = context.getProxyApiKeyData();
         AccessService accessService = proxy.getAccessService();
-        if (accessService.hasWriteAccess(resource, context)
-                || accessService.isSharedResource(resource, context)
-                || sourceApiKeyData.getAttachedFiles().contains(resourceUrl)) {
+        if (sourceApiKeyData.getAttachedFiles().contains(resourceUrl) || accessService.hasReadAccess(resource, context)) {
             if (resource.isFolder()) {
                 destApiKeyData.getAttachedFolders().add(resourceUrl);
             } else {
@@ -284,7 +282,7 @@ private ResourceDescription getResourceDescription(String url) {
             // skip public resource
             return null;
         }
-        return ResourceDescription.fromLink(url, proxy.getEncryptionService());
+        return ResourceDescription.fromAnyUrl(url, proxy.getEncryptionService());
     }
 
     /**

src/main/java/com/epam/aidial/core/controller/FileMetadataController.java

@@ -23,6 +23,7 @@ protected Future<?> handle(ResourceDescription resource) {
             try {
                 MetadataBase metadata = storage.listMetadata(resource);
                 if (metadata != null) {
+                    proxy.getAccessService().filterForbidden(context, resource, metadata);
                     context.respond(HttpStatus.OK, metadata);
                 } else {
                     context.respond(HttpStatus.NOT_FOUND);

src/main/java/com/epam/aidial/core/controller/InvitationController.java

@@ -54,7 +54,7 @@ public Future<?> getOrAcceptInvitation(String invitationId) {
                         if (invitationResource == null) {
                             throw new ResourceNotFoundException();
                         }
-                        return lockService.underBucketLock(proxy, invitationResource.getBucketLocation(), () -> {
+                        return lockService.underBucketLock(invitationResource.getBucketLocation(), () -> {
                             shareService.acceptSharedResources(bucket, bucketLocation, invitationId);
                             return null;
                         });
@@ -88,7 +88,7 @@ public Future<?> deleteInvitation(String invitationId) {
                 .executeBlocking(() -> {
                     String bucketLocation = BlobStorageUtil.buildInitiatorBucket(context);
                     String bucket = encryptionService.encrypt(bucketLocation);
-                    return lockService.underBucketLock(proxy, bucketLocation, () -> {
+                    return lockService.underBucketLock(bucketLocation, () -> {
                         invitationService.deleteInvitation(bucket, bucketLocation, invitationId);
                         return null;
                     });