Merge remote-tracking branch 'origin/development' into publications

# Conflicts:
#	src/main/java/com/epam/aidial/core/AiDial.java
#	src/main/java/com/epam/aidial/core/Proxy.java
#	src/main/java/com/epam/aidial/core/storage/ResourceDescription.java
#	src/main/java/com/epam/aidial/core/util/ProxyUtil.java

.gitignore

@@ -3,5 +3,5 @@
 .vscode/
 build/
 bin/
-data/
+/data/
 *.log

README.md

@@ -72,7 +72,7 @@ Static settings are used on startup and cannot be changed while application is r
 
 ### Google Cloud Storage
 
-There are two types of credentials providers supported:
+There are two types of credential providers supported:
  - User credentials. You can create a service account and authenticate using its private key obtained from Developer console
  - Temporary credentials. Application default credentials (ADC)
 
@@ -113,6 +113,35 @@ JClouds property `jclouds.oauth.credential-type` should be set `bearerTokenCrede
 }
 ```
 
+### Azure Blob Store
+
+There are two types of credential providers supported:
+- User credentials. You can create a service principle and authenticate using its secret from Azure console
+- Temporary credentials with Azure AD Workload Identity
+
+#### User credentials
+
+You should set `storage.credential` to service principle secret and `storage.identity` - service principle ID.
+
+#### Temporary credentials
+
+You should follow [instructions](https://azure.github.io/azure-workload-identity/docs/) to setup your pod in Azure k8s.
+`storage.credential` and `storage.identity` must be unset.
+
+The properties to be overridden are below:
+
+```
+{
+  "storage": {
+    "endpoint": "https://<Azure Blob storage account>.blob.core.windows.net"
+    "overrides": {
+      "jclouds.azureblob.auth": "azureAd",
+      "jclouds.oauth.credential-type": "bearerTokenCredentials"
+    }
+  }
+}
+```
+
 ### Redis
 The Redis can be used as a cache with volatile-* eviction policies:
 ```
@@ -144,7 +173,7 @@ Dynamic settings include:
 | models.<model_name>             | `type`: Model type—`chat` or `embedding`.<br />`iconUrl`: Icon path for the model on UI.<br />`description`: Brief model description.<br />`displayName`: Model name on UI.<br />`displayVersion`: Model version on UI.<br />`endpoint`: Model API for chat completions or embeddings.<br />`tokenizerModel`: Identifies the specific model whose tokenization algorithm exactly matches that of the referenced model. This is typically the name of the earliest-released model in a series of models sharing an identical tokenization algorithm (e.g. `gpt-3.5-turbo-0301`, `gpt-4-0314`, or `gpt-4-1106-vision-preview`). This parameter is essential for DIAL clients that reimplement tokenization algorithms on their side, instead of utilizing the `tokenizeEndpoint` provided by the model.<br />`features`: Model features.<br />`limits`: Model token limits.<br />`pricing`: Model pricing.<br />`upstreams`: Used for load-balancing—request is sent to model endpoint containing X-UPSTREAM-ENDPOINT and X-UPSTREAM-KEY headers. |
 | models.<model_name>.limits      | `maxPromptTokens`: maximum number of tokens in a completion request.<br />`maxCompletionTokens`: maximum number of tokens in a completion response.<br />`maxTotalTokens`: maximum number of tokens in completion request and response combined.<br />Typically either `maxTotalTokens` is specified or `maxPromptTokens` and `maxCompletionTokens`. |
 | models.<model_name>.pricing      | `unit`: the pricing units (currently `token` and `char_without_whitespace` are supported).<br />`prompt`: per-unit price for the completion request in USD.<br />`completion`: per-unit price for the completion response in USD. |
-| models.<model_name>.features    | `rateEndpoint`: endpoint for rate requests *(exposed by core as `<deployment name>/rate`)*.<br />`tokenizeEndpoint`: endpoint for requests to the model tokenizer *(exposed by core as `<deployment name>/tokenize`)*.<br />`truncatePromptEndpoint`: endpoint for truncating prompt requests *(exposed by core as `<deployment name>/truncate_prompt`)*.<br />`systemPromptSupported`: does the model support system prompt (default is `true`).<br />`toolsSupported`: does the model support tools (default is `false`).<br />`seedSupported`: does the model support `seed` request parameter (default is `false`) |
+| models.<model_name>.features    | `rateEndpoint`: endpoint for rate requests *(exposed by core as `<deployment name>/rate`)*.<br />`tokenizeEndpoint`: endpoint for requests to the model tokenizer *(exposed by core as `<deployment name>/tokenize`)*.<br />`truncatePromptEndpoint`: endpoint for truncating prompt requests *(exposed by core as `<deployment name>/truncate_prompt`)*.<br />`systemPromptSupported`: does the model support system prompt (default is `true`).<br />`toolsSupported`: does the model support tools (default is `false`).<br />`seedSupported`: does the model support `seed` request parameter (default is `false`).<br />`urlAttachmentsSupported`: does the model/application support attachments with URLs (default is `false`) |
 | models.<model_name>.upstreams   | `endpoint`: Model endpoint.<br />`key`: Your API key. |
 | keys                            | API Keys parameters:<br />`<core_key>`: Your API key. |
 | keys.<core_key>                 | `project`: Project name assigned to this key.<br />`role`: A configured role name that defines key permissions. |

build.gradle

@@ -47,10 +47,12 @@ dependencies {
     implementation 'com.auth0:jwks-rsa:0.22.1'
     implementation 'org.apache.jclouds:jclouds-allblobstore:2.5.0'
     implementation 'org.apache.jclouds.api:filesystem:2.5.0'
-    implementation 'org.redisson:redisson:3.25.1'
+    implementation 'org.redisson:redisson:3.27.0'
     implementation group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.12.663'
     implementation group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.12.663'
     implementation group: 'com.google.auth', name: 'google-auth-library-oauth2-http', version: '1.23.0'
+    implementation group: 'com.azure', name: 'azure-identity', version: '1.11.2'
+
 
 
     runtimeOnly 'com.epam.deltix:gflog-slf4j:3.0.5'

src/main/java/com/epam/aidial/core/AiDial.java

@@ -106,23 +106,19 @@ void start() throws Exception {
 
             redis = openRedis();
 
-            if (redis != null) {
-                LockService lockService = new LockService(redis);
-                resourceService = new ResourceService(vertx, redis, storage, lockService, settings("resources"), storage.getPrefix());
-                invitationService = new InvitationService(resourceService, encryptionService, settings("invitations"));
-                shareService = new ShareService(resourceService, invitationService, encryptionService);
-                publicationService = new PublicationService(encryptionService, resourceService, storage, generator, clock);
-            } else {
-                log.warn("Redis config is not found, some features may be unavailable");
-            }
+            LockService lockService = new LockService(redis);
+            resourceService = new ResourceService(vertx, redis, storage, lockService, settings("resources"), storage.getPrefix());
+            InvitationService invitationService = new InvitationService(resourceService, encryptionService, settings("invitations"));
+            ShareService shareService = new ShareService(resourceService, invitationService, encryptionService);
+            PublicationService publicationService = new PublicationService(encryptionService, resourceService, storage, generator, clock);
 
             AccessService accessService = new AccessService(encryptionService, shareService, publicationService);
             RateLimiter rateLimiter = new RateLimiter(vertx, resourceService);
 
             proxy = new Proxy(vertx, client, configStore, logStore,
                     rateLimiter, upstreamBalancer, accessTokenValidator,
                     storage, encryptionService, apiKeyStore, tokenStatsTracker, resourceService, invitationService,
-                    shareService, publicationService, accessService);
+                    shareService, publicationService, accessService, lockService);
 
             server = vertx.createHttpServer(new HttpServerOptions(settings("server"))).requestHandler(proxy);
             open(server, HttpServer::listen);
@@ -138,7 +134,7 @@ void start() throws Exception {
     private RedissonClient openRedis() throws IOException {
         JsonObject conf = settings("redis");
         if (conf.isEmpty()) {
-            return null;
+            throw new IllegalArgumentException("Redis configuration not found");
         }
 
         ConfigSupport support = new ConfigSupport();

src/main/java/com/epam/aidial/core/Proxy.java

@@ -14,6 +14,7 @@
 import com.epam.aidial.core.security.ExtractedClaims;
 import com.epam.aidial.core.service.InvitationService;
 import com.epam.aidial.core.service.PublicationService;
+import com.epam.aidial.core.service.LockService;
 import com.epam.aidial.core.service.ResourceService;
 import com.epam.aidial.core.service.ShareService;
 import com.epam.aidial.core.storage.BlobStorage;
@@ -78,6 +79,7 @@ public class Proxy implements Handler<HttpServerRequest> {
     private final ShareService shareService;
     private final PublicationService publicationService;
     private final AccessService accessService;
+    private final LockService lockService;
 
     @Override
     public void handle(HttpServerRequest request) {

src/main/java/com/epam/aidial/core/config/Features.java

@@ -11,4 +11,5 @@ public class Features {
     private Boolean systemPromptSupported;
     private Boolean toolsSupported;
     private Boolean seedSupported;
+    private Boolean urlAttachmentsSupported;
 }

src/main/java/com/epam/aidial/core/config/FileConfigStore.java

@@ -156,5 +156,8 @@ private static void setMissingFeatures(Deployment model, Features features) {
         if (modelFeatures.getSeedSupported() == null) {
             modelFeatures.setSeedSupported(features.getSeedSupported());
         }
+        if (modelFeatures.getUrlAttachmentsSupported() == null) {
+            modelFeatures.setUrlAttachmentsSupported(features.getUrlAttachmentsSupported());
+        }
     }
 }

src/main/java/com/epam/aidial/core/controller/DeleteFileController.java

@@ -4,6 +4,8 @@
 import com.epam.aidial.core.ProxyContext;
 import com.epam.aidial.core.data.ResourceLink;
 import com.epam.aidial.core.data.ResourceLinkCollection;
+import com.epam.aidial.core.service.InvitationService;
+import com.epam.aidial.core.service.LockService;
 import com.epam.aidial.core.service.ShareService;
 import com.epam.aidial.core.storage.BlobStorage;
 import com.epam.aidial.core.storage.ResourceDescription;
@@ -18,10 +20,14 @@
 public class DeleteFileController extends AccessControlBaseController {
 
     private final ShareService shareService;
+    private final InvitationService invitationService;
+    private final LockService lockService;
 
     public DeleteFileController(Proxy proxy, ProxyContext context) {
         super(proxy, context, true);
         this.shareService = proxy.getShareService();
+        this.invitationService = proxy.getInvitationService();
+        this.lockService = proxy.getLockService();
     }
 
     @Override
@@ -34,18 +40,19 @@ protected Future<?> handle(ResourceDescription resource) {
 
         BlobStorage storage = proxy.getStorage();
         Future<Void> result = proxy.getVertx().executeBlocking(() -> {
+            String bucketName = resource.getBucketName();
+            String bucketLocation = resource.getBucketLocation();
             try {
-                // clean shared access
-                // TODO remove check when redis become mandatory
-                if (shareService != null) {
-                    Set<ResourceLink> resourceLinks = new HashSet<>();
-                    resourceLinks.add(new ResourceLink(resource.getUrl()));
-                    shareService.revokeSharedAccess(resource.getBucketName(), resource.getBucketLocation(), new ResourceLinkCollection(resourceLinks));
-                }
-                storage.delete(absoluteFilePath);
-                return null;
+                Set<ResourceLink> resourceLinks = new HashSet<>();
+                resourceLinks.add(new ResourceLink(resource.getUrl()));
+                return lockService.underBucketLock(proxy, bucketLocation, () -> {
+                    invitationService.cleanUpResourceLinks(bucketName, bucketLocation, resourceLinks);
+                    shareService.revokeSharedAccess(bucketName, bucketLocation, new ResourceLinkCollection(resourceLinks));
+                    storage.delete(absoluteFilePath);
+                    return null;
+                });
             } catch (Exception ex) {
-                log.error("Failed to delete file  %s/%s".formatted(resource.getBucketName(), resource.getOriginalPath()), ex);
+                log.error("Failed to delete file  %s/%s".formatted(bucketName, resource.getOriginalPath()), ex);
                 throw new RuntimeException(ex);
             }
         });

src/main/java/com/epam/aidial/core/controller/DeploymentController.java

@@ -122,6 +122,10 @@ static FeaturesData createFeatures(Features features) {
             data.setSeed(features.getSeedSupported());
         }
 
+        if (features.getUrlAttachmentsSupported() != null) {
+            data.setUrlAttachments(features.getUrlAttachmentsSupported());
+        }
+
         return data;
     }
 }

src/main/java/com/epam/aidial/core/controller/InvitationController.java

@@ -4,10 +4,12 @@
 import com.epam.aidial.core.ProxyContext;
 import com.epam.aidial.core.security.EncryptionService;
 import com.epam.aidial.core.service.InvitationService;
+import com.epam.aidial.core.service.LockService;
 import com.epam.aidial.core.service.PermissionDeniedException;
 import com.epam.aidial.core.service.ResourceNotFoundException;
 import com.epam.aidial.core.service.ShareService;
 import com.epam.aidial.core.storage.BlobStorageUtil;
+import com.epam.aidial.core.storage.ResourceDescription;
 import com.epam.aidial.core.util.HttpStatus;
 import io.vertx.core.Future;
 
@@ -18,13 +20,15 @@ public class InvitationController {
     private final InvitationService invitationService;
     private final ShareService shareService;
     private final EncryptionService encryptionService;
+    private final LockService lockService;
 
     public InvitationController(Proxy proxy, ProxyContext context) {
         this.proxy = proxy;
         this.context = context;
         this.invitationService = proxy.getInvitationService();
         this.shareService = proxy.getShareService();
         this.encryptionService = proxy.getEncryptionService();
+        this.lockService = proxy.getLockService();
     }
 
     public Future<?> getInvitations() {
@@ -40,14 +44,20 @@ public Future<?> getInvitations() {
     }
 
     public Future<?> getOrAcceptInvitation(String invitationId) {
-        String accept = context.getRequest().getParam("accept");
-        if (accept != null) {
+        boolean accept = Boolean.parseBoolean(context.getRequest().getParam("accept"));
+        if (accept) {
             proxy.getVertx()
                     .executeBlocking(() -> {
                         String bucketLocation = BlobStorageUtil.buildInitiatorBucket(context);
                         String bucket = encryptionService.encrypt(bucketLocation);
-                        shareService.acceptSharedResources(bucket, bucketLocation, invitationId);
-                        return null;
+                        ResourceDescription invitationResource = invitationService.getInvitationResource(invitationId);
+                        if (invitationResource == null) {
+                            throw new ResourceNotFoundException();
+                        }
+                        return lockService.underBucketLock(proxy, invitationResource.getBucketLocation(), () -> {
+                            shareService.acceptSharedResources(bucket, bucketLocation, invitationId);
+                            return null;
+                        });
                     })
                     .onSuccess(ignore -> context.respond(HttpStatus.OK))
                     .onFailure(error -> {
@@ -78,8 +88,10 @@ public Future<?> deleteInvitation(String invitationId) {
                 .executeBlocking(() -> {
                     String bucketLocation = BlobStorageUtil.buildInitiatorBucket(context);
                     String bucket = encryptionService.encrypt(bucketLocation);
-                    invitationService.deleteInvitation(bucket, bucketLocation, invitationId);
-                    return null;
+                    return lockService.underBucketLock(proxy, bucketLocation, () -> {
+                        invitationService.deleteInvitation(bucket, bucketLocation, invitationId);
+                        return null;
+                    });
                 })
                 .onSuccess(ignore -> context.respond(HttpStatus.OK))
                 .onFailure(error -> {