Skip to content

Commit

Permalink
feat: Add resource type to user metadata of a blob #465 (#466)
Browse files Browse the repository at this point in the history
  • Loading branch information
@PMitrafanau
PMitrafanau authored Sep 6, 2024
1 parent b1ab791 commit c871022
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 10 deletions.
36 changes: 26 additions & 10 deletions src/main/java/com/epam/aidial/core/service/ResourceService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,14 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
import javax.annotation.Nullable;

import static com.epam.aidial.core.util.ResourceUtil.CREATED_AT_ATTRIBUTE;
import static com.epam.aidial.core.util.ResourceUtil.ETAG_ATTRIBUTE;
import static com.epam.aidial.core.util.ResourceUtil.RESOURCE_TYPE_ATTRIBUTE;
import static com.epam.aidial.core.util.ResourceUtil.UPDATED_AT_ATTRIBUTE;

@Slf4j
Expand All @@ -72,6 +73,7 @@ public class ResourceService implements AutoCloseable {
ResourceUtil.ETAG_ATTRIBUTE,
ResourceUtil.CREATED_AT_ATTRIBUTE,
ResourceUtil.UPDATED_AT_ATTRIBUTE,
RESOURCE_TYPE_ATTRIBUTE,
CONTENT_TYPE_ATTRIBUTE,
CONTENT_LENGTH_ATTRIBUTE,
SYNCED_ATTRIBUTE,
Expand Down Expand Up @@ -374,7 +376,7 @@ private ResourceItemMetadata putResource(
Long updatedAt = time();
Long createdAt = metadata == null ? updatedAt : metadata.getCreatedAt();
String newEtag = EtagBuilder.generateEtag(body);
Result result = new Result(body, newEtag, createdAt, updatedAt, contentType, (long) body.length, false);
Result result = new Result(body, newEtag, createdAt, updatedAt, contentType, (long) body.length, descriptor.getType(), false);
if (body.length <= maxSize) {
redisPut(redisKey, result);
if (metadata == null) {
Expand Down Expand Up @@ -423,7 +425,7 @@ public FileMetadata finishFileUpload(
Long createdAt = metadata == null ? updatedAt : metadata.getCreatedAt();
MultipartUpload multipartUpload = multipartData.multipartUpload;
Map<String, String> userMetadata = multipartUpload.blobMetadata().getUserMetadata();
userMetadata.putAll(toUserMetadata(multipartData.etag, createdAt, updatedAt));
userMetadata.putAll(toUserMetadata(multipartData.etag, createdAt, updatedAt, descriptor.getType()));
blobStore.completeMultipartUpload(multipartUpload, multipartData.parts);

ResourceEvent.Action action = metadata == null
Expand Down Expand Up @@ -610,6 +612,9 @@ private static Result blobToResult(Blob blob, BlobMetadata meta) {
Long updatedAt = meta.getUserMetadata().containsKey(UPDATED_AT_ATTRIBUTE)
? Long.parseLong(meta.getUserMetadata().get(UPDATED_AT_ATTRIBUTE))
: null;
ResourceType resourceType = Optional.ofNullable(meta.getUserMetadata().get(RESOURCE_TYPE_ATTRIBUTE))
.map(ResourceType::valueOf)
.orElse(null);

// Get times from blob metadata if available for files that didn't store it in user metadata
if (createdAt == null && meta.getCreationDate() != null) {
Expand All @@ -632,7 +637,7 @@ private static Result blobToResult(Blob blob, BlobMetadata meta) {
}
}

return new Result(body, etag, createdAt, updatedAt, contentType, contentLength, true);
return new Result(body, etag, createdAt, updatedAt, contentType, contentLength, resourceType, true);
}

private void blobPut(String key, Result result, boolean compress) {
Expand All @@ -643,7 +648,7 @@ private void blobPut(String key, Result result, boolean compress) {
bytes = Compression.compress(encoding, bytes);
}

Map<String, String> metadata = toUserMetadata(result.etag, result.createdAt, result.updatedAt);
Map<String, String> metadata = toUserMetadata(result.etag, result.createdAt, result.updatedAt, result.resourceType);
blobStore.store(key, result.contentType, encoding, metadata, bytes);
}

Expand Down Expand Up @@ -684,7 +689,11 @@ private Result redisGet(String key, boolean withBody) {
Long createdAt = RedisUtil.redisToLong(fields.get(ResourceUtil.CREATED_AT_ATTRIBUTE));
Long updatedAt = RedisUtil.redisToLong(fields.get(ResourceUtil.UPDATED_AT_ATTRIBUTE));

return new Result(body, etag, createdAt, updatedAt, contentType, contentLength, synced);
ResourceType resourceType = Optional.ofNullable(RedisUtil.redisToString(fields.get(RESOURCE_TYPE_ATTRIBUTE), null))
.map(ResourceType::valueOf)
.orElse(null);

return new Result(body, etag, createdAt, updatedAt, contentType, contentLength, resourceType, synced);
}

private void redisPut(String key, Result result) {
Expand All @@ -703,6 +712,9 @@ private void redisPut(String key, Result result) {
fields.put(ResourceUtil.ETAG_ATTRIBUTE, RedisUtil.stringToRedis(result.etag));
fields.put(ResourceUtil.CREATED_AT_ATTRIBUTE, RedisUtil.longToRedis(result.createdAt));
fields.put(ResourceUtil.UPDATED_AT_ATTRIBUTE, RedisUtil.longToRedis(result.updatedAt));
fields.put(RESOURCE_TYPE_ATTRIBUTE, RedisUtil.stringToRedis(Optional.ofNullable(result.resourceType)
.map(ResourceType::name)
.orElse(null)));
fields.put(CONTENT_TYPE_ATTRIBUTE, RedisUtil.stringToRedis(result.contentType));
fields.put(CONTENT_LENGTH_ATTRIBUTE, RedisUtil.longToRedis(result.contentLength));
fields.put(EXISTS_ATTRIBUTE, RedisUtil.BOOLEAN_TRUE_ARRAY);
Expand Down Expand Up @@ -753,7 +765,7 @@ public String getEtag(ResourceDescription descriptor) {
return metadata.getEtag();
}

private static Map<String, String> toUserMetadata(String etag, Long createdAt, Long updatedAt) {
private static Map<String, String> toUserMetadata(String etag, Long createdAt, Long updatedAt, ResourceType resourceType) {
Map<String, String> metadata = new HashMap<>();
metadata.put(ResourceUtil.ETAG_ATTRIBUTE, etag);
if (createdAt != null) {
Expand All @@ -762,6 +774,9 @@ private static Map<String, String> toUserMetadata(String etag, Long createdAt, L
if (updatedAt != null) {
metadata.put(ResourceUtil.UPDATED_AT_ATTRIBUTE, Long.toString(updatedAt));
}
if (resourceType != null) {
metadata.put(RESOURCE_TYPE_ATTRIBUTE, resourceType.name());
}

return metadata;
}
Expand All @@ -774,16 +789,17 @@ private record Result(
Long updatedAt,
String contentType,
Long contentLength,
ResourceType resourceType,
boolean synced) {
public static final Result DELETED_SYNCED = new Result(null, null, null, null, null, null, true);
public static final Result DELETED_NOT_SYNCED = new Result(null, null, null, null, null, null, false);
public static final Result DELETED_SYNCED = new Result(null, null, null, null, null, null, null, true);
public static final Result DELETED_NOT_SYNCED = new Result(null, null, null, null, null, null, null, false);

public boolean exists() {
return body != null;
}

public Result toStub() {
return new Result(ArrayUtils.EMPTY_BYTE_ARRAY, etag, createdAt, updatedAt, contentType, 0L, synced);
return new Result(ArrayUtils.EMPTY_BYTE_ARRAY, etag, createdAt, updatedAt, contentType, 0L, resourceType, synced);
}
}

Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/epam/aidial/core/util/ResourceUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class ResourceUtil {
public static final String ETAG_ATTRIBUTE = "etag";
public static final String CREATED_AT_ATTRIBUTE = "created_at";
public static final String UPDATED_AT_ATTRIBUTE = "updated_at";
public static final String RESOURCE_TYPE_ATTRIBUTE = "resource_type";
// Default ETag for old records
public static final String DEFAULT_ETAG = "0";

Expand Down

0 comments on commit c871022

Please sign in to comment.