Merge pull request #244 from dlcs/feature/respondAcceptedWhileIngesting

setting status code to Accepted when ingesting

src/IIIFPresentation/API.Tests/Integration/GetManifestTests.cs

@@ -24,7 +24,10 @@ namespace API.Tests.Integration;
 public class GetManifestTests : IClassFixture<PresentationAppFactory<Program>>
 {
     private readonly HttpClient httpClient;
+
     private readonly PresentationContext dbContext;
+
+    private readonly IAmazonS3 amazonS3;
     private readonly IDlcsApiClient dlcsApiClient;
     private readonly IAmazonS3 s3;
     private readonly JObject sampleAsset;
@@ -38,6 +41,8 @@ public GetManifestTests(StorageFixture storageFixture, PresentationAppFactory<Pr
             appFactory => appFactory.WithLocalStack(storageFixture.LocalStackFixture),
             services => services.AddSingleton(dlcsApiClient));
 
+        amazonS3 = storageFixture.LocalStackFixture.AWSS3ClientFactory();
+
         storageFixture.DbFixture.CleanUp();
 
         sampleAsset = JObject.Parse(
@@ -93,7 +98,7 @@ public async Task Get_IiifManifest_Flat_ReturnsManifestFromS3_DecoratedWithDbVal
         manifest.FlatId.Should().Be("FirstChildManifest");
         manifest.PublicId.Should().Be("http://localhost/1/iiif-manifest", "iiif-manifest is slug and under root");
     }
-    
+
     [Fact]
     public async Task Get_IiifManifest_Flat_ReturnsManifestFromS3_DecoratedWithPaintedResources()
     {
@@ -110,7 +115,7 @@ await s3.PutObjectAsync(new()
             Key = $"1/manifests/{id}",
             ContentBody = TestContent.ManifestJson,
         });
-        
+
         var requestMessage =
             HttpRequestMessageBuilder.GetPrivateRequest(HttpMethod.Get, $"1/manifests/{id}");
         var response = await httpClient.AsCustomer().SendAsync(requestMessage);
@@ -154,4 +159,95 @@ public async Task Get_IiifManifest_Hierarchical_ReturnsManifestFromS3()
         manifest.Id.Should().Be("http://localhost/1/iiif-manifest", "requested by hierarchical URI");
         manifest.Items.Should().HaveCount(3, "the test content contains 3 children");
     }
+
+    [Fact]
+    public async Task Get_IiifManifest_Flat_ReturnsAccepted_WhenIngesting()
+    {
+        var id = nameof(Get_IiifManifest_Flat_ReturnsAccepted_WhenIngesting);
+
+        // Arrange and Act
+        await dbContext.Manifests.AddTestManifest(id, batchId: 1);
+
+        await amazonS3.PutObjectAsync(new()
+        {
+            BucketName = LocalStackFixture.StorageBucketName,
+            Key = $"1/manifests/{id}",
+            ContentBody = TestContent.ManifestJson
+        });
+
+        await dbContext.SaveChangesAsync();
+
+        var requestMessage =
+            HttpRequestMessageBuilder.GetPrivateRequest(HttpMethod.Get, $"1/manifests/{id}");
+        var response = await httpClient.AsCustomer().SendAsync(requestMessage);
+
+        var manifest = await response.ReadAsPresentationJsonAsync<PresentationManifest>();
+
+        // Assert
+        response.StatusCode.Should().Be(HttpStatusCode.Accepted);
+        response.Headers.Should().ContainKey(HeaderNames.ETag);
+        response.Headers.Vary.Should().HaveCount(2);
+        manifest.Should().NotBeNull();
+        manifest!.Type.Should().Be("Manifest");
+        manifest.Id.Should().Be($"http://localhost/1/manifests/{id}", "requested by flat URI");
+        manifest.Items.Should().HaveCount(3, "the test content contains 3 children");
+        manifest.FlatId.Should().Be(id);
+        manifest.PublicId.Should().Be($"http://localhost/1/sm_{id}", "iiif-manifest is slug and under root");
+    }
+
+    [Fact]
+    public async Task Get_IiifManifest_Hierarchical_ReturnsNotFoundWhenIngesting()
+    {
+        // Arrange
+        var id = nameof(Get_IiifManifest_Hierarchical_ReturnsNotFoundWhenIngesting);
+        await dbContext.Manifests.AddTestManifest(id, batchId: 2);
+
+        await amazonS3.PutObjectAsync(new()
+        {
+            BucketName = LocalStackFixture.StorageBucketName,
+            Key = $"1/manifests/{id}",
+            ContentBody = TestContent.ManifestJson
+        });
+
+        await dbContext.SaveChangesAsync();
+
+        // Act
+        var response = await httpClient.GetAsync($"1/{id}");
+
+        // Assert
+        response.StatusCode.Should().Be(HttpStatusCode.NotFound);
+        response.Headers.Vary.Should().HaveCount(2);
+    }
+
+    [Fact]
+    public async Task Get_IiifManifest_Hierarchical_ReturnsOkWhenIngestingButHasIngestedBefore()
+    {
+        // Arrange
+        var id = nameof(Get_IiifManifest_Hierarchical_ReturnsOkWhenIngestingButHasIngestedBefore);
+
+        await dbContext.Manifests.AddTestManifest(id, batchId: 3, ingested: true);
+
+        await amazonS3.PutObjectAsync(new()
+        {
+            BucketName = LocalStackFixture.StorageBucketName,
+            Key = $"1/manifests/{id}",
+            ContentBody = TestContent.ManifestJson
+        });
+
+        await dbContext.SaveChangesAsync();
+
+        // Act
+        var response = await httpClient.GetAsync($"1/sm_{id}");
+
+        var manifest = await response.ReadAsPresentationJsonAsync<PresentationManifest>();
+
+        // Assert
+        response.StatusCode.Should().Be(HttpStatusCode.OK);
+        response.Headers.Should().ContainKey(HeaderNames.ETag);
+        response.Headers.Vary.Should().HaveCount(2);
+        manifest.Should().NotBeNull();
+        manifest!.Type.Should().Be("Manifest");
+        manifest.Id.Should().Be($"http://localhost/1/sm_{id}", "requested by hierarchical URI");
+        manifest.Items.Should().HaveCount(3, "the test content contains 3 children");
+    }
 }

src/IIIFPresentation/API.Tests/Integration/ModifyManifestAssetCreationTests.cs

@@ -164,7 +164,7 @@ public async Task CreateManifest_CreateSpace_ForSpacelessAssets_WhenNoSpaceHeade
             HttpRequestMessageBuilder.GetPrivateRequest(HttpMethod.Get, response.Headers.Location!.ToString());
         response = await httpClient.AsCustomer().SendAsync(requestMessage);
 
-        response.StatusCode.Should().Be(HttpStatusCode.OK);
+        response.StatusCode.Should().Be(HttpStatusCode.Accepted);
         var responseManifest = await response.ReadAsPresentationResponseAsync<PresentationManifest>();
         responseManifest.Should().NotBeNull();
         responseManifest!.PaintedResources.Should().NotBeNull();
@@ -270,6 +270,7 @@ public async Task CreateManifest_CorrectlyCreatesAssetRequests_WithSpace()
         dbManifest.Batches.Should().HaveCount(1);
         dbManifest.Batches!.First().Status.Should().Be(BatchStatus.Ingesting);
         dbManifest.Batches!.First().Id.Should().Be(batchId);
+        dbManifest.LastProcessed.Should().BeNull();
 
         var savedS3 =
             await amazonS3.GetObjectAsync(LocalStackFixture.StorageBucketName,

src/IIIFPresentation/API.Tests/Integration/ModifyManifestCreateTests.cs

@@ -467,6 +467,7 @@ public async Task CreateManifest_CreatedDBRecord()
 
         fromDatabase.Should().NotBeNull();
         fromDatabase.SpaceId.Should().BeNull("No space was requested");
+        fromDatabase.LastProcessed.Should().NotBeNull();
         hierarchy.Type.Should().Be(ResourceType.IIIFManifest);
         hierarchy.Canonical.Should().BeTrue();
     }

src/IIIFPresentation/API/Attributes/EtagCachingAttribute.cs

@@ -37,7 +37,7 @@ ResultExecutionDelegate next
         await next();
         memoryStream.Position = 0;
 
-        if (response.StatusCode is StatusCodes.Status200OK or StatusCodes.Status201Created)
+        if (response.StatusCode is StatusCodes.Status200OK or StatusCodes.Status201Created or StatusCodes.Status202Accepted)
         {
             var responseHeaders = response.GetTypedHeaders();
 
@@ -102,4 +102,4 @@ private static bool IsClientCacheValid(RequestHeaders reqHeaders, ResponseHeader
 
         return false;
     }
-}
+}

src/IIIFPresentation/API/Features/Manifest/CanvasPaintingResolver.cs

@@ -19,11 +19,8 @@ namespace API.Features.Manifest;
 public class CanvasPaintingResolver(
     IdentityManager identityManager,
     ManifestItemsParser manifestItemsParser,
-    IOptions<DlcsSettings> dlcsOptions,
     ILogger<CanvasPaintingResolver> logger)
 {
-    private DlcsSettings settings = dlcsOptions.Value;
-
     /// <summary>
     /// Generate new CanvasPainting objects for items in provided <see cref="PresentationManifest"/>
     /// </summary>
@@ -108,7 +105,6 @@ public class CanvasPaintingResolver(
                 processedCanvasPaintingIds.Add(matching.CanvasPaintingId);
             }
         }
-
         // Delete canvasPaintings from DB that are not in payload
         foreach (var toRemove in existingManifest.CanvasPaintings
                      .Where(cp => !processedCanvasPaintingIds.Contains(cp.CanvasPaintingId)).ToList())

src/IIIFPresentation/API/Features/Manifest/ManifestController.cs

@@ -9,12 +9,10 @@
 using API.Infrastructure.Helpers;
 using API.Infrastructure.Requests;
 using API.Settings;
-using Core.IIIF;
 using MediatR;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
-using Models.API.Collection;
 using Models.API.Manifest;
 
 namespace API.Features.Manifest;
@@ -45,7 +43,10 @@ public async Task<IActionResult> GetManifestFlat([FromRoute] int customerId, [Fr
                 ? SeeOther(fullPath)
                 : this.PresentationNotFound();
 
-        return Ok(entityResult.Entity);
+        return entityResult.Entity!.Ingesting
+            ? this.PresentationWithBodyResponse(Request.GetDisplayUrl(), entityResult.Entity,
+                (int)HttpStatusCode.Accepted)
+            : Ok(entityResult.Entity);
     }
 
     /// <summary>

src/IIIFPresentation/API/Features/Manifest/ManifestReadService.cs

@@ -6,6 +6,7 @@
 using DLCS.API;
 using DLCS.Exceptions;
 using Models.API.Manifest;
+using Models.Database.Collections;
 using Models.Database.General;
 using Newtonsoft.Json.Linq;
 using Repository;
@@ -67,8 +68,8 @@ public class ManifestReadService(
 
     public async Task<FetchEntityResult<PresentationManifest>> GetManifest(int customerId, string manifestId, bool pathOnly, CancellationToken cancellationToken)
     {
-        var dbManifest =
-            await dbContext.RetrieveManifestAsync(customerId, manifestId, cancellationToken: cancellationToken);
+        var dbManifest = await dbContext.RetrieveManifestAsync(customerId, manifestId, withBatches: true,
+            cancellationToken: cancellationToken);
 
         if (dbManifest == null) return FetchEntityResult<PresentationManifest>.NotFound();
 
@@ -96,6 +97,11 @@ public async Task<FetchEntityResult<PresentationManifest>> GetManifest(int custo
         manifest = manifest.SetGeneratedFields(dbManifest, pathGenerator, assets,
             m => Enumerable.Single<Hierarchy>(m.Hierarchy!, h => h.Canonical));
 
+        if (dbManifest.IsIngesting())
+        {
+            manifest.Ingesting = true;
+        }
+
         return FetchEntityResult<PresentationManifest>.Success(manifest);
     }
 }

src/IIIFPresentation/API/Features/Manifest/ManifestWriteService.cs

@@ -86,7 +86,8 @@ public async Task<PresUpdateResult> Upsert(UpsertManifestRequest request, Cancel
         try
         {
             var existingManifest =
-                await dbContext.RetrieveManifestAsync(request.CustomerId, request.ManifestId, true, true, cancellationToken);
+                await dbContext.RetrieveManifestAsync(request.CustomerId, request.ManifestId, true,
+                    withCanvasPaintings: true, cancellationToken: cancellationToken);
 
             if (existingManifest == null)
             {
@@ -279,7 +280,8 @@ await canvasPaintingResolver.GenerateCanvasPaintings(request.CustomerId, request
                 }
             ],
             CanvasPaintings = canvasPaintings,
-            SpaceId = spaceId
+            SpaceId = spaceId,
+            LastProcessed = canvasPaintings?.Any(cp => cp.AssetId != null) ?? false ? null : timeStamp
         };
 
         await dbContext.AddAsync(dbManifest, cancellationToken);

src/IIIFPresentation/API/Features/Manifest/Requests/DeleteManifest.cs

@@ -28,7 +28,8 @@ public async Task<ResultMessage<DeleteResult, DeleteManifestType>> Handle(Delete
             request.CustomerId);
 
         var manifest =
-            await dbContext.RetrieveManifestAsync(request.CustomerId, request.ManifestId, true, false, cancellationToken);
+            await dbContext.RetrieveManifestAsync(request.CustomerId, request.ManifestId, true,
+                withCanvasPaintings: false, cancellationToken: cancellationToken);
 
         if (manifest is null) return new(DeleteResult.NotFound);
 
@@ -50,4 +51,4 @@ public async Task<ResultMessage<DeleteResult, DeleteManifestType>> Handle(Delete
 
         return new(DeleteResult.Deleted);
     }
-}
+}

src/IIIFPresentation/API/Features/Manifest/Requests/GetManifest.cs

@@ -17,10 +17,10 @@ public class GetManifest(
     public bool PathOnly { get; } = pathOnly;
 }
 
-public class GetManifestHandler(
-    IManifestRead manifestRead) : IRequestHandler<GetManifest, FetchEntityResult<PresentationManifest>>
+public class GetManifestHandler(IManifestRead manifestRead) : 
+    IRequestHandler<GetManifest, FetchEntityResult<PresentationManifest>>
 {
     public Task<FetchEntityResult<PresentationManifest>> Handle(GetManifest request,
         CancellationToken cancellationToken)
-        => manifestRead.GetManifest(request.CustomerId, request.Id, request.PathOnly, cancellationToken);
+            => manifestRead.GetManifest(request.CustomerId, request.Id, request.PathOnly, cancellationToken);
 }

src/IIIFPresentation/API/Features/Manifest/Requests/GetManifestHierarchical.cs

@@ -6,6 +6,7 @@
 using Core.Streams;
 using MediatR;
 using Microsoft.Extensions.Options;
+using Models.Database.Collections;
 using Models.Database.General;
 using Repository;
 using Repository.Helpers;
@@ -32,6 +33,11 @@ public class GetManifestHierarchicalHandler(
         var flatId = request.Hierarchy.ManifestId ??
                      throw new InvalidOperationException(
                          "The differentiation of requests should prevent this from happening.");
+
+        if (!request.Hierarchy.Manifest!.LastProcessed.HasValue)
+        {
+            return null;
+        }
 
         // So db can respond while we talk to S3
         var fetchFullPath =
@@ -41,8 +47,7 @@ public class GetManifestHierarchicalHandler(
             new(settings.S3.StorageBucket, BucketHelperX.GetManifestBucketKey(request.Hierarchy.CustomerId, flatId)),
             cancellationToken);
 
-        if (objectFromS3.Stream.IsNull())
-            return null;
+        if (objectFromS3.Stream.IsNull()) return null;
 
         request.Hierarchy.FullPath = await fetchFullPath;
 

src/IIIFPresentation/API/Features/Storage/Helpers/PresentationContextX.cs

@@ -58,10 +58,11 @@ public static class PresentationContextX
     /// <param name="manifestId">The manifest to retrieve</param>
     /// <param name="tracked">Whether the resource should be tracked or not</param>
     /// <param name="withCanvasPaintings">Whether the CanvasPaintings records should be included</param>
+    /// <param name="withBatches">Whether the Batches records should be included</param>
     /// <param name="cancellationToken">A cancellation token</param>
     /// <returns>The retrieved collection</returns>
     public static Task<DbManifest?> RetrieveManifestAsync(this PresentationContext dbContext, int customerId,
-        string manifestId, bool tracked = false, bool withCanvasPaintings = true, CancellationToken cancellationToken = default)
+        string manifestId, bool tracked = false, bool withCanvasPaintings = true, bool withBatches = false, CancellationToken cancellationToken = default)
     {
         IQueryable<DbManifest> dbContextManifests = dbContext.Manifests;
 
@@ -70,6 +71,11 @@ public static class PresentationContextX
             dbContextManifests = dbContextManifests.Include(m => m.CanvasPaintings).AsSplitQuery();
         }
 
+        if (withBatches)
+        {
+            dbContextManifests = dbContextManifests.Include(m => m.Batches);
+        }
+
         return dbContextManifests.Retrieve(customerId, manifestId, tracked, cancellationToken);
     }
 

src/IIIFPresentation/API/Features/Storage/StorageController.cs

@@ -43,10 +43,12 @@ public async Task<IActionResult> GetHierarchical(int customerId, string slug = "
         {
             case ResourceType.IIIFManifest:
                 if (Request.HasShowExtraHeader() && await authenticator.ValidateRequest(Request) == AuthResult.Success)
-                    return hierarchy.ManifestId == null ? NotFound() : SeeOther($"manifests/{hierarchy.ManifestId}");
-
+                    return hierarchy.ManifestId == null
+                        ? this.PresentationNotFound()
+                        : SeeOther($"manifests/{hierarchy.ManifestId}");
+
                 var storedManifest = await mediator.Send(new GetManifestHierarchical(hierarchy));
-                return storedManifest == null ? NotFound() : Content(storedManifest, ContentTypes.V3);
+                return storedManifest == null ? this.PresentationNotFound() : Content(storedManifest, ContentTypes.V3);
 
             case ResourceType.IIIFCollection:
             case ResourceType.StorageCollection: