From de10c7fc2b1367b989c46f69b7c2075ff25c162a Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Tue, 14 Jan 2025 16:38:52 +0000 Subject: [PATCH] review comment fixes: Updating to use assetid, adding finished date and tightening up how we get assets from the generated manifest body + general fixes --- .../Converters/ManifestConverterTests.cs | 6 +- .../API.Tests/Helpers/PathGeneratorTests.cs | 18 +- .../ModifyManifestAssetCreationTests.cs | 8 +- .../API/Converters/ManifestConverter.cs | 2 +- .../Manifest/CanvasPaintingResolver.cs | 3 +- .../API/Helpers/PathGenerator.cs | 18 +- .../BatchCompletionMessageHandlerTests.cs | 33 +- .../Helpers/ManifestMergerTests.cs | 52 ++- .../BatchCompletionMessageHandler.cs | 90 ++-- .../Helpers/ManifestMerger.cs | 10 +- .../API/DlcsOrchestratorClientTests.cs | 12 +- .../DLCS/API/DlcsApiClient.cs | 4 - .../DLCS/API/DlcsHttpContent.cs | 25 +- .../DLCS/API/DlcsOrchestratorClient.cs | 12 +- src/IIIFPresentation/DLCS/DLCS.csproj | 1 - src/IIIFPresentation/DLCS/DlcsSettings.cs | 11 +- src/IIIFPresentation/DLCS/Models/AllImages.cs | 23 - src/IIIFPresentation/DLCS/Models/AssetId.cs | 51 --- .../DLCS/ServiceCollectionX.cs | 4 +- src/IIIFPresentation/Models/DLCS/AssetId.cs | 82 ++++ .../Models/Database/CanvasPainting.cs | 3 +- .../Models/Database/General/Batch.cs | 9 +- ...seAssetIdAndAddFinishedToBatch.Designer.cs | 412 ++++++++++++++++++ ...4154941_useAssetIdAndAddFinishedToBatch.cs | 29 ++ .../PresentationContextModelSnapshot.cs | 4 + .../Repository/PresentationContext.cs | 4 + 26 files changed, 701 insertions(+), 225 deletions(-) delete mode 100644 src/IIIFPresentation/DLCS/Models/AllImages.cs delete mode 100644 src/IIIFPresentation/DLCS/Models/AssetId.cs create mode 100644 src/IIIFPresentation/Models/DLCS/AssetId.cs create mode 100644 src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.Designer.cs create mode 100644 src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.cs diff --git a/src/IIIFPresentation/API.Tests/Converters/ManifestConverterTests.cs b/src/IIIFPresentation/API.Tests/Converters/ManifestConverterTests.cs index 4e9eef60..71323032 100644 --- a/src/IIIFPresentation/API.Tests/Converters/ManifestConverterTests.cs +++ b/src/IIIFPresentation/API.Tests/Converters/ManifestConverterTests.cs @@ -3,7 +3,7 @@ using API.Tests.Helpers; using Models.API.Manifest; using Models.Database.General; -using Newtonsoft.Json.Linq; +using Models.DLCS; using CanvasPainting = Models.Database.CanvasPainting; using DBManifest = Models.Database.Collections.Manifest; @@ -169,7 +169,7 @@ public void SetGeneratedFields_SetsCanvasPainting_IfPresent() Id = "the-canvas", ChoiceOrder = 10, CanvasOrder = 100, - AssetId = "1/2/assetId" + AssetId = new AssetId(1, 2, "assetId") } ] }; @@ -184,7 +184,7 @@ public void SetGeneratedFields_SetsCanvasPainting_IfPresent() paintedResource.CanvasPainting.CanvasOrder.Should().Be(100); paintedResource.CanvasPainting.CanvasOriginalId.Should().Be("http://example.test/canvas1"); - paintedResource.Asset.GetValue("assetId").ToString().Should().Be("https://dlcs.test/customers/1/spaces/2/images/assetId"); + paintedResource.Asset.GetValue("@id").ToString().Should().Be("https://dlcs.test/customers/1/spaces/2/images/assetId"); } [Fact] diff --git a/src/IIIFPresentation/API.Tests/Helpers/PathGeneratorTests.cs b/src/IIIFPresentation/API.Tests/Helpers/PathGeneratorTests.cs index 682dbe19..405b5cdb 100644 --- a/src/IIIFPresentation/API.Tests/Helpers/PathGeneratorTests.cs +++ b/src/IIIFPresentation/API.Tests/Helpers/PathGeneratorTests.cs @@ -3,6 +3,7 @@ using Models.Database; using Models.Database.Collections; using Models.Database.General; +using Models.DLCS; namespace API.Tests.Helpers; @@ -385,7 +386,7 @@ public void GenerateAssetUri_Correct_IfCanvasPaintingHasAssetWithThreeSlashes() { Id = "hello", CustomerId = 123, - AssetId = "5/4/12" + AssetId = new AssetId(5, 4, "12") }; var expected = "https://dlcs.test/customers/5/spaces/4/images/12"; @@ -393,23 +394,10 @@ public void GenerateAssetUri_Correct_IfCanvasPaintingHasAssetWithThreeSlashes() pathGenerator.GenerateAssetUri(manifest).Should().Be(expected); } - [Fact] - public void GenerateAssetUri_Null_IfCanvasPaintingHasAssetWithNotThreeSlashes() - { - var manifest = new CanvasPainting() - { - Id = "hello", - CustomerId = 123, - AssetId = "5/4" - }; - - pathGenerator.GenerateAssetUri(manifest).Should().BeNull(); - } - [Fact] public void GenerateAssetUri_Null_IfCanvasPaintingDoesNotHaveAsset() { - var manifest = new CanvasPainting() + var manifest = new CanvasPainting { Id = "hello", CustomerId = 123, diff --git a/src/IIIFPresentation/API.Tests/Integration/ModifyManifestAssetCreationTests.cs b/src/IIIFPresentation/API.Tests/Integration/ModifyManifestAssetCreationTests.cs index 9f7b51af..be606802 100644 --- a/src/IIIFPresentation/API.Tests/Integration/ModifyManifestAssetCreationTests.cs +++ b/src/IIIFPresentation/API.Tests/Integration/ModifyManifestAssetCreationTests.cs @@ -187,7 +187,7 @@ public async Task CreateManifest_CorrectlyCreatesAssetRequests_WithSpace() dbManifest.CanvasPaintings.Should().HaveCount(1); dbManifest.CanvasPaintings!.First().Label!.First().Value[0].Should().Be("canvas testing"); // space comes from the asset - dbManifest.CanvasPaintings!.First().AssetId.Should().Be($"{Customer}/{space}/{assetId}"); + dbManifest.CanvasPaintings!.First().AssetId.ToString().Should().Be($"{Customer}/{space}/{assetId}"); dbManifest.Batches.Should().HaveCount(1); dbManifest.Batches!.First().Status.Should().Be(BatchStatus.Ingesting); dbManifest.Batches!.First().Id.Should().Be(batchId); @@ -288,7 +288,7 @@ public async Task CreateManifest_CorrectlyCreatesAssetRequests_WithoutSpace() dbManifest.CanvasPaintings.Should().HaveCount(1); dbManifest.CanvasPaintings!.First().Label!.First().Value[0].Should().Be("canvas testing"); // space added using the manifest space - dbManifest.CanvasPaintings!.First().AssetId.Should() + dbManifest.CanvasPaintings!.First().AssetId.ToString().Should() .Be($"{Customer}/{NewlyCreatedSpace}/{assetId}"); dbManifest.Batches.Should().HaveCount(1); dbManifest.Batches!.First().Status.Should().Be(BatchStatus.Ingesting); @@ -383,7 +383,7 @@ await amazonS3.GetObjectAsync(LocalStackFixture.StorageBucketName, dbManifest.CanvasPaintings.Should().HaveCount(1); dbManifest.CanvasPaintings!.First().Label?.First().Should().BeNull(); // space added using the manifest space - dbManifest.CanvasPaintings!.First().AssetId.Should() + dbManifest.CanvasPaintings!.First().AssetId.ToString().Should() .Be($"{Customer}/{NewlyCreatedSpace}/{assetId}"); dbManifest.Batches.Should().HaveCount(1); dbManifest.Batches!.First().Status.Should().Be(BatchStatus.Ingesting); @@ -595,7 +595,7 @@ public async Task CreateManifest_CorrectlyCreatesAssetRequests_WhenMultipleAsset foreach (var canvasPainting in dbManifest.CanvasPaintings!) { canvasPainting.CanvasOrder.Should().Be(currentCanvasOrder); - canvasPainting.AssetId.Should() + canvasPainting.AssetId.ToString().Should() .Be($"{Customer}/{NewlyCreatedSpace}/testAssetByPresentation-multipleAssets-{currentCanvasOrder}"); currentCanvasOrder++; } diff --git a/src/IIIFPresentation/API/Converters/ManifestConverter.cs b/src/IIIFPresentation/API/Converters/ManifestConverter.cs index 191fb76a..4389a1d2 100644 --- a/src/IIIFPresentation/API/Converters/ManifestConverter.cs +++ b/src/IIIFPresentation/API/Converters/ManifestConverter.cs @@ -67,7 +67,7 @@ public static PresentationManifest SetGeneratedFields(this PresentationManifest }, Asset = cp.AssetId == null ? null : new JObject { - ["assetId"] = pathGenerator.GenerateAssetUri(cp)?.ToString() + ["@id"] = pathGenerator.GenerateAssetUri(cp)?.ToString() } }).ToList(); } diff --git a/src/IIIFPresentation/API/Features/Manifest/CanvasPaintingResolver.cs b/src/IIIFPresentation/API/Features/Manifest/CanvasPaintingResolver.cs index 1130f7b6..2e92971d 100644 --- a/src/IIIFPresentation/API/Features/Manifest/CanvasPaintingResolver.cs +++ b/src/IIIFPresentation/API/Features/Manifest/CanvasPaintingResolver.cs @@ -10,6 +10,7 @@ using Models.API.General; using Models.API.Manifest; using Models.Database; +using Models.DLCS; using Newtonsoft.Json.Linq; using Repository.Manifests; using CanvasPainting = Models.Database.CanvasPainting; @@ -217,7 +218,7 @@ public class CanvasPaintingResolver( Created = DateTime.UtcNow, CustomerId = customerId, CanvasOrder = count, - AssetId = $"{customerId}/{space}/{id}", + AssetId = AssetId.FromString($"{customerId}/{space}/{id}"), ChoiceOrder = -1, Ingesting = true }; diff --git a/src/IIIFPresentation/API/Helpers/PathGenerator.cs b/src/IIIFPresentation/API/Helpers/PathGenerator.cs index ae874b3b..d40c9327 100644 --- a/src/IIIFPresentation/API/Helpers/PathGenerator.cs +++ b/src/IIIFPresentation/API/Helpers/PathGenerator.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Options; using Models.Database.Collections; using Models.Database.General; +using Models.DLCS; using CanvasPainting = Models.Database.CanvasPainting; namespace API.Helpers; @@ -92,22 +93,11 @@ public string GenerateCanvasId(CanvasPainting canvasPainting) public Uri? GenerateAssetUri(CanvasPainting canvasPainting) { - if (string.IsNullOrEmpty(canvasPainting.AssetId)) return null; - - AssetId assetId; - - try - { - assetId = AssetId.FromString(canvasPainting.AssetId); - } - catch // swallow error as it's not needed - { - return null; - } - + if (canvasPainting.AssetId == null) return null; + var uriBuilder = new UriBuilder(dlcsSettings.ApiUri) { - Path = $"/customers/{assetId.Customer}/spaces/{assetId.Space}/images/{assetId.Asset}", + Path = $"/customers/{canvasPainting.AssetId.Customer}/spaces/{canvasPainting.AssetId.Space}/images/{canvasPainting.AssetId.Asset}", }; return uriBuilder.Uri; } diff --git a/src/IIIFPresentation/BackgroundHandler.Tests/BatchCompletion/BatchCompletionMessageHandlerTests.cs b/src/IIIFPresentation/BackgroundHandler.Tests/BatchCompletion/BatchCompletionMessageHandlerTests.cs index a745ebce..2c3e0719 100644 --- a/src/IIIFPresentation/BackgroundHandler.Tests/BatchCompletion/BatchCompletionMessageHandlerTests.cs +++ b/src/IIIFPresentation/BackgroundHandler.Tests/BatchCompletion/BatchCompletionMessageHandlerTests.cs @@ -4,7 +4,6 @@ using BackgroundHandler.Tests.infrastructure; using DLCS; using DLCS.API; -using DLCS.Models; using FakeItEasy; using FluentAssertions; using IIIF; @@ -17,6 +16,7 @@ using Microsoft.Extensions.Options; using Models.Database.Collections; using Models.Database.General; +using Models.DLCS; using Repository; using Test.Helpers.Helpers; using Test.Helpers.Integration; @@ -34,6 +34,7 @@ public class BatchCompletionMessageHandlerTests private readonly BatchCompletionMessageHandler sut; private readonly IDlcsOrchestratorClient dlcsClient; private readonly IIIIFS3Service iiifS3; + private readonly DlcsSettings dlcsSettings; private readonly int customerId = 1; public BatchCompletionMessageHandlerTests(PresentationContextFixture dbFixture) @@ -42,7 +43,13 @@ public BatchCompletionMessageHandlerTests(PresentationContextFixture dbFixture) dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.TrackAll; dlcsClient = A.Fake(); iiifS3 = A.Fake(); - sut = new BatchCompletionMessageHandler(dbFixture.DbContext, dlcsClient, iiifS3, + dlcsSettings = new DlcsSettings() + { + ApiUri = new Uri("https://localhost:5000"), + OrchestratorUri = new Uri("https://localhost:5000") + }; + + sut = new BatchCompletionMessageHandler(dbFixture.DbContext, dlcsClient, Options.Create(dlcsSettings), iiifS3, new NullLogger()); } @@ -64,7 +71,7 @@ public async Task HandleMessage_DoesNotUpdateAnything_WhenBatchNotTracked() // Act and Assert (await sut.HandleMessage(message, CancellationToken.None)).Should().BeTrue(); - A.CallTo(() => dlcsClient.RetrieveImagesForManifest(A._, A>._, A._)) + A.CallTo(() => dlcsClient.RetrieveAssetsForManifest(A._, A>._, A._)) .MustNotHaveHappened(); } @@ -106,14 +113,14 @@ public async Task HandleMessage_UpdatesBatchedImages_WhenBatchTracked() var message = GetMessage(batchMessage); - A.CallTo(() => dlcsClient.RetrieveImagesForManifest(A._, A>._, A._)) + A.CallTo(() => dlcsClient.RetrieveAssetsForManifest(A._, A>._, A._)) .Returns(new IIIF.Presentation.V3.Manifest { Items = new List { new () { - Id = fullAssetId.ToString(), + Id = $"{dlcsSettings.OrchestratorUri}/iiif-img/{fullAssetId}/canvas/c/1", Width = 100, Height = 100, Annotations = new List @@ -149,16 +156,17 @@ public async Task HandleMessage_UpdatesBatchedImages_WhenBatchTracked() // Assert handleMessage.Should().BeTrue(); - A.CallTo(() => dlcsClient.RetrieveImagesForManifest(A._, A>._, A._)) + A.CallTo(() => dlcsClient.RetrieveAssetsForManifest(A._, A>._, A._)) .MustHaveHappened(); var batch = dbContext.Batches.Single(b => b.Id == batchId); batch.Status.Should().Be(BatchStatus.Completed); - batch.Processed.Should().BeCloseTo(finished, TimeSpan.FromSeconds(10)); - var canvasPainting = dbContext.CanvasPaintings.Single(c => c.AssetId == fullAssetId.ToString()); + batch.Finished.Should().BeCloseTo(finished, TimeSpan.FromSeconds(10)); + batch.Processed.Should().BeCloseTo(DateTime.UtcNow, TimeSpan.FromSeconds(10)); + var canvasPainting = dbContext.CanvasPaintings.Single(c => c.AssetId == fullAssetId); canvasPainting.Ingesting.Should().BeFalse(); canvasPainting.StaticWidth.Should().Be(100); canvasPainting.StaticHeight.Should().Be(100); - canvasPainting.AssetId.Should() + canvasPainting.AssetId.ToString().Should() .Be(fullAssetId.ToString()); } @@ -201,8 +209,10 @@ public async Task HandleMessage_DoesNotUpdateBatchedImages_WhenAnotherBatchWaiti // Act and Assert (await sut.HandleMessage(message, CancellationToken.None)).Should().BeTrue(); - A.CallTo(() => dlcsClient.RetrieveImagesForManifest(A._, A>._, A._)) + A.CallTo(() => dlcsClient.RetrieveAssetsForManifest(A._, A>._, A._)) .MustNotHaveHappened(); + var batch = await dbContext.Batches.SingleOrDefaultAsync(b => b.Id == batchId); + batch.Status.Should().Be(BatchStatus.Completed); } private Manifest CreateManifest(string manifestId, string slug, string assetId, int space, int batchId) @@ -227,7 +237,7 @@ private Manifest CreateManifest(string manifestId, string slug, string assetId, CanvasPaintings = [ new CanvasPainting { - AssetId = $"{customerId}/{space}/{assetId}", + AssetId = new AssetId(customerId, space, assetId), CanvasOrder = 0, ChoiceOrder = 1, CustomerId = customerId, @@ -238,6 +248,7 @@ private Manifest CreateManifest(string manifestId, string slug, string assetId, new Batch { Id = batchId, + ManifestId = manifestId, CustomerId = customerId, Submitted = DateTime.UtcNow, Status = BatchStatus.Ingesting diff --git a/src/IIIFPresentation/BackgroundHandler.Tests/Helpers/ManifestMergerTests.cs b/src/IIIFPresentation/BackgroundHandler.Tests/Helpers/ManifestMergerTests.cs index 41e9009a..d4bac824 100644 --- a/src/IIIFPresentation/BackgroundHandler.Tests/Helpers/ManifestMergerTests.cs +++ b/src/IIIFPresentation/BackgroundHandler.Tests/Helpers/ManifestMergerTests.cs @@ -6,6 +6,7 @@ using IIIF.Presentation.V3.Content; using IIIF.Presentation.V3.Strings; using Models.Database; +using Models.DLCS; using Canvas = IIIF.Presentation.V3.Canvas; using Manifest = IIIF.Presentation.V3.Manifest; @@ -18,11 +19,15 @@ public void Merge_MergesBlankManifestWithGeneratedManifest() { // Arrange var blankManifest = new Manifest(); - var generatedManifest = GeneratedManifest(nameof(Merge_MergesBlankManifestWithGeneratedManifest)); + var assetId = $"1/2/{nameof(Merge_MergesBlankManifestWithGeneratedManifest)}"; + + var generatedManifest = GeneratedManifest(assetId); + var itemDictionary = generatedManifest.Items.ToDictionary(i => AssetId.FromString(i.Id), i => i); // Act - var mergedManifest = ManifestMerger.Merge(blankManifest, generatedManifest, - GenerateCanvasPaintings([nameof(Merge_MergesBlankManifestWithGeneratedManifest)])); + var mergedManifest = ManifestMerger.Merge(blankManifest, + GenerateCanvasPaintings([assetId]), itemDictionary, + generatedManifest.Thumbnail); // Assert mergedManifest.Items.Count.Should().Be(1); @@ -38,14 +43,17 @@ public void Merge_MergesBlankManifestWithGeneratedManifest() public void Merge_MergesFullManifestWithGeneratedManifest() { // Arrange - var blankManifest = GeneratedManifest(nameof(Merge_MergesFullManifestWithGeneratedManifest)); + var assetId = $"1/2/{nameof(Merge_MergesFullManifestWithGeneratedManifest)}"; + var blankManifest = GeneratedManifest(assetId); blankManifest.Items[0].Width = 200; blankManifest.Items[0].Height = 200; - var generatedManifest = GeneratedManifest(nameof(Merge_MergesFullManifestWithGeneratedManifest)); + var generatedManifest = GeneratedManifest(assetId); + var itemDictionary = generatedManifest.Items.ToDictionary(i => AssetId.FromString(i.Id), i => i); // Act - var mergedManifest = ManifestMerger.Merge(blankManifest, generatedManifest, - GenerateCanvasPaintings([nameof(Merge_MergesFullManifestWithGeneratedManifest)])); + var mergedManifest = ManifestMerger.Merge(blankManifest, + GenerateCanvasPaintings([assetId]), itemDictionary, + generatedManifest.Thumbnail); // Assert mergedManifest.Items.Count.Should().Be(1); @@ -58,16 +66,19 @@ public void Merge_MergesFullManifestWithGeneratedManifest() public void Merge_ShouldNotUpdateAttachedManifestThumbnail() { // Arrange - var blankManifest = GeneratedManifest(nameof(Merge_ShouldNotUpdateAttachedManifestThumbnail)); + var assetId = $"1/2/{nameof(Merge_ShouldNotUpdateAttachedManifestThumbnail)}"; + var blankManifest = GeneratedManifest(assetId); blankManifest.Items[0].Width = 200; blankManifest.Items[0].Height = 200; blankManifest.Thumbnail.Add(GenerateImage()); blankManifest.Thumbnail[0].Service[0].Id = "generatedId"; - var generatedManifest = GeneratedManifest(nameof(Merge_ShouldNotUpdateAttachedManifestThumbnail)); + var generatedManifest = GeneratedManifest(assetId); + var itemDictionary = generatedManifest.Items.ToDictionary(i => AssetId.FromString(i.Id), i => i); // Act - var mergedManifest = ManifestMerger.Merge(blankManifest, generatedManifest, - GenerateCanvasPaintings([nameof(Merge_ShouldNotUpdateAttachedManifestThumbnail)])); + var mergedManifest = ManifestMerger.Merge(blankManifest, + GenerateCanvasPaintings([assetId]), itemDictionary, + generatedManifest.Thumbnail); // Assert mergedManifest.Items.Count.Should().Be(1); @@ -82,34 +93,35 @@ public void Merge_CorrectlyOrdersMultipleItems() { // Arrange var blankManifest = new Manifest(); - var generatedManifest = GeneratedManifest($"{nameof(Merge_ShouldNotUpdateAttachedManifestThumbnail)}_1"); - generatedManifest.Items.Add(GenerateCanvas($"{nameof(Merge_ShouldNotUpdateAttachedManifestThumbnail)}_2")); + var generatedManifest = GeneratedManifest($"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_1"); + generatedManifest.Items.Add(GenerateCanvas($"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_2")); + var itemDictionary = generatedManifest.Items.ToDictionary(i => AssetId.FromString(i.Id), i => i); var canvasPaintings = GenerateCanvasPaintings([ - $"{nameof(Merge_ShouldNotUpdateAttachedManifestThumbnail)}_1", - $"{nameof(Merge_ShouldNotUpdateAttachedManifestThumbnail)}_2" + $"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_1", + $"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_2" ]); canvasPaintings[0].CanvasOrder = 1; canvasPaintings[1].CanvasOrder = 0; // Act - var mergedManifest = ManifestMerger.Merge(blankManifest, generatedManifest, - canvasPaintings); + var mergedManifest = + ManifestMerger.Merge(blankManifest, canvasPaintings, itemDictionary, generatedManifest.Thumbnail); // Assert mergedManifest.Items.Count.Should().Be(2); mergedManifest.Thumbnail[0].Service[0].Id.Should().Be("imageId"); mergedManifest.Thumbnail.Count.Should().Be(1); // order flipped due to canvas order - mergedManifest.Items[0].Id.Should().Be($"{nameof(Merge_ShouldNotUpdateAttachedManifestThumbnail)}_2"); - mergedManifest.Items[1].Id.Should().Be($"{nameof(Merge_ShouldNotUpdateAttachedManifestThumbnail)}_1"); + mergedManifest.Items[0].Id.Should().Be($"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_2"); + mergedManifest.Items[1].Id.Should().Be($"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_1"); } private List GenerateCanvasPaintings(List idList) { var canvasOrder = 0; - return idList.Select(id => new CanvasPainting{Id = id, AssetId = id, CanvasOrder = canvasOrder++}).ToList(); + return idList.Select(id => new CanvasPainting{Id = id, AssetId = AssetId.FromString(id), CanvasOrder = canvasOrder++}).ToList(); } private Manifest GeneratedManifest(string id) diff --git a/src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessageHandler.cs b/src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessageHandler.cs index 804c2377..f502bdc3 100644 --- a/src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessageHandler.cs +++ b/src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessageHandler.cs @@ -6,12 +6,12 @@ using Core.IIIF; using DLCS; using DLCS.API; -using DLCS.Models; using IIIF.Presentation.V3; using IIIF.Presentation.V3.Content; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using Models.Database.General; +using Models.DLCS; using Repository; using Batch = Models.Database.General.Batch; @@ -20,11 +20,13 @@ namespace BackgroundHandler.BatchCompletion; public class BatchCompletionMessageHandler( PresentationContext dbContext, IDlcsOrchestratorClient dlcsOrchestratorClient, + IOptions dlcsOptions, IIIIFS3Service iiifS3, ILogger logger) : IMessageHandler { private static readonly JsonSerializerOptions JsonSerializerOptions = new(JsonSerializerDefaults.Web); + private readonly DlcsSettings dlcsSettings = dlcsOptions.Value; public async Task HandleMessage(QueueMessage message, CancellationToken cancellationToken) { @@ -50,65 +52,85 @@ private async Task UpdateAssetsIfRequired(BatchCompletionMessage batchCompletion { var batch = await dbContext.Batches.Include(b => b.Manifest) .ThenInclude(m => m.CanvasPaintings) - .FirstOrDefaultAsync(b => b.Id == batchCompletionMessage.Id, cancellationToken); + .SingleOrDefaultAsync(b => b.Id == batchCompletionMessage.Id, cancellationToken); // batch isn't tracked by presentation, so nothing to do if (batch == null) return; - // Other batches haven't completed, so no point populating items until all are complete - if (await dbContext.Batches.AnyAsync(b => b.ManifestId == batch.ManifestId && - b.Status != BatchStatus.Completed && - b.Id != batch.Id, cancellationToken)) + // Other batches haven't completed, so no point populating items until all are complete + if (await dbContext.Batches.AnyAsync(b => b.ManifestId == batch.ManifestId && + b.Status != BatchStatus.Completed && + b.Id != batch.Id, cancellationToken)) + { + CompleteBatch(batch, batchCompletionMessage.Finished); + } + else + { + logger.LogInformation( + "Attempting to complete assets in batch {BatchId} for customer {CustomerId} with the manifest {ManifestId}", + batch.Id, batch.CustomerId, batch.ManifestId); + + var batches = dbContext.Batches.Where(b => b.ManifestId == batch.ManifestId).Select(b => b.Id).ToList(); + + var generatedManifest = + await dlcsOrchestratorClient.RetrieveAssetsForManifest(batch.CustomerId, batches, + cancellationToken); + + Dictionary itemDictionary; + + try { - CompleteBatch(batch, batchCompletionMessage.Finished); + itemDictionary = generatedManifest.Items.Select(i => + new KeyValuePair( + AssetId.FromString(i.Id!.Remove(i.Id.IndexOf("/canvas", StringComparison.Ordinal)) + .Remove(0, $"{dlcsSettings.OrchestratorUri!.ToString()}/iii-img/".Length + 1)), i)) + .ToDictionary(); } - else + catch (Exception e) { - logger.LogInformation( - "Attempting to complete assets in batch {BatchId} for customer {CustomerId} with the manifest {ManifestId}", - batch.Id, batch.CustomerId, batch.ManifestId); - - var batches = dbContext.Batches.Where(b => b.ManifestId == batch.ManifestId).Select(b => b.Id).ToList(); - - var generatedManifest = - await dlcsOrchestratorClient.RetrieveImagesForManifest(batch.CustomerId, batches, - cancellationToken); - - UpdateCanvasPaintings(generatedManifest, batch); - CompleteBatch(batch, batchCompletionMessage.Finished); - await UpdateManifestInS3(generatedManifest, batch, cancellationToken); + logger.LogError(e, "Error retrieving the canvas id from an item in {ManifestId}", generatedManifest?.Id); + throw; } - await dbContext.SaveChangesAsync(cancellationToken); - logger.LogTrace("updating batch {BatchId} has been completed", batch.Id); + UpdateCanvasPaintings(generatedManifest, batch, itemDictionary!); + CompleteBatch(batch, batchCompletionMessage.Finished); + await UpdateManifestInS3(generatedManifest.Thumbnail, itemDictionary, batch, cancellationToken); + } + + await dbContext.SaveChangesAsync(cancellationToken); + logger.LogTrace("updating batch {BatchId} has been completed", batch.Id); } - private async Task UpdateManifestInS3(Manifest generatedManifest, Batch batch, + private async Task UpdateManifestInS3(List? thumbnail, Dictionary itemDictionary, Batch batch, CancellationToken cancellationToken = default) { var manifest = await iiifS3.ReadIIIFFromS3(batch.Manifest!, cancellationToken); + + var mergedManifest = ManifestMerger.Merge(manifest.ThrowIfNull(nameof(manifest)), + batch.Manifest?.CanvasPaintings, itemDictionary, thumbnail); - var mergedManifest = ManifestMerger.Merge(manifest, generatedManifest, batch.Manifest?.CanvasPaintings); - manifest.ThrowIfNull("Failed to retrieve manifest"); - - await iiifS3.SaveIIIFToS3(mergedManifest, batch.Manifest, "", cancellationToken); + await iiifS3.SaveIIIFToS3(mergedManifest, batch.Manifest!, "", cancellationToken); } private void CompleteBatch(Batch batch, DateTime finished) { - batch.Processed = finished; + batch.Processed = DateTime.UtcNow; + batch.Finished = finished; batch.Status = BatchStatus.Completed; } - private void UpdateCanvasPaintings(Manifest generatedManifest, Batch batch) + private void UpdateCanvasPaintings(Manifest generatedManifest, Batch batch, Dictionary itemDictionary) { - if (batch.Manifest?.CanvasPaintings == null) return; + if (batch.Manifest?.CanvasPaintings == null) + { + logger.LogWarning( + "Received a batch completion notification with no canvas paintings on the batch {BatchId}", batch.Id); + return; + } foreach (var canvasPainting in batch.Manifest.CanvasPaintings) { - var assetId = AssetId.FromString(canvasPainting.AssetId!); - - var item = generatedManifest.Items?.FirstOrDefault(i => i.Id!.Contains(assetId.ToString())); + itemDictionary.TryGetValue(canvasPainting.AssetId!, out var item); if (item == null) continue; diff --git a/src/IIIFPresentation/BackgroundHandler/Helpers/ManifestMerger.cs b/src/IIIFPresentation/BackgroundHandler/Helpers/ManifestMerger.cs index 1943d794..4a847c0e 100644 --- a/src/IIIFPresentation/BackgroundHandler/Helpers/ManifestMerger.cs +++ b/src/IIIFPresentation/BackgroundHandler/Helpers/ManifestMerger.cs @@ -1,5 +1,7 @@ using Core.Helpers; using IIIF.Presentation.V3; +using IIIF.Presentation.V3.Content; +using Models.DLCS; using CanvasPainting = Models.Database.CanvasPainting; namespace BackgroundHandler.Helpers; @@ -9,7 +11,8 @@ public static class ManifestMerger /// /// Merges a generated DLCS manifest with the current manifest in S3 /// - public static Manifest Merge(Manifest baseManifest, Manifest generatedManifest, List? canvasPaintings) + public static Manifest Merge(Manifest baseManifest, List? canvasPaintings, + Dictionary itemDictionary, List? thumbnail) { if (baseManifest.Items == null) baseManifest.Items = []; @@ -19,8 +22,7 @@ public static Manifest Merge(Manifest baseManifest, Manifest generatedManifest, // We want to use the canvas order set when creating assets, rather than the foreach (var canvasPainting in orderedCanvasPaintings) { - var generatedItem = - generatedManifest.Items?.SingleOrDefault(gm => gm.Id!.Contains(canvasPainting.AssetId!)); + itemDictionary.TryGetValue(canvasPainting.AssetId!, out var generatedItem); if (generatedItem == null) continue; @@ -41,7 +43,7 @@ public static Manifest Merge(Manifest baseManifest, Manifest generatedManifest, if (baseManifest.Thumbnail.IsNullOrEmpty()) { - baseManifest.Thumbnail = generatedManifest.Thumbnail; + baseManifest.Thumbnail = thumbnail; } return baseManifest; diff --git a/src/IIIFPresentation/DLCS.Tests/API/DlcsOrchestratorClientTests.cs b/src/IIIFPresentation/DLCS.Tests/API/DlcsOrchestratorClientTests.cs index 27765f43..89e0b151 100644 --- a/src/IIIFPresentation/DLCS.Tests/API/DlcsOrchestratorClientTests.cs +++ b/src/IIIFPresentation/DLCS.Tests/API/DlcsOrchestratorClientTests.cs @@ -18,10 +18,10 @@ public async Task RetrieveImagesForManifest_Throws_IfDownstreamNon200_NoReturned { using var stub = new ApiStub(); const int customerId = 3; - stub.Get($"/iiif-resource/{customerId}/batch-query/1,2", (_, _) => string.Empty).StatusCode((int)httpStatusCode); + stub.Get($"/iiif-resource/v3/{customerId}/batch-query/1,2", (_, _) => string.Empty).StatusCode((int)httpStatusCode); var sut = GetClient(stub); - Func action = () => sut.RetrieveImagesForManifest(customerId, [1, 2], CancellationToken.None); + Func action = () => sut.RetrieveAssetsForManifest(customerId, [1, 2], CancellationToken.None); await action.Should().ThrowAsync().WithMessage("Could not find a DlcsError in response"); } @@ -33,11 +33,11 @@ public async Task RetrieveImagesForManifest_Throws_IfDownstreamNon200_WithReturn { using var stub = new ApiStub(); const int customerId = 4; - stub.Get($"/iiif-resource/{customerId}/batch-query/1,2", (_, _) => "{\"description\":\"I am broken\"}") + stub.Get($"/iiif-resource/v3/{customerId}/batch-query/1,2", (_, _) => "{\"description\":\"I am broken\"}") .StatusCode((int)httpStatusCode); var sut = GetClient(stub); - Func action = () => sut.RetrieveImagesForManifest(customerId, [1, 2], CancellationToken.None); + Func action = () => sut.RetrieveAssetsForManifest(customerId, [1, 2], CancellationToken.None); await action.Should().ThrowAsync().WithMessage("I am broken"); } @@ -46,13 +46,13 @@ public async Task RetrieveImagesForManifest_ReturnsManifest() { using var stub = new ApiStub(); const int customerId = 5; - stub.Get($"/iiif-resource/{customerId}/batch-query/1", + stub.Get($"/iiif-resource/v3/{customerId}/batch-query/1", (_, _) => "{\"id\":\"some/id\", \"type\": \"Manifest\" }") .StatusCode(200); var sut = GetClient(stub); var expected = new Manifest() { Id = "some/id" }; - var retrievedImages = await sut.RetrieveImagesForManifest(customerId, [1], CancellationToken.None); + var retrievedImages = await sut.RetrieveAssetsForManifest(customerId, [1], CancellationToken.None); retrievedImages.Should().BeEquivalentTo(expected); } diff --git a/src/IIIFPresentation/DLCS/API/DlcsApiClient.cs b/src/IIIFPresentation/DLCS/API/DlcsApiClient.cs index f175dfdd..c426c352 100644 --- a/src/IIIFPresentation/DLCS/API/DlcsApiClient.cs +++ b/src/IIIFPresentation/DLCS/API/DlcsApiClient.cs @@ -1,14 +1,10 @@ using System.Collections.Concurrent; using System.Net; -using System.Security.Claims; using DLCS.Exceptions; using DLCS.Handlers; using DLCS.Models; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.IdentityModel.JsonWebTokens; -using Microsoft.IdentityModel.Tokens; -using JwtRegisteredClaimNames = System.IdentityModel.Tokens.Jwt.JwtRegisteredClaimNames; namespace DLCS.API; diff --git a/src/IIIFPresentation/DLCS/API/DlcsHttpContent.cs b/src/IIIFPresentation/DLCS/API/DlcsHttpContent.cs index dcbb1807..987055c0 100644 --- a/src/IIIFPresentation/DLCS/API/DlcsHttpContent.cs +++ b/src/IIIFPresentation/DLCS/API/DlcsHttpContent.cs @@ -40,21 +40,7 @@ public static StringContent GenerateJsonContent(T body) return await response.ReadDlcsModel(true, cancellationToken); } - try - { - var error = await response.Content.ReadFromJsonAsync(JsonSerializerOptions, cancellationToken); - - if (error != null) - { - throw new DlcsException(error.Description); - } - - throw new DlcsException("Unable to process error condition"); - } - catch (Exception ex) when (ex is not DlcsException) - { - throw new DlcsException("Could not find a DlcsError in response", ex); - } + throw await CheckAndThrowResponseError(response, cancellationToken); } public static async Task ReadAsIIIFResponse(this HttpResponseMessage response, @@ -65,20 +51,25 @@ public static StringContent GenerateJsonContent(T body) return (await response.Content.ReadAsStreamAsync(cancellationToken)).FromJsonStream(); } + throw await CheckAndThrowResponseError(response, cancellationToken); + } + + private static async Task CheckAndThrowResponseError(HttpResponseMessage response, CancellationToken cancellationToken) + { try { var error = await response.Content.ReadFromJsonAsync(JsonSerializerOptions, cancellationToken); if (error != null) { - throw new DlcsException(error.Description); + return new DlcsException(error.Description); } throw new DlcsException("Unable to process error condition"); } catch (Exception ex) when (ex is not DlcsException) { - throw new DlcsException("Could not find a DlcsError in response", ex); + return new DlcsException("Could not find a DlcsError in response", ex); } } diff --git a/src/IIIFPresentation/DLCS/API/DlcsOrchestratorClient.cs b/src/IIIFPresentation/DLCS/API/DlcsOrchestratorClient.cs index 72341aa0..63467a22 100644 --- a/src/IIIFPresentation/DLCS/API/DlcsOrchestratorClient.cs +++ b/src/IIIFPresentation/DLCS/API/DlcsOrchestratorClient.cs @@ -9,7 +9,7 @@ public interface IDlcsOrchestratorClient /// /// Retrieves a DLCS generated manifest of images for a given presentation manifest id /// - public Task RetrieveImagesForManifest(int customerId, List batches, + public Task RetrieveAssetsForManifest(int customerId, List batches, CancellationToken cancellationToken = default); } @@ -20,19 +20,15 @@ public class DlcsOrchestratorClient( { private readonly DlcsSettings settings = dlcsOptions.Value; - public async Task RetrieveImagesForManifest(int customerId, List batches, + public async Task RetrieveAssetsForManifest(int customerId, List batches, CancellationToken cancellationToken = default) { var batchString = string.Join(',', batches); - logger.LogTrace( - "performing a call to retrieve images for customer {CustomerId} for the batches '{Batches}'", - customerId, batchString); - var response = - await httpClient.GetAsync($"/iiif-resource/{customerId}/{settings.ManifestNamedQueryName}/{batchString}", + await httpClient.GetAsync($"/iiif-resource/v3/{customerId}/{settings.ManifestNamedQueryName}/{batchString}", cancellationToken); - return await response.ReadAsIIIFResponse(cancellationToken); + return await response.ReadAsIIIFResponse(cancellationToken); } } diff --git a/src/IIIFPresentation/DLCS/DLCS.csproj b/src/IIIFPresentation/DLCS/DLCS.csproj index b4c9fd65..60dde0e0 100644 --- a/src/IIIFPresentation/DLCS/DLCS.csproj +++ b/src/IIIFPresentation/DLCS/DLCS.csproj @@ -12,7 +12,6 @@ - diff --git a/src/IIIFPresentation/DLCS/DlcsSettings.cs b/src/IIIFPresentation/DLCS/DlcsSettings.cs index d9fceba6..c1614a62 100644 --- a/src/IIIFPresentation/DLCS/DlcsSettings.cs +++ b/src/IIIFPresentation/DLCS/DlcsSettings.cs @@ -10,14 +10,19 @@ public class DlcsSettings public required Uri ApiUri { get; set; } /// - /// URL root of DLCS API + /// URL root of DLCS Orchestrator /// public Uri? OrchestratorUri { get; set; } /// - /// Default timeout (in ms) use for HttpClient.Timeout. + /// Default timeout (in ms) use for HttpClient.Timeout in the API. + /// + public int ApiDefaultTimeoutMs { get; set; } = 30000; + + /// + /// Default timeout (in ms) use for HttpClient.Timeout in orchestrator. /// - public int DefaultTimeoutMs { get; set; } = 30000; + public int OrchestratorDefaultTimeoutMs { get; set; } = 30000; /// /// The maximum size of an individual batch request diff --git a/src/IIIFPresentation/DLCS/Models/AllImages.cs b/src/IIIFPresentation/DLCS/Models/AllImages.cs deleted file mode 100644 index c9da21fd..00000000 --- a/src/IIIFPresentation/DLCS/Models/AllImages.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Text.Json.Serialization; - -namespace DLCS.Models; - -public class AllImages : JsonLdBase -{ - public AllImages(List members) - { - Members = members.Select(m => new AllImagesMember(m)).ToList(); - Type = "Collection" ; - } - - [JsonPropertyOrder(3)] - [JsonPropertyName("member")] - public List Members { get; set; } -} - -public class AllImagesMember(string id) -{ - public string Id { get; set; } = id; -} - - diff --git a/src/IIIFPresentation/DLCS/Models/AssetId.cs b/src/IIIFPresentation/DLCS/Models/AssetId.cs deleted file mode 100644 index 5b150e2e..00000000 --- a/src/IIIFPresentation/DLCS/Models/AssetId.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace DLCS.Models; - - -/// -/// A record that represents an identifier for a DLCS Asset. -/// -public class AssetId -{ - /// Id of customer - public int Customer { get; } - - /// Id of space - public int Space { get; } - - /// Id of asset - public string Asset { get; } - - /// - /// A record that represents an identifier for a DLCS Asset. - /// - public AssetId(int customer, int space, string asset) - { - Customer = customer; - Space = space; - Asset = asset; - } - - public override string ToString() => $"{Customer}/{Space}/{Asset}"; - - /// - /// Create a new AssetId from string in format customer/space/image - /// - public static AssetId FromString(string assetImageId) - { - var parts = assetImageId.Split("/", StringSplitOptions.RemoveEmptyEntries); - if (parts.Length != 3) - { - throw new ArgumentException($"AssetId '{assetImageId}' is invalid. Must be in format customer/space/asset"); - } - - try - { - return new AssetId(int.Parse(parts[0]), int.Parse(parts[1]), parts[2]); - } - catch (FormatException fmEx) - { - throw new ArgumentException($"AssetId '{assetImageId}' is invalid. Must be in format customer/space/asset", - fmEx); - } - } -} diff --git a/src/IIIFPresentation/DLCS/ServiceCollectionX.cs b/src/IIIFPresentation/DLCS/ServiceCollectionX.cs index c6e102f1..8b98faa6 100644 --- a/src/IIIFPresentation/DLCS/ServiceCollectionX.cs +++ b/src/IIIFPresentation/DLCS/ServiceCollectionX.cs @@ -21,7 +21,7 @@ public static IServiceCollection AddDlcsApiClient(this IServiceCollection servic .AddHttpClient(client => { client.BaseAddress = dlcsSettings.ApiUri; - client.Timeout = TimeSpan.FromMilliseconds(dlcsSettings.DefaultTimeoutMs); + client.Timeout = TimeSpan.FromMilliseconds(dlcsSettings.ApiDefaultTimeoutMs); }).AddHttpMessageHandler() .AddHttpMessageHandler(); @@ -39,7 +39,7 @@ public static IServiceCollection AddDlcsOrchestratorClient(this IServiceCollecti .AddHttpClient(client => { client.BaseAddress = dlcsSettings.OrchestratorUri; - client.Timeout = TimeSpan.FromMilliseconds(dlcsSettings.DefaultTimeoutMs); + client.Timeout = TimeSpan.FromMilliseconds(dlcsSettings.OrchestratorDefaultTimeoutMs); }) .AddHttpMessageHandler(); diff --git a/src/IIIFPresentation/Models/DLCS/AssetId.cs b/src/IIIFPresentation/Models/DLCS/AssetId.cs new file mode 100644 index 00000000..5e31176f --- /dev/null +++ b/src/IIIFPresentation/Models/DLCS/AssetId.cs @@ -0,0 +1,82 @@ +namespace Models.DLCS; + + +/// +/// A record that represents an identifier for a DLCS Asset. +/// +public class AssetId +{ + /// Id of customer + public int Customer { get; } + + /// Id of space + public int Space { get; } + + /// Id of asset + public string Asset { get; } + + /// + /// A record that represents an identifier for a DLCS Asset. + /// + public AssetId(int customer, int space, string asset) + { + Customer = customer; + Space = space; + Asset = asset; + } + + public override string ToString() => $"{Customer}/{Space}/{Asset}"; + + /// + /// Create a new AssetId from string in format customer/space/image + /// + public static AssetId? FromString(string assetImageId) + { + var parts = assetImageId.Split("/", StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 3) + { + throw new ArgumentException( + $"AssetId '{assetImageId}' is invalid. Must be in format customer/space/asset"); + } + + try + { + return new AssetId(int.Parse(parts[0]), int.Parse(parts[1]), parts[2]); + } + catch (FormatException fmEx) + { + throw new ArgumentException( + $"AssetId '{assetImageId}' is invalid. Must be in format customer/space/asset", + fmEx); + } + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + var asset = (AssetId)obj; + return asset.ToString() == this.ToString(); + } + + public static bool operator ==(AssetId? assetId1, AssetId? assetId2) + { + if (assetId1 is null) + { + return assetId2 is null; + } + + if (assetId2 is null) + { + return false; + } + + return assetId1.Equals(assetId2); + } + + public static bool operator !=(AssetId? assetId1, AssetId? assetId2) + => !(assetId1 == assetId2); + + public override int GetHashCode() => HashCode.Combine(Customer, Space, Asset); +} diff --git a/src/IIIFPresentation/Models/Database/CanvasPainting.cs b/src/IIIFPresentation/Models/Database/CanvasPainting.cs index e0e2955f..756603d4 100644 --- a/src/IIIFPresentation/Models/Database/CanvasPainting.cs +++ b/src/IIIFPresentation/Models/Database/CanvasPainting.cs @@ -1,5 +1,6 @@ using IIIF.Presentation.V3.Strings; using Models.Database.Collections; +using Models.DLCS; namespace Models.Database; @@ -118,7 +119,7 @@ public int? ChoiceOrder /// /// An asset id showing this asset is an internal item /// - public string? AssetId { get; set; } + public AssetId? AssetId { get; set; } /// /// Whether the asset is currently being ingested into the DLCS diff --git a/src/IIIFPresentation/Models/Database/General/Batch.cs b/src/IIIFPresentation/Models/Database/General/Batch.cs index c6481e78..47d3ae47 100644 --- a/src/IIIFPresentation/Models/Database/General/Batch.cs +++ b/src/IIIFPresentation/Models/Database/General/Batch.cs @@ -25,14 +25,19 @@ public class Batch public DateTime Submitted { get; set; } /// - /// When the batch was added to the DLCS + /// When the batch was finished by the DLCS + /// + public DateTime? Finished { get; set; } + + /// + /// When the batch was processed by presentation /// public DateTime? Processed { get; set; } /// /// Id of related manifest /// - public string? ManifestId { get; set; } + public required string ManifestId { get; set; } public Manifest? Manifest { get; set; } } diff --git a/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.Designer.cs b/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.Designer.cs new file mode 100644 index 00000000..1cd704cb --- /dev/null +++ b/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.Designer.cs @@ -0,0 +1,412 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Repository; + +#nullable disable + +namespace Repository.Migrations +{ + [DbContext(typeof(PresentationContext))] + [Migration("20250114154941_useAssetIdAndAddFinishedToBatch")] + partial class useAssetIdAndAddFinishedToBatch + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "citext"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Models.Database.CanvasPainting", b => + { + b.Property("CanvasPaintingId") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("canvas_painting_id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("CanvasPaintingId")); + + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("CanvasLabel") + .HasColumnType("text") + .HasColumnName("canvas_label"); + + b.Property("CanvasOrder") + .HasColumnType("integer") + .HasColumnName("canvas_order"); + + b.Property("CanvasOriginalId") + .HasColumnType("text") + .HasColumnName("canvas_original_id"); + + b.Property("ChoiceOrder") + .IsRequired() + .HasColumnType("integer") + .HasColumnName("choice_order"); + + b.Property("Created") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created") + .HasDefaultValueSql("now()"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Id") + .HasColumnType("text") + .HasColumnName("canvas_id"); + + b.Property("Ingesting") + .HasColumnType("boolean") + .HasColumnName("ingesting"); + + b.Property("Label") + .HasColumnType("jsonb") + .HasColumnName("label"); + + b.Property("ManifestId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Modified") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("modified") + .HasDefaultValueSql("now()"); + + b.Property("StaticHeight") + .HasColumnType("integer") + .HasColumnName("static_height"); + + b.Property("StaticWidth") + .HasColumnType("integer") + .HasColumnName("static_width"); + + b.Property("Target") + .HasColumnType("text") + .HasColumnName("target"); + + b.Property("Thumbnail") + .HasColumnType("text") + .HasColumnName("thumbnail"); + + b.HasKey("CanvasPaintingId") + .HasName("pk_canvas_paintings"); + + b.HasIndex("ManifestId", "CustomerId") + .HasDatabaseName("ix_canvas_paintings_manifest_id_customer_id"); + + b.HasIndex("Id", "CustomerId", "ManifestId", "AssetId", "CanvasOrder", "ChoiceOrder") + .IsUnique() + .HasDatabaseName("ix_canvas_paintings_canvas_id_customer_id_manifest_id_asset_id") + .HasFilter("canvas_original_id is null"); + + b.HasIndex("Id", "CustomerId", "ManifestId", "CanvasOriginalId", "CanvasOrder", "ChoiceOrder") + .IsUnique() + .HasDatabaseName("ix_canvas_paintings_canvas_id_customer_id_manifest_id_canvas_o") + .HasFilter("asset_id is null"); + + b.ToTable("canvas_paintings", (string)null); + }); + + modelBuilder.Entity("Models.Database.Collections.Collection", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Created") + .HasColumnType("timestamp with time zone") + .HasColumnName("created"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("IsPublic") + .HasColumnType("boolean") + .HasColumnName("is_public"); + + b.Property("IsStorageCollection") + .HasColumnType("boolean") + .HasColumnName("is_storage_collection"); + + b.Property("Label") + .HasColumnType("jsonb") + .HasColumnName("label"); + + b.Property("LockedBy") + .HasColumnType("text") + .HasColumnName("locked_by"); + + b.Property("Modified") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("Tags") + .HasColumnType("text") + .HasColumnName("tags"); + + b.Property("Thumbnail") + .HasColumnType("text") + .HasColumnName("thumbnail"); + + b.Property("UsePath") + .HasColumnType("boolean") + .HasColumnName("use_path"); + + b.HasKey("Id", "CustomerId") + .HasName("pk_collections"); + + b.ToTable("collections", (string)null); + }); + + modelBuilder.Entity("Models.Database.Collections.Manifest", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Created") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created") + .HasDefaultValueSql("now()"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("Label") + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("Modified") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("modified") + .HasDefaultValueSql("now()"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("SpaceId") + .HasColumnType("integer") + .HasColumnName("space_id"); + + b.HasKey("Id", "CustomerId") + .HasName("pk_manifests"); + + b.ToTable("manifests", (string)null); + }); + + modelBuilder.Entity("Models.Database.General.Batch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Finished") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished"); + + b.Property("ManifestId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Processed") + .HasColumnType("timestamp with time zone") + .HasColumnName("processed"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("Submitted") + .HasColumnType("timestamp with time zone") + .HasColumnName("submitted"); + + b.HasKey("Id") + .HasName("pk_batches"); + + b.HasIndex("ManifestId", "CustomerId") + .HasDatabaseName("ix_batches_manifest_id_customer_id"); + + b.ToTable("batches", (string)null); + }); + + modelBuilder.Entity("Models.Database.General.Hierarchy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Canonical") + .HasColumnType("boolean") + .HasColumnName("canonical"); + + b.Property("CollectionId") + .HasColumnType("text") + .HasColumnName("collection_id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("ItemsOrder") + .HasColumnType("integer") + .HasColumnName("items_order"); + + b.Property("ManifestId") + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Parent") + .HasColumnType("text") + .HasColumnName("parent"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("citext") + .HasColumnName("slug"); + + b.Property("Type") + .HasColumnType("integer") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_hierarchy"); + + b.HasIndex("Parent", "CustomerId") + .HasDatabaseName("ix_hierarchy_parent_customer_id"); + + b.HasIndex("CollectionId", "CustomerId", "Canonical") + .IsUnique() + .HasDatabaseName("ix_hierarchy_collection_id_customer_id_canonical") + .HasFilter("canonical is true"); + + b.HasIndex("CustomerId", "Slug", "Parent") + .IsUnique() + .HasDatabaseName("ix_hierarchy_customer_id_slug_parent"); + + b.HasIndex("ManifestId", "CustomerId", "Canonical") + .IsUnique() + .HasDatabaseName("ix_hierarchy_manifest_id_customer_id_canonical") + .HasFilter("canonical is true"); + + b.ToTable("hierarchy", null, t => + { + t.HasCheckConstraint("stop_collection_and_manifest_in_same_record", "num_nonnulls(manifest_id, collection_id) = 1"); + }); + }); + + modelBuilder.Entity("Models.Database.CanvasPainting", b => + { + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("CanvasPaintings") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_canvas_paintings_manifests_manifest_id_customer_id"); + + b.Navigation("Manifest"); + }); + + modelBuilder.Entity("Models.Database.General.Batch", b => + { + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("Batches") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_batches_manifests_manifest_id_customer_id"); + + b.Navigation("Manifest"); + }); + + modelBuilder.Entity("Models.Database.General.Hierarchy", b => + { + b.HasOne("Models.Database.Collections.Collection", "Collection") + .WithMany("Hierarchy") + .HasForeignKey("CollectionId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_hierarchy_collections_collection_id_customer_id"); + + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("Hierarchy") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_hierarchy_manifests_manifest_id_customer_id"); + + b.HasOne("Models.Database.Collections.Collection", "ParentCollection") + .WithMany("Children") + .HasForeignKey("Parent", "CustomerId") + .OnDelete(DeleteBehavior.NoAction) + .HasConstraintName("fk_hierarchy_collections_parent_customer_id"); + + b.Navigation("Collection"); + + b.Navigation("Manifest"); + + b.Navigation("ParentCollection"); + }); + + modelBuilder.Entity("Models.Database.Collections.Collection", b => + { + b.Navigation("Children"); + + b.Navigation("Hierarchy"); + }); + + modelBuilder.Entity("Models.Database.Collections.Manifest", b => + { + b.Navigation("Batches"); + + b.Navigation("CanvasPaintings"); + + b.Navigation("Hierarchy"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.cs b/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.cs new file mode 100644 index 00000000..1b910226 --- /dev/null +++ b/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.cs @@ -0,0 +1,29 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Repository.Migrations +{ + /// + public partial class useAssetIdAndAddFinishedToBatch : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "finished", + table: "batches", + type: "timestamp with time zone", + nullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "finished", + table: "batches"); + } + } +} diff --git a/src/IIIFPresentation/Repository/Migrations/PresentationContextModelSnapshot.cs b/src/IIIFPresentation/Repository/Migrations/PresentationContextModelSnapshot.cs index 5185e9d5..cfc2b33b 100644 --- a/src/IIIFPresentation/Repository/Migrations/PresentationContextModelSnapshot.cs +++ b/src/IIIFPresentation/Repository/Migrations/PresentationContextModelSnapshot.cs @@ -238,6 +238,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("integer") .HasColumnName("customer_id"); + b.Property("Finished") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished"); + b.Property("ManifestId") .IsRequired() .HasColumnType("text") diff --git a/src/IIIFPresentation/Repository/PresentationContext.cs b/src/IIIFPresentation/Repository/PresentationContext.cs index eb72a8c6..f675245b 100644 --- a/src/IIIFPresentation/Repository/PresentationContext.cs +++ b/src/IIIFPresentation/Repository/PresentationContext.cs @@ -4,6 +4,7 @@ using Models.Database; using Models.Database.Collections; using Models.Database.General; +using Models.DLCS; using Repository.Converters; namespace Repository; @@ -120,6 +121,9 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .IsUnique() .HasFilter("canvas_original_id is null"); + entity.Property(cp => cp.AssetId) + .HasConversion(id => id.ToString(), id => AssetId.FromString(id)); + entity .HasOne(cp => cp.Manifest) .WithMany(m => m.CanvasPaintings)