diff --git a/src/IIIFPresentation/API.Tests/Converters/ManifestConverterTests.cs b/src/IIIFPresentation/API.Tests/Converters/ManifestConverterTests.cs index b88388ab..71323032 100644 --- a/src/IIIFPresentation/API.Tests/Converters/ManifestConverterTests.cs +++ b/src/IIIFPresentation/API.Tests/Converters/ManifestConverterTests.cs @@ -3,6 +3,7 @@ using API.Tests.Helpers; using Models.API.Manifest; using Models.Database.General; +using Models.DLCS; using CanvasPainting = Models.Database.CanvasPainting; using DBManifest = Models.Database.Collections.Manifest; @@ -168,7 +169,7 @@ public void SetGeneratedFields_SetsCanvasPainting_IfPresent() Id = "the-canvas", ChoiceOrder = 10, CanvasOrder = 100, - AssetId = "1/2/assetId" + AssetId = new AssetId(1, 2, "assetId") } ] }; @@ -177,12 +178,52 @@ public void SetGeneratedFields_SetsCanvasPainting_IfPresent() var result = iiifManifest.SetGeneratedFields(dbManifest, pathGenerator); // Assert - var cp = result.PaintedResources.Single().CanvasPainting; - cp.CanvasId.Should().Be("http://base/123/canvases/the-canvas"); - cp.ChoiceOrder.Should().Be(10); - cp.CanvasOrder.Should().Be(100); - cp.CanvasOriginalId.Should().Be("http://example.test/canvas1"); - cp.AssetId.Should().Be("https://dlcs.test/customers/1/spaces/2/images/assetId"); + var paintedResource = result.PaintedResources.Single(); + paintedResource.CanvasPainting.CanvasId.Should().Be("http://base/123/canvases/the-canvas"); + paintedResource.CanvasPainting.ChoiceOrder.Should().Be(10); + paintedResource.CanvasPainting.CanvasOrder.Should().Be(100); + paintedResource.CanvasPainting.CanvasOriginalId.Should().Be("http://example.test/canvas1"); + + paintedResource.Asset.GetValue("@id").ToString().Should().Be("https://dlcs.test/customers/1/spaces/2/images/assetId"); + } + + [Fact] + public void SetGeneratedFields_SetsCanvasPainting_WithoutAssetId() + { + // Arrange + var iiifManifest = new PresentationManifest(); + var dbManifest = new DBManifest + { + CustomerId = 123, + Created = DateTime.UtcNow, + Modified = DateTime.UtcNow.AddDays(1), + CreatedBy = "creator", + ModifiedBy = "modifier", + Id = "id", + Hierarchy = [new Hierarchy { Slug = "slug" }], + CanvasPaintings = + [ + new CanvasPainting + { + CanvasOriginalId = new Uri("http://example.test/canvas1"), + CustomerId = 123, + Id = "the-canvas", + ChoiceOrder = 10, + CanvasOrder = 100 + } + ] + }; + + // Act + var result = iiifManifest.SetGeneratedFields(dbManifest, pathGenerator); + + // Assert + var paintedResource = result.PaintedResources.Single(); + paintedResource.CanvasPainting.CanvasId.Should().Be("http://base/123/canvases/the-canvas"); + paintedResource.CanvasPainting.ChoiceOrder.Should().Be(10); + paintedResource.CanvasPainting.CanvasOrder.Should().Be(100); + paintedResource.CanvasPainting.CanvasOriginalId.Should().Be("http://example.test/canvas1"); + paintedResource.Asset.Should().BeNull(); } [Fact] diff --git a/src/IIIFPresentation/API.Tests/Helpers/PathGeneratorTests.cs b/src/IIIFPresentation/API.Tests/Helpers/PathGeneratorTests.cs index 6661c918..405b5cdb 100644 --- a/src/IIIFPresentation/API.Tests/Helpers/PathGeneratorTests.cs +++ b/src/IIIFPresentation/API.Tests/Helpers/PathGeneratorTests.cs @@ -1,7 +1,9 @@ using API.Helpers; +using AWS.Helpers; using Models.Database; using Models.Database.Collections; using Models.Database.General; +using Models.DLCS; namespace API.Tests.Helpers; @@ -384,7 +386,7 @@ public void GenerateAssetUri_Correct_IfCanvasPaintingHasAssetWithThreeSlashes() { Id = "hello", CustomerId = 123, - AssetId = "5/4/12" + AssetId = new AssetId(5, 4, "12") }; var expected = "https://dlcs.test/customers/5/spaces/4/images/12"; @@ -392,23 +394,10 @@ public void GenerateAssetUri_Correct_IfCanvasPaintingHasAssetWithThreeSlashes() pathGenerator.GenerateAssetUri(manifest).Should().Be(expected); } - [Fact] - public void GenerateAssetUri_Null_IfCanvasPaintingHasAssetWithNotThreeSlashes() - { - var manifest = new CanvasPainting() - { - Id = "hello", - CustomerId = 123, - AssetId = "5/4" - }; - - pathGenerator.GenerateAssetUri(manifest).Should().BeNull(); - } - [Fact] public void GenerateAssetUri_Null_IfCanvasPaintingDoesNotHaveAsset() { - var manifest = new CanvasPainting() + var manifest = new CanvasPainting { Id = "hello", CustomerId = 123, diff --git a/src/IIIFPresentation/API.Tests/Helpers/TestPathGenerator.cs b/src/IIIFPresentation/API.Tests/Helpers/TestPathGenerator.cs index e7c58f4f..077f00b3 100644 --- a/src/IIIFPresentation/API.Tests/Helpers/TestPathGenerator.cs +++ b/src/IIIFPresentation/API.Tests/Helpers/TestPathGenerator.cs @@ -7,16 +7,18 @@ namespace API.Tests.Helpers; public static class TestPathGenerator { - public static PathGenerator CreatePathGenerator(string baseUrl, string scheme) + public static PathGenerator CreatePathGenerator(string baseUrl, string scheme) => new(new HttpContextAccessor - { - HttpContext = new DefaultHttpContext { - Request = + HttpContext = new DefaultHttpContext { - Scheme = scheme, - Host = new HostString(baseUrl) + Request = + { + Scheme = scheme, + Host = new HostString(baseUrl) + } } - } - }, Options.Create(new DlcsSettings { ApiUri = new Uri("https://dlcs.test") })); -} \ No newline at end of file + }, + Options.Create(new DlcsSettings + { ApiUri = new Uri("https://dlcs.test") })); +} diff --git a/src/IIIFPresentation/API.Tests/Integration/ModifyManifestAssetCreationTests.cs b/src/IIIFPresentation/API.Tests/Integration/ModifyManifestAssetCreationTests.cs index 8fe9e6d1..f53a966b 100644 --- a/src/IIIFPresentation/API.Tests/Integration/ModifyManifestAssetCreationTests.cs +++ b/src/IIIFPresentation/API.Tests/Integration/ModifyManifestAssetCreationTests.cs @@ -187,7 +187,7 @@ public async Task CreateManifest_CorrectlyCreatesAssetRequests_WithSpace() dbManifest.CanvasPaintings.Should().HaveCount(1); dbManifest.CanvasPaintings!.First().Label!.First().Value[0].Should().Be("canvas testing"); // space comes from the asset - dbManifest.CanvasPaintings!.First().AssetId.Should().Be($"{Customer}/{space}/{assetId}"); + dbManifest.CanvasPaintings!.First().AssetId.ToString().Should().Be($"{Customer}/{space}/{assetId}"); dbManifest.Batches.Should().HaveCount(1); dbManifest.Batches!.First().Status.Should().Be(BatchStatus.Ingesting); dbManifest.Batches!.First().Id.Should().Be(batchId); @@ -288,7 +288,7 @@ public async Task CreateManifest_CorrectlyCreatesAssetRequests_WithoutSpace() dbManifest.CanvasPaintings.Should().HaveCount(1); dbManifest.CanvasPaintings!.First().Label!.First().Value[0].Should().Be("canvas testing"); // space added using the manifest space - dbManifest.CanvasPaintings!.First().AssetId.Should() + dbManifest.CanvasPaintings!.First().AssetId.ToString().Should() .Be($"{Customer}/{NewlyCreatedSpace}/{assetId}"); dbManifest.Batches.Should().HaveCount(1); dbManifest.Batches!.First().Status.Should().Be(BatchStatus.Ingesting); @@ -383,7 +383,7 @@ await amazonS3.GetObjectAsync(LocalStackFixture.StorageBucketName, dbManifest.CanvasPaintings.Should().HaveCount(1); dbManifest.CanvasPaintings!.First().Label?.First().Should().BeNull(); // space added using the manifest space - dbManifest.CanvasPaintings!.First().AssetId.Should() + dbManifest.CanvasPaintings!.First().AssetId.ToString().Should() .Be($"{Customer}/{NewlyCreatedSpace}/{assetId}"); dbManifest.Batches.Should().HaveCount(1); dbManifest.Batches!.First().Status.Should().Be(BatchStatus.Ingesting); @@ -595,7 +595,7 @@ public async Task CreateManifest_CorrectlyCreatesAssetRequests_WhenMultipleAsset foreach (var canvasPainting in dbManifest.CanvasPaintings!) { canvasPainting.CanvasOrder.Should().Be(currentCanvasOrder); - canvasPainting.AssetId.Should() + canvasPainting.AssetId.ToString().Should() .Be($"{Customer}/{NewlyCreatedSpace}/testAssetByPresentation-multipleAssets-{currentCanvasOrder}"); currentCanvasOrder++; } diff --git a/src/IIIFPresentation/API/Converters/ManifestConverter.cs b/src/IIIFPresentation/API/Converters/ManifestConverter.cs index afe49dca..4389a1d2 100644 --- a/src/IIIFPresentation/API/Converters/ManifestConverter.cs +++ b/src/IIIFPresentation/API/Converters/ManifestConverter.cs @@ -1,11 +1,13 @@ using API.Helpers; using Core.Helpers; using Core.IIIF; +using DLCS.Models; using IIIF; using IIIF.Presentation; using Models.API.Manifest; using Models.Database.Collections; using Models.Database.General; +using Newtonsoft.Json.Linq; namespace API.Converters; @@ -62,7 +64,10 @@ public static PresentationManifest SetGeneratedFields(this PresentationManifest Label = cp.Label, CanvasOriginalId = cp.CanvasOriginalId?.ToString(), CanvasLabel = cp.CanvasLabel, - AssetId = pathGenerator.GenerateAssetUri(cp)?.ToString() + }, + Asset = cp.AssetId == null ? null : new JObject + { + ["@id"] = pathGenerator.GenerateAssetUri(cp)?.ToString() } }).ToList(); } diff --git a/src/IIIFPresentation/API/Features/Manifest/CanvasPaintingResolver.cs b/src/IIIFPresentation/API/Features/Manifest/CanvasPaintingResolver.cs index e490201d..2e92971d 100644 --- a/src/IIIFPresentation/API/Features/Manifest/CanvasPaintingResolver.cs +++ b/src/IIIFPresentation/API/Features/Manifest/CanvasPaintingResolver.cs @@ -10,6 +10,7 @@ using Models.API.General; using Models.API.Manifest; using Models.Database; +using Models.DLCS; using Newtonsoft.Json.Linq; using Repository.Manifests; using CanvasPainting = Models.Database.CanvasPainting; @@ -217,8 +218,9 @@ public class CanvasPaintingResolver( Created = DateTime.UtcNow, CustomerId = customerId, CanvasOrder = count, - AssetId = $"{customerId}/{space}/{id}", - ChoiceOrder = -1 + AssetId = AssetId.FromString($"{customerId}/{space}/{id}"), + ChoiceOrder = -1, + Ingesting = true }; count++; diff --git a/src/IIIFPresentation/API/Features/Manifest/ManifestService.cs b/src/IIIFPresentation/API/Features/Manifest/ManifestService.cs index 8b3d9d78..60b27964 100644 --- a/src/IIIFPresentation/API/Features/Manifest/ManifestService.cs +++ b/src/IIIFPresentation/API/Features/Manifest/ManifestService.cs @@ -3,11 +3,11 @@ using API.Features.Manifest.Helpers; using API.Features.Storage.Helpers; using API.Helpers; -using API.Infrastructure.AWS; using API.Infrastructure.Helpers; using API.Infrastructure.IdGenerator; using API.Infrastructure.Validation; using API.Settings; +using AWS.Helpers; using Core; using Core.Auth; using Core.Helpers; @@ -19,6 +19,7 @@ using Models.API.Manifest; using Models.Database.Collections; using Models.Database.General; +using Newtonsoft.Json.Linq; using Repository; using Repository.Helpers; using Collection = Models.Database.Collections.Collection; @@ -254,7 +255,7 @@ await canvasPaintingResolver.GenerateCanvasPaintings(request.CustomerId, request var saveErrors = await SaveAndPopulateEntity(request, dbManifest, cancellationToken); return (saveErrors, dbManifest); } - + private async Task CreateSpaceIfRequired(int customerId, string manifestId, bool createSpace, CancellationToken cancellationToken) { diff --git a/src/IIIFPresentation/API/Features/Manifest/Requests/DeleteManifest.cs b/src/IIIFPresentation/API/Features/Manifest/Requests/DeleteManifest.cs index b93267da..a905e4b1 100644 --- a/src/IIIFPresentation/API/Features/Manifest/Requests/DeleteManifest.cs +++ b/src/IIIFPresentation/API/Features/Manifest/Requests/DeleteManifest.cs @@ -1,5 +1,5 @@ using API.Features.Storage.Helpers; -using API.Infrastructure.AWS; +using AWS.Helpers; using Core; using MediatR; using Microsoft.EntityFrameworkCore; diff --git a/src/IIIFPresentation/API/Features/Manifest/Requests/GetManifest.cs b/src/IIIFPresentation/API/Features/Manifest/Requests/GetManifest.cs index 9a751732..1e9746e9 100644 --- a/src/IIIFPresentation/API/Features/Manifest/Requests/GetManifest.cs +++ b/src/IIIFPresentation/API/Features/Manifest/Requests/GetManifest.cs @@ -1,8 +1,8 @@ using API.Converters; using API.Features.Storage.Helpers; using API.Helpers; -using API.Infrastructure.AWS; using API.Infrastructure.Requests; +using AWS.Helpers; using MediatR; using Models.API.Manifest; using Repository; diff --git a/src/IIIFPresentation/API/Features/Manifest/Requests/GetManifestHierarchical.cs b/src/IIIFPresentation/API/Features/Manifest/Requests/GetManifestHierarchical.cs index 00bba856..746e2886 100644 --- a/src/IIIFPresentation/API/Features/Manifest/Requests/GetManifestHierarchical.cs +++ b/src/IIIFPresentation/API/Features/Manifest/Requests/GetManifestHierarchical.cs @@ -1,6 +1,6 @@ -using API.Converters; -using API.Converters.Streaming; +using API.Converters.Streaming; using API.Helpers; +using AWS.Helpers; using AWS.S3; using AWS.Settings; using Core.Streams; @@ -56,4 +56,4 @@ public class GetManifestHierarchicalHandler( memoryStream.Seek(0, SeekOrigin.Begin); return await reader.ReadToEndAsync(cancellationToken); } -} \ No newline at end of file +} diff --git a/src/IIIFPresentation/API/Features/Storage/Requests/CreateCollection.cs b/src/IIIFPresentation/API/Features/Storage/Requests/CreateCollection.cs index 5f10d54c..b69a62dd 100644 --- a/src/IIIFPresentation/API/Features/Storage/Requests/CreateCollection.cs +++ b/src/IIIFPresentation/API/Features/Storage/Requests/CreateCollection.cs @@ -2,11 +2,11 @@ using API.Features.Storage.Helpers; using API.Features.Storage.Models; using API.Helpers; -using API.Infrastructure.AWS; using API.Infrastructure.IdGenerator; using API.Infrastructure.Requests; using API.Infrastructure.Validation; using API.Settings; +using AWS.Helpers; using Core; using Core.Auth; using Core.Helpers; diff --git a/src/IIIFPresentation/API/Features/Storage/Requests/DeleteCollection.cs b/src/IIIFPresentation/API/Features/Storage/Requests/DeleteCollection.cs index 12784fce..78341610 100644 --- a/src/IIIFPresentation/API/Features/Storage/Requests/DeleteCollection.cs +++ b/src/IIIFPresentation/API/Features/Storage/Requests/DeleteCollection.cs @@ -1,5 +1,5 @@ using API.Features.Storage.Helpers; -using API.Infrastructure.AWS; +using AWS.Helpers; using Core; using MediatR; using Microsoft.EntityFrameworkCore; diff --git a/src/IIIFPresentation/API/Features/Storage/Requests/GetHierarchicalCollection.cs b/src/IIIFPresentation/API/Features/Storage/Requests/GetHierarchicalCollection.cs index ce750be4..0d5f4169 100644 --- a/src/IIIFPresentation/API/Features/Storage/Requests/GetHierarchicalCollection.cs +++ b/src/IIIFPresentation/API/Features/Storage/Requests/GetHierarchicalCollection.cs @@ -2,6 +2,7 @@ using API.Features.Storage.Helpers; using API.Features.Storage.Models; using API.Helpers; +using AWS.Helpers; using AWS.S3; using AWS.S3.Models; using AWS.Settings; diff --git a/src/IIIFPresentation/API/Features/Storage/Requests/PostHierarchicalCollection.cs b/src/IIIFPresentation/API/Features/Storage/Requests/PostHierarchicalCollection.cs index b7a778b8..a516567b 100644 --- a/src/IIIFPresentation/API/Features/Storage/Requests/PostHierarchicalCollection.cs +++ b/src/IIIFPresentation/API/Features/Storage/Requests/PostHierarchicalCollection.cs @@ -1,9 +1,9 @@ using System.Data; using API.Features.Storage.Helpers; using API.Helpers; -using API.Infrastructure.AWS; using API.Infrastructure.IdGenerator; using API.Infrastructure.Requests; +using AWS.Helpers; using Core; using Core.Auth; using Core.IIIF; diff --git a/src/IIIFPresentation/API/Features/Storage/Requests/UpsertCollection.cs b/src/IIIFPresentation/API/Features/Storage/Requests/UpsertCollection.cs index 5b9cacee..6c59b0b8 100644 --- a/src/IIIFPresentation/API/Features/Storage/Requests/UpsertCollection.cs +++ b/src/IIIFPresentation/API/Features/Storage/Requests/UpsertCollection.cs @@ -1,11 +1,11 @@ using API.Features.Storage.Helpers; using API.Features.Storage.Models; using API.Helpers; -using API.Infrastructure.AWS; using API.Infrastructure.Helpers; using API.Infrastructure.Requests; using API.Infrastructure.Validation; using API.Settings; +using AWS.Helpers; using Core; using Core.Auth; using Core.Exceptions; diff --git a/src/IIIFPresentation/API/Helpers/PathGenerator.cs b/src/IIIFPresentation/API/Helpers/PathGenerator.cs index 530cc571..cfa75c44 100644 --- a/src/IIIFPresentation/API/Helpers/PathGenerator.cs +++ b/src/IIIFPresentation/API/Helpers/PathGenerator.cs @@ -1,8 +1,10 @@ using API.Infrastructure.Requests; using DLCS; +using DLCS.Models; using Microsoft.Extensions.Options; using Models.Database.Collections; using Models.Database.General; +using Models.DLCS; using CanvasPainting = Models.Database.CanvasPainting; namespace API.Helpers; @@ -94,13 +96,11 @@ public string GenerateCanvasId(CanvasPainting canvasPainting) public Uri? GenerateAssetUri(CanvasPainting canvasPainting) { - if (string.IsNullOrEmpty(canvasPainting.AssetId)) return null; - var assetId = canvasPainting.AssetId.Split('/'); - if (assetId.Length != 3) return null; - + if (canvasPainting.AssetId == null) return null; + var uriBuilder = new UriBuilder(dlcsSettings.ApiUri) { - Path = $"/customers/{assetId[0]}/spaces/{assetId[1]}/images/{assetId[2]}", + Path = $"/customers/{canvasPainting.AssetId.Customer}/spaces/{canvasPainting.AssetId.Space}/images/{canvasPainting.AssetId.Asset}", }; return uriBuilder.Uri; } diff --git a/src/IIIFPresentation/API/Infrastructure/PresentationController.cs b/src/IIIFPresentation/API/Infrastructure/PresentationController.cs index b2d3a62a..ab49ef0e 100644 --- a/src/IIIFPresentation/API/Infrastructure/PresentationController.cs +++ b/src/IIIFPresentation/API/Infrastructure/PresentationController.cs @@ -1,5 +1,4 @@ using System.Net; -using API.Converters; using API.Exceptions; using API.Infrastructure.Requests; using API.Settings; @@ -180,4 +179,4 @@ protected async Task HandleRequest(Func> hand return Problem(ex.Message, null, 500, errorTitle); } } -} \ No newline at end of file +} diff --git a/src/IIIFPresentation/API/Infrastructure/ServiceCollectionX.cs b/src/IIIFPresentation/API/Infrastructure/ServiceCollectionX.cs index 4be781db..c99ce1a8 100644 --- a/src/IIIFPresentation/API/Infrastructure/ServiceCollectionX.cs +++ b/src/IIIFPresentation/API/Infrastructure/ServiceCollectionX.cs @@ -1,13 +1,11 @@ using System.Reflection; -using API.Infrastructure.AWS; using API.Infrastructure.IdGenerator; using API.Infrastructure.Mediatr.Behaviours; using API.Infrastructure.Requests.Pipelines; using API.Settings; using AWS.Configuration; +using AWS.Helpers; using AWS.S3; -using AWS.Settings; -using AWS.SQS; using MediatR; using Repository; using Sqids; @@ -92,4 +90,4 @@ public static IServiceCollection ConfigureDefaultCors(this IServiceCollection se .AllowAnyMethod() .AllowAnyHeader()); }); -} \ No newline at end of file +} diff --git a/src/IIIFPresentation/API/Program.cs b/src/IIIFPresentation/API/Program.cs index b507c09f..453e7246 100644 --- a/src/IIIFPresentation/API/Program.cs +++ b/src/IIIFPresentation/API/Program.cs @@ -52,7 +52,7 @@ var dlcs = dlcsSettings.Get()!; builder.Services - .AddDlcsClient(dlcs) + .AddDlcsApiClient(dlcs) .AddDelegatedAuthHandler(opts => { opts.Realm = "DLCS-API"; }); builder.Services.ConfigureDefaultCors(corsPolicyName); builder.Services.AddDataAccess(builder.Configuration); diff --git a/src/IIIFPresentation/AWS/AWS.csproj b/src/IIIFPresentation/AWS/AWS.csproj index a7bb8a41..a6ced17c 100644 --- a/src/IIIFPresentation/AWS/AWS.csproj +++ b/src/IIIFPresentation/AWS/AWS.csproj @@ -8,6 +8,7 @@ + diff --git a/src/IIIFPresentation/API/Helpers/BucketHelperX.cs b/src/IIIFPresentation/AWS/Helpers/BucketHelperX.cs similarity index 97% rename from src/IIIFPresentation/API/Helpers/BucketHelperX.cs rename to src/IIIFPresentation/AWS/Helpers/BucketHelperX.cs index a53dc333..82710f55 100644 --- a/src/IIIFPresentation/API/Helpers/BucketHelperX.cs +++ b/src/IIIFPresentation/AWS/Helpers/BucketHelperX.cs @@ -1,6 +1,6 @@ using Models.Database.Collections; -namespace API.Helpers; +namespace AWS.Helpers; public static class BucketHelperX { @@ -25,4 +25,4 @@ public static string GetManifestBucketKey(int customerId, string flatId) private static string GetResourceBucketKey(int customerId, string slug, string flatId) => $"{customerId}/{slug}/{flatId}"; -} \ No newline at end of file +} diff --git a/src/IIIFPresentation/API/Infrastructure/AWS/IIIFS3Service.cs b/src/IIIFPresentation/AWS/Helpers/IIIFS3Service.cs similarity index 73% rename from src/IIIFPresentation/API/Infrastructure/AWS/IIIFS3Service.cs rename to src/IIIFPresentation/AWS/Helpers/IIIFS3Service.cs index 9ae6852a..af620d5c 100644 --- a/src/IIIFPresentation/API/Infrastructure/AWS/IIIFS3Service.cs +++ b/src/IIIFPresentation/AWS/Helpers/IIIFS3Service.cs @@ -1,18 +1,32 @@ -using API.Helpers; -using API.Settings; -using AWS.S3; +using AWS.S3; using AWS.S3.Models; +using AWS.Settings; using Core.Helpers; using Core.IIIF; using Core.Streams; using IIIF.Presentation; using IIIF.Presentation.V3; using IIIF.Serialisation; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Models.Database.Collections; using Models.Infrastucture; -namespace API.Infrastructure.AWS; +namespace AWS.Helpers; + +public interface IIIIFS3Service +{ + public Task ReadIIIFFromS3(IHierarchyResource dbResource, + CancellationToken cancellationToken) where T : ResourceBase, new(); + + public Task ReadIIIFFromS3(string bucketKey, + CancellationToken cancellationToken) where T : ResourceBase, new(); + + public Task SaveIIIFToS3(ResourceBase iiifResource, IHierarchyResource dbResource, string flatId, + CancellationToken cancellationToken); + + public Task DeleteIIIFFromS3(IHierarchyResource dbResource); +} /// /// Class containing higher-level functions to aid interacting with S3 @@ -21,7 +35,7 @@ public class IIIFS3Service( IBucketWriter bucketWriter, IBucketReader bucketReader, ILogger logger, - IOptionsMonitor options) + IOptionsMonitor options) : IIIIFS3Service { public Task ReadIIIFFromS3(IHierarchyResource dbResource, CancellationToken cancellationToken) where T : ResourceBase, new() => @@ -31,7 +45,7 @@ public class IIIFS3Service( CancellationToken cancellationToken) where T : ResourceBase, new() { var objectFromBucket = await bucketReader.GetObjectFromBucket( - new(options.CurrentValue.AWS.S3.StorageBucket, bucketKey), cancellationToken); + new(options.CurrentValue.S3.StorageBucket, bucketKey), cancellationToken); if (objectFromBucket.Stream.IsNull()) return null; @@ -48,7 +62,7 @@ public async Task SaveIIIFToS3(ResourceBase iiifResource, IHierarchyResource dbR logger.LogDebug("Uploading resource {Customer}:{ResourceId} file to S3", dbResource.CustomerId, dbResource.Id); EnsureIIIFValid(iiifResource, flatId); var iiifJson = iiifResource.AsJson(); - var item = new ObjectInBucket(options.CurrentValue.AWS.S3.StorageBucket, dbResource.GetResourceBucketKey()); + var item = new ObjectInBucket(options.CurrentValue.S3.StorageBucket, dbResource.GetResourceBucketKey()); await bucketWriter.WriteToBucket(item, iiifJson, "application/json", cancellationToken); } @@ -58,7 +72,7 @@ public async Task SaveIIIFToS3(ResourceBase iiifResource, IHierarchyResource dbR public async Task DeleteIIIFFromS3(IHierarchyResource dbResource) { logger.LogDebug("Deleting resource {Customer}:{ResourceId} file from S3", dbResource.CustomerId, dbResource.Id); - var item = new ObjectInBucket(options.CurrentValue.AWS.S3.StorageBucket, dbResource.GetResourceBucketKey()); + var item = new ObjectInBucket(options.CurrentValue.S3.StorageBucket, dbResource.GetResourceBucketKey()); await bucketWriter.DeleteFromBucket(item); } @@ -79,4 +93,4 @@ private static void RemovePresentationBehaviours(ResourceBase iiifResource) iiifResource.Behavior = iiifResource.Behavior.Where(b => !toRemove.Contains(b)).ToList(); } -} \ No newline at end of file +} diff --git a/src/IIIFPresentation/AWS/Settings/AWSSettings.cs b/src/IIIFPresentation/AWS/Settings/AWSSettings.cs index fce00432..9c4b0fa5 100644 --- a/src/IIIFPresentation/AWS/Settings/AWSSettings.cs +++ b/src/IIIFPresentation/AWS/Settings/AWSSettings.cs @@ -2,6 +2,8 @@ public class AWSSettings { + public const string SettingsName = "AWS"; + /// /// If true, service will use LocalStack and custom ServiceUrl /// @@ -16,4 +18,4 @@ public class AWSSettings /// SQS Settings /// public SQSSettings SQS { get; set; } = new(); -} \ No newline at end of file +} diff --git a/src/IIIFPresentation/AWS/Settings/SQSSettings.cs b/src/IIIFPresentation/AWS/Settings/SQSSettings.cs index b23e5d10..20b9ec98 100644 --- a/src/IIIFPresentation/AWS/Settings/SQSSettings.cs +++ b/src/IIIFPresentation/AWS/Settings/SQSSettings.cs @@ -7,6 +7,11 @@ public class SQSSettings /// public string? CustomerCreatedQueueName { get; set; } + /// + /// Name of queue that will receive notifications when a batch is completed + /// + public string? BatchCompletionQueueName { get; set; } + /// /// The duration (in seconds) for which the call waits for a message to arrive in the queue before returning /// @@ -21,4 +26,4 @@ public class SQSSettings /// Service root for SQS. Ignored if not running LocalStack /// public string ServiceUrl { get; set; } = "http://localhost:4566/"; -} \ No newline at end of file +} diff --git a/src/IIIFPresentation/BackgroundHandler.Tests/BatchCompletion/BatchCompletionMessageHandlerTests.cs b/src/IIIFPresentation/BackgroundHandler.Tests/BatchCompletion/BatchCompletionMessageHandlerTests.cs new file mode 100644 index 00000000..2c3e0719 --- /dev/null +++ b/src/IIIFPresentation/BackgroundHandler.Tests/BatchCompletion/BatchCompletionMessageHandlerTests.cs @@ -0,0 +1,262 @@ +using AWS.Helpers; +using AWS.SQS; +using BackgroundHandler.BatchCompletion; +using BackgroundHandler.Tests.infrastructure; +using DLCS; +using DLCS.API; +using FakeItEasy; +using FluentAssertions; +using IIIF; +using IIIF.ImageApi.V3; +using IIIF.Presentation.V3; +using IIIF.Presentation.V3.Annotation; +using IIIF.Presentation.V3.Content; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Models.Database.Collections; +using Models.Database.General; +using Models.DLCS; +using Repository; +using Test.Helpers.Helpers; +using Test.Helpers.Integration; +using Batch = Models.Database.General.Batch; +using CanvasPainting = Models.Database.CanvasPainting; +using Manifest = Models.Database.Collections.Manifest; + +namespace BackgroundHandler.Tests.BatchCompletion; + +[Trait("Category", "Database")] +[Collection(CollectionDefinitions.DatabaseCollection.CollectionName)] +public class BatchCompletionMessageHandlerTests +{ + private readonly PresentationContext dbContext; + private readonly BatchCompletionMessageHandler sut; + private readonly IDlcsOrchestratorClient dlcsClient; + private readonly IIIIFS3Service iiifS3; + private readonly DlcsSettings dlcsSettings; + private readonly int customerId = 1; + + public BatchCompletionMessageHandlerTests(PresentationContextFixture dbFixture) + { + dbContext = dbFixture.DbContext; + dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.TrackAll; + dlcsClient = A.Fake(); + iiifS3 = A.Fake(); + dlcsSettings = new DlcsSettings() + { + ApiUri = new Uri("https://localhost:5000"), + OrchestratorUri = new Uri("https://localhost:5000") + }; + + sut = new BatchCompletionMessageHandler(dbFixture.DbContext, dlcsClient, Options.Create(dlcsSettings), iiifS3, + new NullLogger()); + } + + [Fact] + public async Task HandleMessage_False_IfMessageInvalid() + { + // Arrange + var message = GetMessage("not-json"); + + // Act and Assert + (await sut.HandleMessage(message, CancellationToken.None)).Should().BeFalse(); + } + + [Fact] + public async Task HandleMessage_DoesNotUpdateAnything_WhenBatchNotTracked() + { + // Arrange + var message = GetMessage("{\"id\":572246,\"customerId\":58,\"total\":1,\"success\":1,\"errors\":0,\"superseded\":false,\"started\":\"2024-12-19T21:03:31.577678Z\",\"finished\":\"2024-12-19T21:03:31.902514Z\"}"); + + // Act and Assert + (await sut.HandleMessage(message, CancellationToken.None)).Should().BeTrue(); + A.CallTo(() => dlcsClient.RetrieveAssetsForManifest(A._, A>._, A._)) + .MustNotHaveHappened(); + } + + [Fact] + public async Task HandleMessage_UpdatesBatchedImages_WhenBatchTracked() + { + // Arrange + var batchId = 1; + var manifestId = nameof(HandleMessage_UpdatesBatchedImages_WhenBatchTracked); + var slug = $"slug_{nameof(HandleMessage_UpdatesBatchedImages_WhenBatchTracked)}"; + var assetId = $"asset_id_{nameof(HandleMessage_UpdatesBatchedImages_WhenBatchTracked)}"; + var space = 2; + var fullAssetId = new AssetId(customerId, space, assetId); + + A.CallTo(() => iiifS3.ReadIIIFFromS3(A._, A._)) + .ReturnsLazily(() => new IIIF.Presentation.V3.Manifest + { + Id = manifestId + }); + + var manifest = CreateManifest(manifestId, slug, assetId, space, batchId); + + await dbContext.Manifests.AddAsync(manifest); + await dbContext.SaveChangesAsync(); + + var finished = DateTime.UtcNow.AddHours(-1); + + var batchMessage = $@" +{{ + ""id"":{batchId}, + ""customerId"": {customerId}, + ""total"":1, + ""success"":1, + ""errors"":0, + ""superseded"":false, + ""started"":""2024-12-19T21:03:31.57Z"", + ""finished"":""{finished:yyyy-MM-ddTHH:mm:ssK}"" +}}"; + + var message = GetMessage(batchMessage); + + A.CallTo(() => dlcsClient.RetrieveAssetsForManifest(A._, A>._, A._)) + .Returns(new IIIF.Presentation.V3.Manifest + { + Items = new List + { + new () + { + Id = $"{dlcsSettings.OrchestratorUri}/iiif-img/{fullAssetId}/canvas/c/1", + Width = 100, + Height = 100, + Annotations = new List + { + new () + { + Items = new List() + { + new PaintingAnnotation() + { + Body = new Image() + { + Width = 100, + Height = 100, + }, + Service = new List() + { + new ImageService3() + { + Profile = "level2" + } + } + } + } + } + } + } + } + }); + + // Act + var handleMessage = await sut.HandleMessage(message, CancellationToken.None); + + // Assert + handleMessage.Should().BeTrue(); + A.CallTo(() => dlcsClient.RetrieveAssetsForManifest(A._, A>._, A._)) + .MustHaveHappened(); + var batch = dbContext.Batches.Single(b => b.Id == batchId); + batch.Status.Should().Be(BatchStatus.Completed); + batch.Finished.Should().BeCloseTo(finished, TimeSpan.FromSeconds(10)); + batch.Processed.Should().BeCloseTo(DateTime.UtcNow, TimeSpan.FromSeconds(10)); + var canvasPainting = dbContext.CanvasPaintings.Single(c => c.AssetId == fullAssetId); + canvasPainting.Ingesting.Should().BeFalse(); + canvasPainting.StaticWidth.Should().Be(100); + canvasPainting.StaticHeight.Should().Be(100); + canvasPainting.AssetId.ToString().Should() + .Be(fullAssetId.ToString()); + } + + [Fact] + public async Task HandleMessage_DoesNotUpdateBatchedImages_WhenAnotherBatchWaiting() + { + // Arrange + var batchId = 2; + var manifestId = nameof(HandleMessage_DoesNotUpdateBatchedImages_WhenAnotherBatchWaiting); + var slug = $"slug_{nameof(HandleMessage_DoesNotUpdateBatchedImages_WhenAnotherBatchWaiting)}"; + var assetId = $"asset_id_{nameof(HandleMessage_DoesNotUpdateBatchedImages_WhenAnotherBatchWaiting)}"; + var space = 2; + + var manifest = CreateManifest(manifestId, slug, assetId, space, batchId); + var additionalBatch = new Batch + { + Id = 3, + ManifestId = manifestId, + CustomerId = customerId, + Status = BatchStatus.Ingesting + }; + + await dbContext.Manifests.AddAsync(manifest); + await dbContext.Batches.AddAsync(additionalBatch); + await dbContext.SaveChangesAsync(); + + var batchMessage = $@" +{{ + ""id"":{batchId}, + ""customerId"": {customerId}, + ""total"": 1, + ""success"": 1, + ""errors"": 0, + ""superseded"": false, + ""started"": ""2024-12-19T21:03:31.57Z"", + ""finished"": ""2024-12-19T21:03:31.57Z"" +}}"; + + var message = GetMessage(batchMessage); + + // Act and Assert + (await sut.HandleMessage(message, CancellationToken.None)).Should().BeTrue(); + A.CallTo(() => dlcsClient.RetrieveAssetsForManifest(A._, A>._, A._)) + .MustNotHaveHappened(); + var batch = await dbContext.Batches.SingleOrDefaultAsync(b => b.Id == batchId); + batch.Status.Should().Be(BatchStatus.Completed); + } + + private Manifest CreateManifest(string manifestId, string slug, string assetId, int space, int batchId) + { + var manifest = new Manifest + { + Id = manifestId, + Created = DateTime.UtcNow, + Modified = DateTime.UtcNow, + CreatedBy = "admin", + CustomerId = customerId, + Hierarchy = + [ + new Hierarchy + { + Slug = slug, + Parent = RootCollection.Id, + Type = ResourceType.StorageCollection, + Canonical = true + } + ], + CanvasPaintings = [ + new CanvasPainting + { + AssetId = new AssetId(customerId, space, assetId), + CanvasOrder = 0, + ChoiceOrder = 1, + CustomerId = customerId, + Ingesting = true + } + ], + Batches = [ + new Batch + { + Id = batchId, + ManifestId = manifestId, + CustomerId = customerId, + Submitted = DateTime.UtcNow, + Status = BatchStatus.Ingesting + } + ] + }; + return manifest; + } + + private static QueueMessage GetMessage(string body) => new(body, new Dictionary(), "foo"); +} diff --git a/src/IIIFPresentation/BackgroundHandler.Tests/Helpers/ManifestMergerTests.cs b/src/IIIFPresentation/BackgroundHandler.Tests/Helpers/ManifestMergerTests.cs new file mode 100644 index 00000000..d4bac824 --- /dev/null +++ b/src/IIIFPresentation/BackgroundHandler.Tests/Helpers/ManifestMergerTests.cs @@ -0,0 +1,201 @@ +using BackgroundHandler.Helpers; +using FluentAssertions; +using IIIF; +using IIIF.ImageApi.V3; +using IIIF.Presentation.V3.Annotation; +using IIIF.Presentation.V3.Content; +using IIIF.Presentation.V3.Strings; +using Models.Database; +using Models.DLCS; +using Canvas = IIIF.Presentation.V3.Canvas; +using Manifest = IIIF.Presentation.V3.Manifest; + +namespace BackgroundHandler.Tests.Helpers; + +public class ManifestMergerTests +{ + [Fact] + public void Merge_MergesBlankManifestWithGeneratedManifest() + { + // Arrange + var blankManifest = new Manifest(); + var assetId = $"1/2/{nameof(Merge_MergesBlankManifestWithGeneratedManifest)}"; + + var generatedManifest = GeneratedManifest(assetId); + var itemDictionary = generatedManifest.Items.ToDictionary(i => AssetId.FromString(i.Id), i => i); + + // Act + var mergedManifest = ManifestMerger.Merge(blankManifest, + GenerateCanvasPaintings([assetId]), itemDictionary, + generatedManifest.Thumbnail); + + // Assert + mergedManifest.Items.Count.Should().Be(1); + mergedManifest.Items[0].Width.Should().Be(100); + mergedManifest.Items[0].Height.Should().Be(100); + mergedManifest.Thumbnail.Count.Should().Be(1); + mergedManifest.Metadata.Should().BeNull(); + mergedManifest.Label.Should().BeNull(); + mergedManifest.Items[0].Metadata.Should().BeNull(); + } + + [Fact] + public void Merge_MergesFullManifestWithGeneratedManifest() + { + // Arrange + var assetId = $"1/2/{nameof(Merge_MergesFullManifestWithGeneratedManifest)}"; + var blankManifest = GeneratedManifest(assetId); + blankManifest.Items[0].Width = 200; + blankManifest.Items[0].Height = 200; + var generatedManifest = GeneratedManifest(assetId); + var itemDictionary = generatedManifest.Items.ToDictionary(i => AssetId.FromString(i.Id), i => i); + + // Act + var mergedManifest = ManifestMerger.Merge(blankManifest, + GenerateCanvasPaintings([assetId]), itemDictionary, + generatedManifest.Thumbnail); + + // Assert + mergedManifest.Items.Count.Should().Be(1); + mergedManifest.Thumbnail.Count.Should().Be(1); + mergedManifest.Items[0].Width.Should().Be(100); + mergedManifest.Items[0].Height.Should().Be(100); + } + + [Fact] + public void Merge_ShouldNotUpdateAttachedManifestThumbnail() + { + // Arrange + var assetId = $"1/2/{nameof(Merge_ShouldNotUpdateAttachedManifestThumbnail)}"; + var blankManifest = GeneratedManifest(assetId); + blankManifest.Items[0].Width = 200; + blankManifest.Items[0].Height = 200; + blankManifest.Thumbnail.Add(GenerateImage()); + blankManifest.Thumbnail[0].Service[0].Id = "generatedId"; + var generatedManifest = GeneratedManifest(assetId); + var itemDictionary = generatedManifest.Items.ToDictionary(i => AssetId.FromString(i.Id), i => i); + + // Act + var mergedManifest = ManifestMerger.Merge(blankManifest, + GenerateCanvasPaintings([assetId]), itemDictionary, + generatedManifest.Thumbnail); + + // Assert + mergedManifest.Items.Count.Should().Be(1); + mergedManifest.Thumbnail[0].Service[0].Id.Should().Be("generatedId"); + mergedManifest.Thumbnail.Count.Should().Be(2); + mergedManifest.Items[0].Width.Should().Be(100); + mergedManifest.Items[0].Height.Should().Be(100); + } + + [Fact] + public void Merge_CorrectlyOrdersMultipleItems() + { + // Arrange + var blankManifest = new Manifest(); + var generatedManifest = GeneratedManifest($"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_1"); + generatedManifest.Items.Add(GenerateCanvas($"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_2")); + var itemDictionary = generatedManifest.Items.ToDictionary(i => AssetId.FromString(i.Id), i => i); + + var canvasPaintings = GenerateCanvasPaintings([ + $"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_1", + $"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_2" + ]); + + canvasPaintings[0].CanvasOrder = 1; + canvasPaintings[1].CanvasOrder = 0; + + // Act + var mergedManifest = + ManifestMerger.Merge(blankManifest, canvasPaintings, itemDictionary, generatedManifest.Thumbnail); + + // Assert + mergedManifest.Items.Count.Should().Be(2); + mergedManifest.Thumbnail[0].Service[0].Id.Should().Be("imageId"); + mergedManifest.Thumbnail.Count.Should().Be(1); + // order flipped due to canvas order + mergedManifest.Items[0].Id.Should().Be($"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_2"); + mergedManifest.Items[1].Id.Should().Be($"1/2/{nameof(Merge_CorrectlyOrdersMultipleItems)}_1"); + } + + private List GenerateCanvasPaintings(List idList) + { + var canvasOrder = 0; + return idList.Select(id => new CanvasPainting{Id = id, AssetId = AssetId.FromString(id), CanvasOrder = canvasOrder++}).ToList(); + } + + private Manifest GeneratedManifest(string id) + { + return new Manifest + { + Thumbnail = + [ + GenerateImage() + ], + Label = new LanguageMap("en", "someLabel"), + Items = + [ + GenerateCanvas(id) + ], + Metadata = GenerateMetadata() + }; + } + + private static Canvas GenerateCanvas(string id) + { + return new Canvas + { + Id = id, + Width = 100, + Height = 100, + Metadata = GenerateMetadata(), + Annotations = + [ + new AnnotationPage + { + Items = + [ + new PaintingAnnotation + { + Body = new Image + { + Width = 100, + Height = 100, + }, + Service = new List + { + new ImageService3 + { + Profile = "level2" + } + } + } + ] + } + ] + }; + } + + private static List GenerateMetadata() + { + return new List() + { + new(new LanguageMap("en", "label1"), new LanguageMap("en", "value1")) + }; + } + + private static Image GenerateImage() + { + return new Image + { + Service = + [ + new ImageService3 + { + Id = "imageId", + Sizes = [new(100, 100)] + } + ] + }; + } +} diff --git a/src/IIIFPresentation/BackgroundHandler/BackgroundHandler.csproj b/src/IIIFPresentation/BackgroundHandler/BackgroundHandler.csproj index a304dab5..67071d95 100644 --- a/src/IIIFPresentation/BackgroundHandler/BackgroundHandler.csproj +++ b/src/IIIFPresentation/BackgroundHandler/BackgroundHandler.csproj @@ -14,6 +14,7 @@ + diff --git a/src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessage.cs b/src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessage.cs new file mode 100644 index 00000000..f2247029 --- /dev/null +++ b/src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessage.cs @@ -0,0 +1,11 @@ +namespace BackgroundHandler.BatchCompletion; + +public record BatchCompletionMessage( + int Id, + int CustomerId, + int Total, + int Success, + int Errors, + bool Superseded, + DateTime Started, + DateTime Finished); diff --git a/src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessageHandler.cs b/src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessageHandler.cs new file mode 100644 index 00000000..a3c4e451 --- /dev/null +++ b/src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessageHandler.cs @@ -0,0 +1,164 @@ +using System.Text.Json; +using AWS.Helpers; +using AWS.SQS; +using BackgroundHandler.Helpers; +using Core.Helpers; +using Core.IIIF; +using DLCS; +using DLCS.API; +using IIIF.Presentation.V3; +using IIIF.Presentation.V3.Content; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; +using Models.Database.General; +using Models.DLCS; +using Repository; +using Batch = Models.Database.General.Batch; + +namespace BackgroundHandler.BatchCompletion; + +public class BatchCompletionMessageHandler( + PresentationContext dbContext, + IDlcsOrchestratorClient dlcsOrchestratorClient, + IOptions dlcsOptions, + IIIIFS3Service iiifS3, + ILogger logger) + : IMessageHandler +{ + private static readonly JsonSerializerOptions JsonSerializerOptions = new(JsonSerializerDefaults.Web); + private readonly DlcsSettings dlcsSettings = dlcsOptions.Value; + + public async Task HandleMessage(QueueMessage message, CancellationToken cancellationToken) + { + using (LogContextHelpers.SetServiceName(nameof(BatchCompletionMessageHandler))) + { + try + { + var batchCompletionMessage = DeserializeMessage(message); + + await UpdateAssetsIfRequired(batchCompletionMessage, cancellationToken); + return true; + } + catch (Exception ex) + { + logger.LogError(ex, "Error handling batch-completion message {MessageId}", message.MessageId); + } + } + + return false; + } + + private async Task UpdateAssetsIfRequired(BatchCompletionMessage batchCompletionMessage, CancellationToken cancellationToken) + { + var batch = await dbContext.Batches.Include(b => b.Manifest) + .ThenInclude(m => m.CanvasPaintings) + .SingleOrDefaultAsync(b => b.Id == batchCompletionMessage.Id, cancellationToken); + + // batch isn't tracked by presentation, so nothing to do + if (batch == null) return; + + // Other batches haven't completed, so no point populating items until all are complete + if (await dbContext.Batches.AnyAsync(b => b.ManifestId == batch.ManifestId && + b.Status != BatchStatus.Completed && + b.Id != batch.Id, cancellationToken)) + { + CompleteBatch(batch, batchCompletionMessage.Finished); + } + else + { + logger.LogInformation( + "Attempting to complete assets in batch {BatchId} for customer {CustomerId} with the manifest {ManifestId}", + batch.Id, batch.CustomerId, batch.ManifestId); + + var batches = dbContext.Batches.Where(b => b.ManifestId == batch.ManifestId).Select(b => b.Id).ToList(); + + var generatedManifest = + await dlcsOrchestratorClient.RetrieveAssetsForManifest(batch.CustomerId, batches, + cancellationToken); + + Dictionary itemDictionary; + + try + { + itemDictionary = generatedManifest.Items.ToDictionary(i => GetAssetIdFromCanvasId(i.Id), i => i); + } + catch (Exception e) + { + logger.LogError(e, "Error retrieving the asset id from an item in {ManifestId}", generatedManifest?.Id); + throw; + } + + try + { + UpdateCanvasPaintings(generatedManifest, batch, itemDictionary); + CompleteBatch(batch, batchCompletionMessage.Finished); + await UpdateManifestInS3(generatedManifest.Thumbnail, itemDictionary, batch, cancellationToken); + } + catch (Exception e) + { + logger.LogError(e, "Error updating completing batch {BatchId} for manifest {ManifestId}", batch.Id, + generatedManifest.Id); + throw; + } + } + + await dbContext.SaveChangesAsync(cancellationToken); + logger.LogTrace("updating batch {BatchId} has been completed", batch.Id); + } + + private static AssetId GetAssetIdFromCanvasId(string canvasId) + { + return AssetId.FromString(String.Join('/', + canvasId.Substring(0, canvasId.IndexOf("/canvas/c/")).Split("/")[^3..])); + } + + private async Task UpdateManifestInS3(List? thumbnail, Dictionary itemDictionary, Batch batch, + CancellationToken cancellationToken = default) + { + var manifest = await iiifS3.ReadIIIFFromS3(batch.Manifest!, cancellationToken); + + var mergedManifest = ManifestMerger.Merge(manifest.ThrowIfNull(nameof(manifest)), + batch.Manifest?.CanvasPaintings, itemDictionary, thumbnail); + + await iiifS3.SaveIIIFToS3(mergedManifest, batch.Manifest!, "", cancellationToken); + } + + private void CompleteBatch(Batch batch, DateTime finished) + { + batch.Processed = DateTime.UtcNow; + batch.Finished = finished; + batch.Status = BatchStatus.Completed; + } + + private void UpdateCanvasPaintings(Manifest generatedManifest, Batch batch, Dictionary itemDictionary) + { + if (batch.Manifest?.CanvasPaintings == null) + { + logger.LogWarning( + "Received a batch completion notification with no canvas paintings on the batch {BatchId}", batch.Id); + return; + } + + foreach (var canvasPainting in batch.Manifest.CanvasPaintings) + { + itemDictionary.TryGetValue(canvasPainting.AssetId!, out var item); + + if (item == null) continue; + + var thumbnailPath = item.Thumbnail?.OfType().GetThumbnailPath(); + + canvasPainting.Thumbnail = thumbnailPath != null ? new Uri(thumbnailPath) : null; + canvasPainting.Ingesting = false; + canvasPainting.Modified = DateTime.UtcNow; + canvasPainting.StaticHeight = item.Height; + canvasPainting.StaticWidth = item.Width; + } + + } + + private static BatchCompletionMessage DeserializeMessage(QueueMessage message) + { + var deserialized = JsonSerializer.Deserialize(message.Body, JsonSerializerOptions); + return deserialized.ThrowIfNull(nameof(deserialized)); + } +} diff --git a/src/IIIFPresentation/BackgroundHandler/Helpers/ManifestMerger.cs b/src/IIIFPresentation/BackgroundHandler/Helpers/ManifestMerger.cs new file mode 100644 index 00000000..ea158532 --- /dev/null +++ b/src/IIIFPresentation/BackgroundHandler/Helpers/ManifestMerger.cs @@ -0,0 +1,49 @@ +using Core.Helpers; +using IIIF.Presentation.V3; +using IIIF.Presentation.V3.Content; +using Models.DLCS; +using CanvasPainting = Models.Database.CanvasPainting; + +namespace BackgroundHandler.Helpers; + +public static class ManifestMerger +{ + /// + /// Merges a generated DLCS manifest with the current manifest in S3 + /// + public static Manifest Merge(Manifest baseManifest, List? canvasPaintings, + Dictionary itemDictionary, List? thumbnail) + { + if (baseManifest.Items == null) baseManifest.Items = []; + + var indexedBaseManifest = baseManifest.Items.Select((item, index) => (item, index)).ToList(); + var orderedCanvasPaintings = canvasPaintings?.OrderBy(cp => cp.CanvasOrder).ToList() ?? []; + + // We want to use the canvas order set when creating assets, rather than the + foreach (var canvasPainting in orderedCanvasPaintings) + { + if (!itemDictionary.TryGetValue(canvasPainting.AssetId!, out var generatedItem)) continue; + + var existingItem = indexedBaseManifest.FirstOrDefault(bm => bm.item.Id == generatedItem.Id); + + // remove canvas metadata as it's not required + generatedItem.Metadata = null; + + if (existingItem.item != null) + { + baseManifest.Items[existingItem.index] = generatedItem; + } + else + { + baseManifest.Items.Add(generatedItem); + } + } + + if (baseManifest.Thumbnail.IsNullOrEmpty()) + { + baseManifest.Thumbnail = thumbnail; + } + + return baseManifest; + } +} diff --git a/src/IIIFPresentation/BackgroundHandler/Infrastructure/ServiceCollectionX.cs b/src/IIIFPresentation/BackgroundHandler/Infrastructure/ServiceCollectionX.cs index 6f436f18..25a19922 100644 --- a/src/IIIFPresentation/BackgroundHandler/Infrastructure/ServiceCollectionX.cs +++ b/src/IIIFPresentation/BackgroundHandler/Infrastructure/ServiceCollectionX.cs @@ -1,6 +1,9 @@ using AWS.Configuration; +using AWS.Helpers; +using AWS.S3; using AWS.Settings; using AWS.SQS; +using BackgroundHandler.BatchCompletion; using BackgroundHandler.CustomerCreation; using BackgroundHandler.Listener; using Repository; @@ -13,12 +16,16 @@ public static IServiceCollection AddAws(this IServiceCollection services, IConfiguration configuration, IWebHostEnvironment webHostEnvironment) { services + .AddSingleton() + .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton(); services .SetupAWS(configuration, webHostEnvironment) - .WithAmazonSQS(); + .WithAmazonSQS() + .WithAmazonS3(); return services; } @@ -33,6 +40,14 @@ public static IServiceCollection AddBackgroundServices(this IServiceCollection s ActivatorUtilities.CreateInstance>(sp, aws.SQS.CustomerCreatedQueueName)) .AddScoped(); } + + if (!string.IsNullOrEmpty(aws.SQS.BatchCompletionQueueName)) + { + services + .AddHostedService(sp => + ActivatorUtilities.CreateInstance>(sp, aws.SQS.BatchCompletionQueueName)) + .AddScoped(); + } return services; } @@ -45,4 +60,4 @@ public static IServiceCollection AddDataAccess(this IServiceCollection services, return services .AddPresentationContext(configuration); } -} \ No newline at end of file +} diff --git a/src/IIIFPresentation/BackgroundHandler/Program.cs b/src/IIIFPresentation/BackgroundHandler/Program.cs index 331b1d8c..bc826faf 100644 --- a/src/IIIFPresentation/BackgroundHandler/Program.cs +++ b/src/IIIFPresentation/BackgroundHandler/Program.cs @@ -1,6 +1,7 @@ using AWS.Settings; using BackgroundHandler.Infrastructure; using BackgroundHandler.Settings; +using DLCS; using Serilog; var builder = WebApplication.CreateBuilder(args); @@ -9,18 +10,26 @@ .Enrich.FromLogContext() .WriteTo.Console() .CreateLogger(); +Log.Information("Application starting..."); -builder.Services.AddSerilog(lc => lc - .ReadFrom.Configuration(builder.Configuration)); +builder.Host.UseSerilog((hostContext, loggerConfig) => + loggerConfig + .ReadFrom.Configuration(hostContext.Configuration) + .Enrich.FromLogContext()); builder.Services.AddOptions() .BindConfiguration(string.Empty); -var aws = builder.Configuration.GetSection("AWS").Get() ?? new AWSSettings(); +var aws = builder.Configuration.GetSection(AWSSettings.SettingsName).Get() ?? new AWSSettings(); +var dlcsSettings = builder.Configuration.GetSection(DlcsSettings.SettingsName); +var dlcs = dlcsSettings.Get()!; -builder.Services.AddAws(builder.Configuration, builder.Environment); -builder.Services.AddDataAccess(builder.Configuration); -builder.Services.AddBackgroundServices(aws); +builder.Services.AddAws(builder.Configuration, builder.Environment) + .AddDataAccess(builder.Configuration) + .AddHttpContextAccessor() + .AddDlcsOrchestratorClient(dlcs) + .AddBackgroundServices(aws) + .Configure(dlcsSettings); var app = builder.Build(); -app.Run(); \ No newline at end of file +app.Run(); diff --git a/src/IIIFPresentation/DLCS.Tests/API/DlcsOrchestratorClientTests.cs b/src/IIIFPresentation/DLCS.Tests/API/DlcsOrchestratorClientTests.cs new file mode 100644 index 00000000..7014418b --- /dev/null +++ b/src/IIIFPresentation/DLCS.Tests/API/DlcsOrchestratorClientTests.cs @@ -0,0 +1,77 @@ +using System.Net; +using DLCS.API; +using DLCS.Exceptions; +using IIIF.Presentation.V3; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Stubbery; + +namespace DLCS.Tests.API; + +public class DlcsOrchestratorClientTests +{ + [Theory] + [InlineData(HttpStatusCode.Forbidden)] + [InlineData(HttpStatusCode.Conflict)] + [InlineData(HttpStatusCode.BadRequest)] + public async Task RetrieveImagesForManifest_Throws_IfDownstreamNon200_NoReturnedError(HttpStatusCode httpStatusCode) + { + using var stub = new ApiStub(); + const int customerId = 3; + stub.Get($"/iiif-resource/v3/{customerId}/batch-query/1,2", (_, _) => string.Empty).StatusCode((int)httpStatusCode); + var sut = GetClient(stub); + + Func action = () => sut.RetrieveAssetsForManifest(customerId, [1, 2], CancellationToken.None); + await action.Should().ThrowAsync().WithMessage("Could not find a DlcsError in response"); + } + + [Theory] + [InlineData(HttpStatusCode.Forbidden)] + [InlineData(HttpStatusCode.Conflict)] + [InlineData(HttpStatusCode.BadRequest)] + public async Task RetrieveImagesForManifest_Throws_IfDownstreamNon200_WithReturnedError(HttpStatusCode httpStatusCode) + { + using var stub = new ApiStub(); + const int customerId = 4; + stub.Get($"/iiif-resource/v3/{customerId}/batch-query/1,2", (_, _) => "{\"description\":\"I am broken\"}") + .StatusCode((int)httpStatusCode); + var sut = GetClient(stub); + + Func action = () => sut.RetrieveAssetsForManifest(customerId, [1, 2], CancellationToken.None); + await action.Should().ThrowAsync().WithMessage("I am broken"); + } + + [Fact] + public async Task RetrieveImagesForManifest_ReturnsManifest() + { + using var stub = new ApiStub(); + const int customerId = 5; + stub.Get($"/iiif-resource/v3/{customerId}/batch-query/1", + (_, _) => "{\"id\":\"some/id\", \"type\": \"Manifest\" }") + .StatusCode(200); + var sut = GetClient(stub); + var expected = new Manifest() { Id = "some/id" }; + + var retrievedImages = await sut.RetrieveAssetsForManifest(customerId, [1], CancellationToken.None); + + retrievedImages.Should().BeEquivalentTo(expected); + } + + private static DlcsOrchestratorClient GetClient(ApiStub stub) + { + stub.EnsureStarted(); + + var httpClient = new HttpClient + { + BaseAddress = new Uri(stub.Address) + }; + + var options = Options.Create(new DlcsSettings() + { + ApiUri = new Uri("https://localhost"), + MaxBatchSize = 1 + }); + + return new DlcsOrchestratorClient(httpClient, options); + } +} diff --git a/src/IIIFPresentation/DLCS/API/DlcsApiClient.cs b/src/IIIFPresentation/DLCS/API/DlcsApiClient.cs index 3e263321..c426c352 100644 --- a/src/IIIFPresentation/DLCS/API/DlcsApiClient.cs +++ b/src/IIIFPresentation/DLCS/API/DlcsApiClient.cs @@ -1,6 +1,5 @@ using System.Collections.Concurrent; using System.Net; -using System.Text.Json; using DLCS.Exceptions; using DLCS.Handlers; using DLCS.Models; @@ -90,7 +89,7 @@ public async Task> IngestAssets(int customerId, List assets, C { var request = new HttpRequestMessage(httpMethod, path); request.Content = DlcsHttpContent.GenerateJsonContent(payload); - + var response = await httpClient.SendAsync(request, cancellationToken); return await response.ReadAsDlcsResponse(cancellationToken); } diff --git a/src/IIIFPresentation/DLCS/API/DlcsHttpContent.cs b/src/IIIFPresentation/DLCS/API/DlcsHttpContent.cs index 11b2aa92..987055c0 100644 --- a/src/IIIFPresentation/DLCS/API/DlcsHttpContent.cs +++ b/src/IIIFPresentation/DLCS/API/DlcsHttpContent.cs @@ -6,6 +6,8 @@ using DLCS.Converters; using DLCS.Exceptions; using DLCS.Models; +using IIIF.Serialisation; +using JsonLdBase = IIIF.JsonLdBase; namespace DLCS.API; @@ -38,20 +40,36 @@ public static StringContent GenerateJsonContent(T body) return await response.ReadDlcsModel(true, cancellationToken); } + throw await CheckAndThrowResponseError(response, cancellationToken); + } + + public static async Task ReadAsIIIFResponse(this HttpResponseMessage response, + CancellationToken cancellationToken = default) where T : JsonLdBase + { + if (response.IsSuccessStatusCode) + { + return (await response.Content.ReadAsStreamAsync(cancellationToken)).FromJsonStream(); + } + + throw await CheckAndThrowResponseError(response, cancellationToken); + } + + private static async Task CheckAndThrowResponseError(HttpResponseMessage response, CancellationToken cancellationToken) + { try { var error = await response.Content.ReadFromJsonAsync(JsonSerializerOptions, cancellationToken); if (error != null) { - throw new DlcsException(error.Description); + return new DlcsException(error.Description); } throw new DlcsException("Unable to process error condition"); } catch (Exception ex) when (ex is not DlcsException) { - throw new DlcsException("Could not find a DlcsError in response", ex); + return new DlcsException("Could not find a DlcsError in response", ex); } } diff --git a/src/IIIFPresentation/DLCS/API/DlcsOrchestratorClient.cs b/src/IIIFPresentation/DLCS/API/DlcsOrchestratorClient.cs new file mode 100644 index 00000000..5fd37ae8 --- /dev/null +++ b/src/IIIFPresentation/DLCS/API/DlcsOrchestratorClient.cs @@ -0,0 +1,33 @@ +using IIIF.Presentation.V3; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace DLCS.API; + +public interface IDlcsOrchestratorClient +{ + /// + /// Retrieves a DLCS generated manifest of images for a given presentation manifest id + /// + public Task RetrieveAssetsForManifest(int customerId, List batches, + CancellationToken cancellationToken = default); +} + +public class DlcsOrchestratorClient( + HttpClient httpClient, + IOptions dlcsOptions) : IDlcsOrchestratorClient +{ + private readonly DlcsSettings settings = dlcsOptions.Value; + + public async Task RetrieveAssetsForManifest(int customerId, List batches, + CancellationToken cancellationToken = default) + { + var batchString = string.Join(',', batches); + + var response = + await httpClient.GetAsync($"/iiif-resource/v3/{customerId}/{settings.ManifestNamedQueryName}/{batchString}", + cancellationToken); + + return await response.ReadAsIIIFResponse(cancellationToken); + } +} diff --git a/src/IIIFPresentation/DLCS/DlcsSettings.cs b/src/IIIFPresentation/DLCS/DlcsSettings.cs index f571229d..c1614a62 100644 --- a/src/IIIFPresentation/DLCS/DlcsSettings.cs +++ b/src/IIIFPresentation/DLCS/DlcsSettings.cs @@ -8,14 +8,29 @@ public class DlcsSettings /// URL root of DLCS API /// public required Uri ApiUri { get; set; } + + /// + /// URL root of DLCS Orchestrator + /// + public Uri? OrchestratorUri { get; set; } /// - /// Default timeout (in ms) use for HttpClient.Timeout. + /// Default timeout (in ms) use for HttpClient.Timeout in the API. /// - public int DefaultTimeoutMs { get; set; } = 30000; + public int ApiDefaultTimeoutMs { get; set; } = 30000; + + /// + /// Default timeout (in ms) use for HttpClient.Timeout in orchestrator. + /// + public int OrchestratorDefaultTimeoutMs { get; set; } = 30000; /// /// The maximum size of an individual batch request /// public int MaxBatchSize { get; set; } = 100; + + /// + /// The name of the query used for retrieving images + /// + public string ManifestNamedQueryName { get; set; } = "batch-query"; } diff --git a/src/IIIFPresentation/DLCS/Handlers/AmbientAuthHandler.cs b/src/IIIFPresentation/DLCS/Handlers/AmbientAuthHandler.cs index a0adf678..b3454bb2 100644 --- a/src/IIIFPresentation/DLCS/Handlers/AmbientAuthHandler.cs +++ b/src/IIIFPresentation/DLCS/Handlers/AmbientAuthHandler.cs @@ -25,4 +25,4 @@ protected override Task SendAsync(HttpRequestMessage reques request.Headers.Authorization = authHeader; return base.SendAsync(request, cancellationToken); } -} \ No newline at end of file +} diff --git a/src/IIIFPresentation/DLCS/ServiceCollectionX.cs b/src/IIIFPresentation/DLCS/ServiceCollectionX.cs index cda11aab..8b98faa6 100644 --- a/src/IIIFPresentation/DLCS/ServiceCollectionX.cs +++ b/src/IIIFPresentation/DLCS/ServiceCollectionX.cs @@ -12,7 +12,7 @@ public static class ServiceCollectionX /// /// Add to services collection. /// - public static IServiceCollection AddDlcsClient(this IServiceCollection services, + public static IServiceCollection AddDlcsApiClient(this IServiceCollection services, DlcsSettings dlcsSettings) { services @@ -21,10 +21,28 @@ public static IServiceCollection AddDlcsClient(this IServiceCollection services, .AddHttpClient(client => { client.BaseAddress = dlcsSettings.ApiUri; - client.Timeout = TimeSpan.FromMilliseconds(dlcsSettings.DefaultTimeoutMs); + client.Timeout = TimeSpan.FromMilliseconds(dlcsSettings.ApiDefaultTimeoutMs); }).AddHttpMessageHandler() .AddHttpMessageHandler(); return services; } -} \ No newline at end of file + + /// + /// Add to services collection. + /// + public static IServiceCollection AddDlcsOrchestratorClient(this IServiceCollection services, + DlcsSettings dlcsSettings) + { + services + .AddTransient() + .AddHttpClient(client => + { + client.BaseAddress = dlcsSettings.OrchestratorUri; + client.Timeout = TimeSpan.FromMilliseconds(dlcsSettings.OrchestratorDefaultTimeoutMs); + }) + .AddHttpMessageHandler(); + + return services; + } +} diff --git a/src/IIIFPresentation/Models/API/Manifest/PresentationManifest.cs b/src/IIIFPresentation/Models/API/Manifest/PresentationManifest.cs index 9d1d2c82..3107b8ec 100644 --- a/src/IIIFPresentation/Models/API/Manifest/PresentationManifest.cs +++ b/src/IIIFPresentation/Models/API/Manifest/PresentationManifest.cs @@ -54,5 +54,4 @@ public class CanvasPainting public string? Target { get; set; } public int? StaticWidth { get; set; } public int? StaticHeight { get; set; } - public string? AssetId { get; set; } } diff --git a/src/IIIFPresentation/Models/DLCS/AssetId.cs b/src/IIIFPresentation/Models/DLCS/AssetId.cs new file mode 100644 index 00000000..5e31176f --- /dev/null +++ b/src/IIIFPresentation/Models/DLCS/AssetId.cs @@ -0,0 +1,82 @@ +namespace Models.DLCS; + + +/// +/// A record that represents an identifier for a DLCS Asset. +/// +public class AssetId +{ + /// Id of customer + public int Customer { get; } + + /// Id of space + public int Space { get; } + + /// Id of asset + public string Asset { get; } + + /// + /// A record that represents an identifier for a DLCS Asset. + /// + public AssetId(int customer, int space, string asset) + { + Customer = customer; + Space = space; + Asset = asset; + } + + public override string ToString() => $"{Customer}/{Space}/{Asset}"; + + /// + /// Create a new AssetId from string in format customer/space/image + /// + public static AssetId? FromString(string assetImageId) + { + var parts = assetImageId.Split("/", StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 3) + { + throw new ArgumentException( + $"AssetId '{assetImageId}' is invalid. Must be in format customer/space/asset"); + } + + try + { + return new AssetId(int.Parse(parts[0]), int.Parse(parts[1]), parts[2]); + } + catch (FormatException fmEx) + { + throw new ArgumentException( + $"AssetId '{assetImageId}' is invalid. Must be in format customer/space/asset", + fmEx); + } + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + var asset = (AssetId)obj; + return asset.ToString() == this.ToString(); + } + + public static bool operator ==(AssetId? assetId1, AssetId? assetId2) + { + if (assetId1 is null) + { + return assetId2 is null; + } + + if (assetId2 is null) + { + return false; + } + + return assetId1.Equals(assetId2); + } + + public static bool operator !=(AssetId? assetId1, AssetId? assetId2) + => !(assetId1 == assetId2); + + public override int GetHashCode() => HashCode.Combine(Customer, Space, Asset); +} diff --git a/src/IIIFPresentation/Models/Database/CanvasPainting.cs b/src/IIIFPresentation/Models/Database/CanvasPainting.cs index 2af173c8..756603d4 100644 --- a/src/IIIFPresentation/Models/Database/CanvasPainting.cs +++ b/src/IIIFPresentation/Models/Database/CanvasPainting.cs @@ -1,5 +1,6 @@ using IIIF.Presentation.V3.Strings; using Models.Database.Collections; +using Models.DLCS; namespace Models.Database; @@ -118,7 +119,12 @@ public int? ChoiceOrder /// /// An asset id showing this asset is an internal item /// - public string? AssetId { get; set; } + public AssetId? AssetId { get; set; } + + /// + /// Whether the asset is currently being ingested into the DLCS + /// + public bool Ingesting { get; set; } } public static class CanvasPaintingX diff --git a/src/IIIFPresentation/Models/Database/General/Batch.cs b/src/IIIFPresentation/Models/Database/General/Batch.cs index 8187dc1f..47d3ae47 100644 --- a/src/IIIFPresentation/Models/Database/General/Batch.cs +++ b/src/IIIFPresentation/Models/Database/General/Batch.cs @@ -24,6 +24,16 @@ public class Batch /// public DateTime Submitted { get; set; } + /// + /// When the batch was finished by the DLCS + /// + public DateTime? Finished { get; set; } + + /// + /// When the batch was processed by presentation + /// + public DateTime? Processed { get; set; } + /// /// Id of related manifest /// diff --git a/src/IIIFPresentation/Repository/Migrations/20241217155051_addFinishedToBatch.Designer.cs b/src/IIIFPresentation/Repository/Migrations/20241217155051_addFinishedToBatch.Designer.cs new file mode 100644 index 00000000..3edc62e3 --- /dev/null +++ b/src/IIIFPresentation/Repository/Migrations/20241217155051_addFinishedToBatch.Designer.cs @@ -0,0 +1,404 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Repository; + +#nullable disable + +namespace Repository.Migrations +{ + [DbContext(typeof(PresentationContext))] + [Migration("20241217155051_addFinishedToBatch")] + partial class addFinishedToBatch + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "citext"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Models.Database.CanvasPainting", b => + { + b.Property("CanvasPaintingId") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("canvas_painting_id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("CanvasPaintingId")); + + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("CanvasLabel") + .HasColumnType("text") + .HasColumnName("canvas_label"); + + b.Property("CanvasOrder") + .HasColumnType("integer") + .HasColumnName("canvas_order"); + + b.Property("CanvasOriginalId") + .HasColumnType("text") + .HasColumnName("canvas_original_id"); + + b.Property("ChoiceOrder") + .IsRequired() + .HasColumnType("integer") + .HasColumnName("choice_order"); + + b.Property("Created") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created") + .HasDefaultValueSql("now()"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Id") + .HasColumnType("text") + .HasColumnName("canvas_id"); + + b.Property("Label") + .HasColumnType("jsonb") + .HasColumnName("label"); + + b.Property("ManifestId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Modified") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("modified") + .HasDefaultValueSql("now()"); + + b.Property("StaticHeight") + .HasColumnType("integer") + .HasColumnName("static_height"); + + b.Property("StaticWidth") + .HasColumnType("integer") + .HasColumnName("static_width"); + + b.Property("Target") + .HasColumnType("text") + .HasColumnName("target"); + + b.Property("Thumbnail") + .HasColumnType("text") + .HasColumnName("thumbnail"); + + b.HasKey("CanvasPaintingId") + .HasName("pk_canvas_paintings"); + + b.HasIndex("ManifestId", "CustomerId") + .HasDatabaseName("ix_canvas_paintings_manifest_id_customer_id"); + + b.HasIndex("Id", "CustomerId", "ManifestId", "AssetId", "CanvasOrder", "ChoiceOrder") + .IsUnique() + .HasDatabaseName("ix_canvas_paintings_canvas_id_customer_id_manifest_id_asset_id") + .HasFilter("canvas_original_id is null"); + + b.HasIndex("Id", "CustomerId", "ManifestId", "CanvasOriginalId", "CanvasOrder", "ChoiceOrder") + .IsUnique() + .HasDatabaseName("ix_canvas_paintings_canvas_id_customer_id_manifest_id_canvas_o") + .HasFilter("asset_id is null"); + + b.ToTable("canvas_paintings", (string)null); + }); + + modelBuilder.Entity("Models.Database.Collections.Collection", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Created") + .HasColumnType("timestamp with time zone") + .HasColumnName("created"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("IsPublic") + .HasColumnType("boolean") + .HasColumnName("is_public"); + + b.Property("IsStorageCollection") + .HasColumnType("boolean") + .HasColumnName("is_storage_collection"); + + b.Property("Label") + .HasColumnType("jsonb") + .HasColumnName("label"); + + b.Property("LockedBy") + .HasColumnType("text") + .HasColumnName("locked_by"); + + b.Property("Modified") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("Tags") + .HasColumnType("text") + .HasColumnName("tags"); + + b.Property("Thumbnail") + .HasColumnType("text") + .HasColumnName("thumbnail"); + + b.Property("UsePath") + .HasColumnType("boolean") + .HasColumnName("use_path"); + + b.HasKey("Id", "CustomerId") + .HasName("pk_collections"); + + b.ToTable("collections", (string)null); + }); + + modelBuilder.Entity("Models.Database.Collections.Manifest", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Created") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created") + .HasDefaultValueSql("now()"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("Label") + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("Modified") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("modified") + .HasDefaultValueSql("now()"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("SpaceId") + .HasColumnType("integer") + .HasColumnName("space_id"); + + b.HasKey("Id", "CustomerId") + .HasName("pk_manifests"); + + b.ToTable("manifests", (string)null); + }); + + modelBuilder.Entity("Models.Database.General.Batch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Finished") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished"); + + b.Property("ManifestId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("Submitted") + .HasColumnType("timestamp with time zone") + .HasColumnName("submitted"); + + b.HasKey("Id") + .HasName("pk_batches"); + + b.HasIndex("ManifestId", "CustomerId") + .HasDatabaseName("ix_batches_manifest_id_customer_id"); + + b.ToTable("batches", (string)null); + }); + + modelBuilder.Entity("Models.Database.General.Hierarchy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Canonical") + .HasColumnType("boolean") + .HasColumnName("canonical"); + + b.Property("CollectionId") + .HasColumnType("text") + .HasColumnName("collection_id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("ItemsOrder") + .HasColumnType("integer") + .HasColumnName("items_order"); + + b.Property("ManifestId") + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Parent") + .HasColumnType("text") + .HasColumnName("parent"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("citext") + .HasColumnName("slug"); + + b.Property("Type") + .HasColumnType("integer") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_hierarchy"); + + b.HasIndex("Parent", "CustomerId") + .HasDatabaseName("ix_hierarchy_parent_customer_id"); + + b.HasIndex("CollectionId", "CustomerId", "Canonical") + .IsUnique() + .HasDatabaseName("ix_hierarchy_collection_id_customer_id_canonical") + .HasFilter("canonical is true"); + + b.HasIndex("CustomerId", "Slug", "Parent") + .IsUnique() + .HasDatabaseName("ix_hierarchy_customer_id_slug_parent"); + + b.HasIndex("ManifestId", "CustomerId", "Canonical") + .IsUnique() + .HasDatabaseName("ix_hierarchy_manifest_id_customer_id_canonical") + .HasFilter("canonical is true"); + + b.ToTable("hierarchy", null, t => + { + t.HasCheckConstraint("stop_collection_and_manifest_in_same_record", "num_nonnulls(manifest_id, collection_id) = 1"); + }); + }); + + modelBuilder.Entity("Models.Database.CanvasPainting", b => + { + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("CanvasPaintings") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_canvas_paintings_manifests_manifest_id_customer_id"); + + b.Navigation("Manifest"); + }); + + modelBuilder.Entity("Models.Database.General.Batch", b => + { + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("Batches") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_batches_manifests_manifest_id_customer_id"); + + b.Navigation("Manifest"); + }); + + modelBuilder.Entity("Models.Database.General.Hierarchy", b => + { + b.HasOne("Models.Database.Collections.Collection", "Collection") + .WithMany("Hierarchy") + .HasForeignKey("CollectionId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_hierarchy_collections_collection_id_customer_id"); + + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("Hierarchy") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_hierarchy_manifests_manifest_id_customer_id"); + + b.HasOne("Models.Database.Collections.Collection", "ParentCollection") + .WithMany("Children") + .HasForeignKey("Parent", "CustomerId") + .OnDelete(DeleteBehavior.NoAction) + .HasConstraintName("fk_hierarchy_collections_parent_customer_id"); + + b.Navigation("Collection"); + + b.Navigation("Manifest"); + + b.Navigation("ParentCollection"); + }); + + modelBuilder.Entity("Models.Database.Collections.Collection", b => + { + b.Navigation("Children"); + + b.Navigation("Hierarchy"); + }); + + modelBuilder.Entity("Models.Database.Collections.Manifest", b => + { + b.Navigation("Batches"); + + b.Navigation("CanvasPaintings"); + + b.Navigation("Hierarchy"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/IIIFPresentation/Repository/Migrations/20241217155051_addFinishedToBatch.cs b/src/IIIFPresentation/Repository/Migrations/20241217155051_addFinishedToBatch.cs new file mode 100644 index 00000000..1250c059 --- /dev/null +++ b/src/IIIFPresentation/Repository/Migrations/20241217155051_addFinishedToBatch.cs @@ -0,0 +1,29 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Repository.Migrations +{ + /// + public partial class addFinishedToBatch : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "finished", + table: "batches", + type: "timestamp with time zone", + nullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "finished", + table: "batches"); + } + } +} diff --git a/src/IIIFPresentation/Repository/Migrations/20241219171537_renameBatchFinishedAndAddIngesting.Designer.cs b/src/IIIFPresentation/Repository/Migrations/20241219171537_renameBatchFinishedAndAddIngesting.Designer.cs new file mode 100644 index 00000000..8407bdee --- /dev/null +++ b/src/IIIFPresentation/Repository/Migrations/20241219171537_renameBatchFinishedAndAddIngesting.Designer.cs @@ -0,0 +1,408 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Repository; + +#nullable disable + +namespace Repository.Migrations +{ + [DbContext(typeof(PresentationContext))] + [Migration("20241219171537_renameBatchFinishedAndAddIngesting")] + partial class renameBatchFinishedAndAddIngesting + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "citext"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Models.Database.CanvasPainting", b => + { + b.Property("CanvasPaintingId") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("canvas_painting_id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("CanvasPaintingId")); + + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("CanvasLabel") + .HasColumnType("text") + .HasColumnName("canvas_label"); + + b.Property("CanvasOrder") + .HasColumnType("integer") + .HasColumnName("canvas_order"); + + b.Property("CanvasOriginalId") + .HasColumnType("text") + .HasColumnName("canvas_original_id"); + + b.Property("ChoiceOrder") + .IsRequired() + .HasColumnType("integer") + .HasColumnName("choice_order"); + + b.Property("Created") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created") + .HasDefaultValueSql("now()"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Id") + .HasColumnType("text") + .HasColumnName("canvas_id"); + + b.Property("Ingesting") + .HasColumnType("boolean") + .HasColumnName("ingesting"); + + b.Property("Label") + .HasColumnType("jsonb") + .HasColumnName("label"); + + b.Property("ManifestId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Modified") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("modified") + .HasDefaultValueSql("now()"); + + b.Property("StaticHeight") + .HasColumnType("integer") + .HasColumnName("static_height"); + + b.Property("StaticWidth") + .HasColumnType("integer") + .HasColumnName("static_width"); + + b.Property("Target") + .HasColumnType("text") + .HasColumnName("target"); + + b.Property("Thumbnail") + .HasColumnType("text") + .HasColumnName("thumbnail"); + + b.HasKey("CanvasPaintingId") + .HasName("pk_canvas_paintings"); + + b.HasIndex("ManifestId", "CustomerId") + .HasDatabaseName("ix_canvas_paintings_manifest_id_customer_id"); + + b.HasIndex("Id", "CustomerId", "ManifestId", "AssetId", "CanvasOrder", "ChoiceOrder") + .IsUnique() + .HasDatabaseName("ix_canvas_paintings_canvas_id_customer_id_manifest_id_asset_id") + .HasFilter("canvas_original_id is null"); + + b.HasIndex("Id", "CustomerId", "ManifestId", "CanvasOriginalId", "CanvasOrder", "ChoiceOrder") + .IsUnique() + .HasDatabaseName("ix_canvas_paintings_canvas_id_customer_id_manifest_id_canvas_o") + .HasFilter("asset_id is null"); + + b.ToTable("canvas_paintings", (string)null); + }); + + modelBuilder.Entity("Models.Database.Collections.Collection", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Created") + .HasColumnType("timestamp with time zone") + .HasColumnName("created"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("IsPublic") + .HasColumnType("boolean") + .HasColumnName("is_public"); + + b.Property("IsStorageCollection") + .HasColumnType("boolean") + .HasColumnName("is_storage_collection"); + + b.Property("Label") + .HasColumnType("jsonb") + .HasColumnName("label"); + + b.Property("LockedBy") + .HasColumnType("text") + .HasColumnName("locked_by"); + + b.Property("Modified") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("Tags") + .HasColumnType("text") + .HasColumnName("tags"); + + b.Property("Thumbnail") + .HasColumnType("text") + .HasColumnName("thumbnail"); + + b.Property("UsePath") + .HasColumnType("boolean") + .HasColumnName("use_path"); + + b.HasKey("Id", "CustomerId") + .HasName("pk_collections"); + + b.ToTable("collections", (string)null); + }); + + modelBuilder.Entity("Models.Database.Collections.Manifest", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Created") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created") + .HasDefaultValueSql("now()"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("Label") + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("Modified") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("modified") + .HasDefaultValueSql("now()"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("SpaceId") + .HasColumnType("integer") + .HasColumnName("space_id"); + + b.HasKey("Id", "CustomerId") + .HasName("pk_manifests"); + + b.ToTable("manifests", (string)null); + }); + + modelBuilder.Entity("Models.Database.General.Batch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("ManifestId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Processed") + .HasColumnType("timestamp with time zone") + .HasColumnName("processed"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("Submitted") + .HasColumnType("timestamp with time zone") + .HasColumnName("submitted"); + + b.HasKey("Id") + .HasName("pk_batches"); + + b.HasIndex("ManifestId", "CustomerId") + .HasDatabaseName("ix_batches_manifest_id_customer_id"); + + b.ToTable("batches", (string)null); + }); + + modelBuilder.Entity("Models.Database.General.Hierarchy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Canonical") + .HasColumnType("boolean") + .HasColumnName("canonical"); + + b.Property("CollectionId") + .HasColumnType("text") + .HasColumnName("collection_id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("ItemsOrder") + .HasColumnType("integer") + .HasColumnName("items_order"); + + b.Property("ManifestId") + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Parent") + .HasColumnType("text") + .HasColumnName("parent"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("citext") + .HasColumnName("slug"); + + b.Property("Type") + .HasColumnType("integer") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_hierarchy"); + + b.HasIndex("Parent", "CustomerId") + .HasDatabaseName("ix_hierarchy_parent_customer_id"); + + b.HasIndex("CollectionId", "CustomerId", "Canonical") + .IsUnique() + .HasDatabaseName("ix_hierarchy_collection_id_customer_id_canonical") + .HasFilter("canonical is true"); + + b.HasIndex("CustomerId", "Slug", "Parent") + .IsUnique() + .HasDatabaseName("ix_hierarchy_customer_id_slug_parent"); + + b.HasIndex("ManifestId", "CustomerId", "Canonical") + .IsUnique() + .HasDatabaseName("ix_hierarchy_manifest_id_customer_id_canonical") + .HasFilter("canonical is true"); + + b.ToTable("hierarchy", null, t => + { + t.HasCheckConstraint("stop_collection_and_manifest_in_same_record", "num_nonnulls(manifest_id, collection_id) = 1"); + }); + }); + + modelBuilder.Entity("Models.Database.CanvasPainting", b => + { + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("CanvasPaintings") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_canvas_paintings_manifests_manifest_id_customer_id"); + + b.Navigation("Manifest"); + }); + + modelBuilder.Entity("Models.Database.General.Batch", b => + { + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("Batches") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_batches_manifests_manifest_id_customer_id"); + + b.Navigation("Manifest"); + }); + + modelBuilder.Entity("Models.Database.General.Hierarchy", b => + { + b.HasOne("Models.Database.Collections.Collection", "Collection") + .WithMany("Hierarchy") + .HasForeignKey("CollectionId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_hierarchy_collections_collection_id_customer_id"); + + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("Hierarchy") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_hierarchy_manifests_manifest_id_customer_id"); + + b.HasOne("Models.Database.Collections.Collection", "ParentCollection") + .WithMany("Children") + .HasForeignKey("Parent", "CustomerId") + .OnDelete(DeleteBehavior.NoAction) + .HasConstraintName("fk_hierarchy_collections_parent_customer_id"); + + b.Navigation("Collection"); + + b.Navigation("Manifest"); + + b.Navigation("ParentCollection"); + }); + + modelBuilder.Entity("Models.Database.Collections.Collection", b => + { + b.Navigation("Children"); + + b.Navigation("Hierarchy"); + }); + + modelBuilder.Entity("Models.Database.Collections.Manifest", b => + { + b.Navigation("Batches"); + + b.Navigation("CanvasPaintings"); + + b.Navigation("Hierarchy"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/IIIFPresentation/Repository/Migrations/20241219171537_renameBatchFinishedAndAddIngesting.cs b/src/IIIFPresentation/Repository/Migrations/20241219171537_renameBatchFinishedAndAddIngesting.cs new file mode 100644 index 00000000..dc088ee0 --- /dev/null +++ b/src/IIIFPresentation/Repository/Migrations/20241219171537_renameBatchFinishedAndAddIngesting.cs @@ -0,0 +1,39 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Repository.Migrations +{ + /// + public partial class renameBatchFinishedAndAddIngesting : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.RenameColumn( + name: "finished", + table: "batches", + newName: "processed"); + + migrationBuilder.AddColumn( + name: "ingesting", + table: "canvas_paintings", + type: "boolean", + nullable: false, + defaultValue: false); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "ingesting", + table: "canvas_paintings"); + + migrationBuilder.RenameColumn( + name: "processed", + table: "batches", + newName: "finished"); + } + } +} diff --git a/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.Designer.cs b/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.Designer.cs new file mode 100644 index 00000000..1cd704cb --- /dev/null +++ b/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.Designer.cs @@ -0,0 +1,412 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Repository; + +#nullable disable + +namespace Repository.Migrations +{ + [DbContext(typeof(PresentationContext))] + [Migration("20250114154941_useAssetIdAndAddFinishedToBatch")] + partial class useAssetIdAndAddFinishedToBatch + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "citext"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Models.Database.CanvasPainting", b => + { + b.Property("CanvasPaintingId") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("canvas_painting_id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("CanvasPaintingId")); + + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("CanvasLabel") + .HasColumnType("text") + .HasColumnName("canvas_label"); + + b.Property("CanvasOrder") + .HasColumnType("integer") + .HasColumnName("canvas_order"); + + b.Property("CanvasOriginalId") + .HasColumnType("text") + .HasColumnName("canvas_original_id"); + + b.Property("ChoiceOrder") + .IsRequired() + .HasColumnType("integer") + .HasColumnName("choice_order"); + + b.Property("Created") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created") + .HasDefaultValueSql("now()"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Id") + .HasColumnType("text") + .HasColumnName("canvas_id"); + + b.Property("Ingesting") + .HasColumnType("boolean") + .HasColumnName("ingesting"); + + b.Property("Label") + .HasColumnType("jsonb") + .HasColumnName("label"); + + b.Property("ManifestId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Modified") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("modified") + .HasDefaultValueSql("now()"); + + b.Property("StaticHeight") + .HasColumnType("integer") + .HasColumnName("static_height"); + + b.Property("StaticWidth") + .HasColumnType("integer") + .HasColumnName("static_width"); + + b.Property("Target") + .HasColumnType("text") + .HasColumnName("target"); + + b.Property("Thumbnail") + .HasColumnType("text") + .HasColumnName("thumbnail"); + + b.HasKey("CanvasPaintingId") + .HasName("pk_canvas_paintings"); + + b.HasIndex("ManifestId", "CustomerId") + .HasDatabaseName("ix_canvas_paintings_manifest_id_customer_id"); + + b.HasIndex("Id", "CustomerId", "ManifestId", "AssetId", "CanvasOrder", "ChoiceOrder") + .IsUnique() + .HasDatabaseName("ix_canvas_paintings_canvas_id_customer_id_manifest_id_asset_id") + .HasFilter("canvas_original_id is null"); + + b.HasIndex("Id", "CustomerId", "ManifestId", "CanvasOriginalId", "CanvasOrder", "ChoiceOrder") + .IsUnique() + .HasDatabaseName("ix_canvas_paintings_canvas_id_customer_id_manifest_id_canvas_o") + .HasFilter("asset_id is null"); + + b.ToTable("canvas_paintings", (string)null); + }); + + modelBuilder.Entity("Models.Database.Collections.Collection", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Created") + .HasColumnType("timestamp with time zone") + .HasColumnName("created"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("IsPublic") + .HasColumnType("boolean") + .HasColumnName("is_public"); + + b.Property("IsStorageCollection") + .HasColumnType("boolean") + .HasColumnName("is_storage_collection"); + + b.Property("Label") + .HasColumnType("jsonb") + .HasColumnName("label"); + + b.Property("LockedBy") + .HasColumnType("text") + .HasColumnName("locked_by"); + + b.Property("Modified") + .HasColumnType("timestamp with time zone") + .HasColumnName("modified"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("Tags") + .HasColumnType("text") + .HasColumnName("tags"); + + b.Property("Thumbnail") + .HasColumnType("text") + .HasColumnName("thumbnail"); + + b.Property("UsePath") + .HasColumnType("boolean") + .HasColumnName("use_path"); + + b.HasKey("Id", "CustomerId") + .HasName("pk_collections"); + + b.ToTable("collections", (string)null); + }); + + modelBuilder.Entity("Models.Database.Collections.Manifest", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Created") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created") + .HasDefaultValueSql("now()"); + + b.Property("CreatedBy") + .HasColumnType("text") + .HasColumnName("created_by"); + + b.Property("Label") + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("Modified") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("modified") + .HasDefaultValueSql("now()"); + + b.Property("ModifiedBy") + .HasColumnType("text") + .HasColumnName("modified_by"); + + b.Property("SpaceId") + .HasColumnType("integer") + .HasColumnName("space_id"); + + b.HasKey("Id", "CustomerId") + .HasName("pk_manifests"); + + b.ToTable("manifests", (string)null); + }); + + modelBuilder.Entity("Models.Database.General.Batch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Finished") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished"); + + b.Property("ManifestId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Processed") + .HasColumnType("timestamp with time zone") + .HasColumnName("processed"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("Submitted") + .HasColumnType("timestamp with time zone") + .HasColumnName("submitted"); + + b.HasKey("Id") + .HasName("pk_batches"); + + b.HasIndex("ManifestId", "CustomerId") + .HasDatabaseName("ix_batches_manifest_id_customer_id"); + + b.ToTable("batches", (string)null); + }); + + modelBuilder.Entity("Models.Database.General.Hierarchy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Canonical") + .HasColumnType("boolean") + .HasColumnName("canonical"); + + b.Property("CollectionId") + .HasColumnType("text") + .HasColumnName("collection_id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("ItemsOrder") + .HasColumnType("integer") + .HasColumnName("items_order"); + + b.Property("ManifestId") + .HasColumnType("text") + .HasColumnName("manifest_id"); + + b.Property("Parent") + .HasColumnType("text") + .HasColumnName("parent"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("citext") + .HasColumnName("slug"); + + b.Property("Type") + .HasColumnType("integer") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_hierarchy"); + + b.HasIndex("Parent", "CustomerId") + .HasDatabaseName("ix_hierarchy_parent_customer_id"); + + b.HasIndex("CollectionId", "CustomerId", "Canonical") + .IsUnique() + .HasDatabaseName("ix_hierarchy_collection_id_customer_id_canonical") + .HasFilter("canonical is true"); + + b.HasIndex("CustomerId", "Slug", "Parent") + .IsUnique() + .HasDatabaseName("ix_hierarchy_customer_id_slug_parent"); + + b.HasIndex("ManifestId", "CustomerId", "Canonical") + .IsUnique() + .HasDatabaseName("ix_hierarchy_manifest_id_customer_id_canonical") + .HasFilter("canonical is true"); + + b.ToTable("hierarchy", null, t => + { + t.HasCheckConstraint("stop_collection_and_manifest_in_same_record", "num_nonnulls(manifest_id, collection_id) = 1"); + }); + }); + + modelBuilder.Entity("Models.Database.CanvasPainting", b => + { + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("CanvasPaintings") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_canvas_paintings_manifests_manifest_id_customer_id"); + + b.Navigation("Manifest"); + }); + + modelBuilder.Entity("Models.Database.General.Batch", b => + { + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("Batches") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_batches_manifests_manifest_id_customer_id"); + + b.Navigation("Manifest"); + }); + + modelBuilder.Entity("Models.Database.General.Hierarchy", b => + { + b.HasOne("Models.Database.Collections.Collection", "Collection") + .WithMany("Hierarchy") + .HasForeignKey("CollectionId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_hierarchy_collections_collection_id_customer_id"); + + b.HasOne("Models.Database.Collections.Manifest", "Manifest") + .WithMany("Hierarchy") + .HasForeignKey("ManifestId", "CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_hierarchy_manifests_manifest_id_customer_id"); + + b.HasOne("Models.Database.Collections.Collection", "ParentCollection") + .WithMany("Children") + .HasForeignKey("Parent", "CustomerId") + .OnDelete(DeleteBehavior.NoAction) + .HasConstraintName("fk_hierarchy_collections_parent_customer_id"); + + b.Navigation("Collection"); + + b.Navigation("Manifest"); + + b.Navigation("ParentCollection"); + }); + + modelBuilder.Entity("Models.Database.Collections.Collection", b => + { + b.Navigation("Children"); + + b.Navigation("Hierarchy"); + }); + + modelBuilder.Entity("Models.Database.Collections.Manifest", b => + { + b.Navigation("Batches"); + + b.Navigation("CanvasPaintings"); + + b.Navigation("Hierarchy"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.cs b/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.cs new file mode 100644 index 00000000..1b910226 --- /dev/null +++ b/src/IIIFPresentation/Repository/Migrations/20250114154941_useAssetIdAndAddFinishedToBatch.cs @@ -0,0 +1,29 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Repository.Migrations +{ + /// + public partial class useAssetIdAndAddFinishedToBatch : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "finished", + table: "batches", + type: "timestamp with time zone", + nullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "finished", + table: "batches"); + } + } +} diff --git a/src/IIIFPresentation/Repository/Migrations/PresentationContextModelSnapshot.cs b/src/IIIFPresentation/Repository/Migrations/PresentationContextModelSnapshot.cs index 02ebdfb8..cfc2b33b 100644 --- a/src/IIIFPresentation/Repository/Migrations/PresentationContextModelSnapshot.cs +++ b/src/IIIFPresentation/Repository/Migrations/PresentationContextModelSnapshot.cs @@ -67,6 +67,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("text") .HasColumnName("canvas_id"); + b.Property("Ingesting") + .HasColumnType("boolean") + .HasColumnName("ingesting"); + b.Property("Label") .HasColumnType("jsonb") .HasColumnName("label"); @@ -234,11 +238,19 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("integer") .HasColumnName("customer_id"); + b.Property("Finished") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished"); + b.Property("ManifestId") .IsRequired() .HasColumnType("text") .HasColumnName("manifest_id"); + b.Property("Processed") + .HasColumnType("timestamp with time zone") + .HasColumnName("processed"); + b.Property("Status") .IsRequired() .HasColumnType("text") diff --git a/src/IIIFPresentation/Repository/PresentationContext.cs b/src/IIIFPresentation/Repository/PresentationContext.cs index eb72a8c6..f675245b 100644 --- a/src/IIIFPresentation/Repository/PresentationContext.cs +++ b/src/IIIFPresentation/Repository/PresentationContext.cs @@ -4,6 +4,7 @@ using Models.Database; using Models.Database.Collections; using Models.Database.General; +using Models.DLCS; using Repository.Converters; namespace Repository; @@ -120,6 +121,9 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .IsUnique() .HasFilter("canvas_original_id is null"); + entity.Property(cp => cp.AssetId) + .HasConversion(id => id.ToString(), id => AssetId.FromString(id)); + entity .HasOne(cp => cp.Manifest) .WithMany(m => m.CanvasPaintings)