initial commit adding ability to update the canvas_paintings table wi…

…th asset details from the DLCS

src/IIIFPresentation/API/Helpers/PathGenerator.cs

@@ -1,5 +1,6 @@
 using API.Infrastructure.Requests;
 using DLCS;
+using DLCS.Models;
 using Microsoft.Extensions.Options;
 using Models.Database.Collections;
 using Models.Database.General;
@@ -92,12 +93,21 @@ public string GenerateCanvasId(CanvasPainting canvasPainting)
     public Uri? GenerateAssetUri(CanvasPainting canvasPainting)
     {
         if (string.IsNullOrEmpty(canvasPainting.AssetId)) return null;
-        var assetId = canvasPainting.AssetId.Split('/');
-        if (assetId.Length != 3) return null;
+
+        AssetId assetId;
+
+        try
+        {
+            assetId = AssetId.FromString(canvasPainting.AssetId);
+        }
+        catch // swallow error as it's not needed
+        {
+            return null;
+        }
 
         var uriBuilder = new UriBuilder(dlcsSettings.ApiUri)
         {
-            Path = $"/customers/{assetId[0]}/spaces/{assetId[1]}/images/{assetId[2]}",
+            Path = $"/customers/{assetId.Customer}/spaces/{assetId.Space}/images/{assetId.Asset}",
         };
         return uriBuilder.Uri;
     }

src/IIIFPresentation/AWS/Settings/AWSSettings.cs

@@ -2,6 +2,8 @@
 
 public class AWSSettings
 {
+    public const string SettingsName = "AWS";
+
     /// <summary>
     /// If true, service will use LocalStack and custom ServiceUrl
     /// </summary>
@@ -16,4 +18,4 @@ public class AWSSettings
     /// SQS Settings
     /// </summary>
     public SQSSettings SQS { get; set; } = new();
-}
+}

src/IIIFPresentation/AWS/Settings/SQSSettings.cs

@@ -7,6 +7,11 @@ public class SQSSettings
     /// </summary>
     public string? CustomerCreatedQueueName { get; set; }
 
+    /// <summary>
+    /// Name of queue that will receive notifications when a batch is completed
+    /// </summary>
+    public string? BatchCompletionQueueName { get; set; }
+
     /// <summary>
     /// The duration (in seconds) for which the call waits for a message to arrive in the queue before returning
     /// </summary>
@@ -21,4 +26,4 @@ public class SQSSettings
     /// Service root for SQS. Ignored if not running LocalStack
     /// </summary>
     public string ServiceUrl { get; set; } = "http://localhost:4566/";
-}
+}

src/IIIFPresentation/BackgroundHandler.Tests/BatchCompletion/BatchCompletionMessageHandlerTests.cs

@@ -0,0 +1,6 @@
+namespace BackgroundHandler.Tests.BatchCompletion;
+
+public class BatchCompletionMessageHandlerTests
+{
+
+}

src/IIIFPresentation/BackgroundHandler/BackgroundHandler.csproj

@@ -14,6 +14,7 @@
 
     <ItemGroup>
       <ProjectReference Include="..\AWS\AWS.csproj" />
+      <ProjectReference Include="..\DLCS\DLCS.csproj" />
       <ProjectReference Include="..\Repository\Repository.csproj" />
     </ItemGroup>
 

src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessage.cs

@@ -0,0 +1,11 @@
+namespace BackgroundHandler.BatchCompletion;
+
+public record BatchCompletionMessage(
+    int Id,
+    int CustomerId,
+    int Total,
+    int Success,
+    int Errors,
+    bool Superseded,
+    DateTime Started,
+    DateTime Finished);

src/IIIFPresentation/BackgroundHandler/BatchCompletion/BatchCompletionMessageHandler.cs

@@ -0,0 +1,122 @@
+using System.Text.Json;
+using AWS.SQS;
+using BackgroundHandler.Helpers;
+using Core.Helpers;
+using DLCS;
+using DLCS.API;
+using DLCS.Models;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+using Models.Database.General;
+using Repository;
+using Batch = Models.Database.General.Batch;
+
+namespace BackgroundHandler.BatchCompletion;
+
+public class BatchCompletionMessageHandler(
+    PresentationContext dbContext,
+    IDlcsApiClient dlcsApiClient,
+    IOptions<DlcsSettings> dlcsOptions,
+    ILogger<BatchCompletionMessageHandler> logger)
+    : IMessageHandler
+{
+    private static readonly JsonSerializerOptions JsonSerializerOptions = new(JsonSerializerDefaults.Web);
+
+    private readonly DlcsSettings dlcsSettings = dlcsOptions.Value;
+
+    public async Task<bool> HandleMessage(QueueMessage message, CancellationToken cancellationToken)
+    {
+        using (LogContextHelpers.SetServiceName(nameof(BatchCompletionMessageHandler)))
+        {
+            try
+            {
+                var batchCompletionMessage = DeserializeMessage(message);
+
+                await UpdateAssetsIfRequired(batchCompletionMessage, cancellationToken);
+                return true;
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error handling batch-completion message {MessageId}", message.MessageId);
+            }
+        }
+
+        return false;
+    }
+
+    private async Task UpdateAssetsIfRequired(BatchCompletionMessage batchCompletionMessage, CancellationToken cancellationToken)
+    {
+        var batch = await dbContext.Batches.Include(b => b.Manifest)
+            .ThenInclude(m => m.CanvasPaintings)
+            .FirstOrDefaultAsync(b => b.Id == batchCompletionMessage.Id, cancellationToken);
+
+        // batch isn't tracked by presentation, so nothing to do
+        if (batch == null) return;
+
+        // Other batches haven't completed, so no point populating items until all are complete
+        if (await dbContext.Batches.AnyAsync(b => b.ManifestId == batch.ManifestId && b.Status != BatchStatus.Completed,
+                cancellationToken))
+        {
+            return;
+        }
+
+        logger.LogInformation(
+            "Attempting to complete assets in batch {BatchId} for customer {CustomerId} with the manifest {ManifestId}",
+            batch.Id, batch.CustomerId, batch.ManifestId);
+
+        var assets = await RetrieveImages(batch, cancellationToken);
+
+        UpdateCanvasPaintings(assets, batch);
+        CompleteBatch(batch);
+
+        await dbContext.SaveChangesAsync(cancellationToken);
+
+        logger.LogTrace("updating batch {BatchId} has been completed", batch.Id);
+    }
+
+    private void CompleteBatch(Batch batch)
+    {
+        batch.Finished = DateTime.UtcNow; //todo: change to actual batch completion time?
+        batch.Status = BatchStatus.Completed;
+    }
+
+    private void UpdateCanvasPaintings(HydraCollection<Asset> assets, Batch batch)
+    {
+        if (batch.Manifest?.CanvasPaintings == null) return;
+
+        foreach (var canvasPainting in batch.Manifest.CanvasPaintings)
+        {
+            if (canvasPainting.CanvasOriginalId == null) // Trying to figure out an asset that hasn't been updated 
+            {
+                var assetId = AssetId.FromString(canvasPainting.AssetId!);
+
+                var asset = assets.Members.FirstOrDefault(a => a.ResourceId!.Contains($"{assetId.Space}/images/{assetId.Asset}"));
+                if (asset == null || asset.Ingesting) continue;
+
+                canvasPainting.CanvasOriginalId =
+                    new Uri(
+                        $"{dlcsSettings.OrchestratorUri}/iiif-img/{assetId.Customer}/{assetId.Space}/{assetId.Asset}/full/max/0/default.jpg"); //todo: do we need this? Supposed to be null for an asset really
+                canvasPainting.Thumbnail =
+                    new Uri(
+                        $"{dlcsSettings.OrchestratorUri}/thumbs/{assetId.Customer}/{assetId.Space}/{assetId.Asset}/100,/max/0/default.jpg"); //todo: move this to class
+                canvasPainting.Modified = DateTime.UtcNow;
+                canvasPainting.StaticHeight = asset.Height;
+                canvasPainting.StaticWidth = asset.Width;
+            }
+        }
+    }
+
+    private async Task<HydraCollection<Asset>> RetrieveImages(Batch batch, CancellationToken cancellationToken)
+    {
+        var assetsRequest =
+            batch.Manifest?.CanvasPaintings?.Where(c => c.AssetId != null).Select(c => c.AssetId!).ToList() ?? [];
+
+        return await dlcsApiClient.RetrieveAllImages(batch.CustomerId, assetsRequest, cancellationToken);
+    }
+
+    private static BatchCompletionMessage DeserializeMessage(QueueMessage message)
+    {
+        var deserialized = JsonSerializer.Deserialize<BatchCompletionMessage>(message.Body, JsonSerializerOptions);
+        return deserialized.ThrowIfNull(nameof(deserialized));
+    }
+}

src/IIIFPresentation/BackgroundHandler/Infrastructure/ServiceCollectionX.cs

@@ -1,6 +1,7 @@
 using AWS.Configuration;
 using AWS.Settings;
 using AWS.SQS;
+using BackgroundHandler.BatchCompletion;
 using BackgroundHandler.CustomerCreation;
 using BackgroundHandler.Listener;
 using Repository;
@@ -33,6 +34,14 @@ public static IServiceCollection AddBackgroundServices(this IServiceCollection s
                     ActivatorUtilities.CreateInstance<CreateBackgroundListenerService<CustomerCreatedMessageHandler>>(sp, aws.SQS.CustomerCreatedQueueName))
                 .AddScoped<CustomerCreatedMessageHandler>();
         }
+
+        if (!string.IsNullOrEmpty(aws.SQS.BatchCompletionQueueName))
+        {
+            services
+                .AddHostedService(sp => 
+                    ActivatorUtilities.CreateInstance<CreateBackgroundListenerService<BatchCompletionMessageHandler>>(sp, aws.SQS.BatchCompletionQueueName))
+                .AddScoped<BatchCompletionMessageHandler>();
+        }
 
         return services;
     }
@@ -45,4 +54,4 @@ public static IServiceCollection AddDataAccess(this IServiceCollection services,
         return services
             .AddPresentationContext(configuration);
     }
-}
+}

src/IIIFPresentation/BackgroundHandler/Program.cs

@@ -1,6 +1,7 @@
 using AWS.Settings;
 using BackgroundHandler.Infrastructure;
 using BackgroundHandler.Settings;
+using DLCS;
 using Serilog;
 
 var builder = WebApplication.CreateBuilder(args);
@@ -15,12 +16,17 @@
 
 builder.Services.AddOptions<BackgroundHandlerSettings>()
     .BindConfiguration(string.Empty);
-var aws = builder.Configuration.GetSection("AWS").Get<AWSSettings>() ?? new AWSSettings();
+var aws = builder.Configuration.GetSection(AWSSettings.SettingsName).Get<AWSSettings>() ?? new AWSSettings();
+var dlcsSettings = builder.Configuration.GetSection(DlcsSettings.SettingsName);
+var dlcs = dlcsSettings.Get<DlcsSettings>()!;
 
-builder.Services.AddAws(builder.Configuration, builder.Environment);
-builder.Services.AddDataAccess(builder.Configuration);
-builder.Services.AddBackgroundServices(aws);
+builder.Services.AddAws(builder.Configuration, builder.Environment)
+    .AddDataAccess(builder.Configuration)
+    .AddHttpContextAccessor()
+    .AddDlcsClientWithLocalAuth(dlcs)
+    .AddBackgroundServices(aws)
+    .Configure<DlcsSettings>(dlcsSettings);
 
 var app = builder.Build();
 
-app.Run();
+app.Run();

src/IIIFPresentation/DLCS/API/DlcsApiClient.cs

@@ -6,6 +6,7 @@
 using DLCS.Models;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Newtonsoft.Json.Linq;
 
 namespace DLCS.API;
 
@@ -26,6 +27,12 @@ public interface IDlcsApiClient
     /// </summary>
     public Task<List<Batch>> IngestAssets<T>(int customerId, List<T> images,
         CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Retrieve a list of assets from the DLCS
+    /// </summary>
+    public Task<HydraCollection<Asset>> RetrieveAllImages(int customerId, List<string> assets,
+        CancellationToken cancellationToken = default);
 }
 
 /// <summary>
@@ -85,6 +92,20 @@ public async Task<List<Batch>> IngestAssets<T>(int customerId, List<T> assets, C
         return batches.ToList();
     }
 
+    public async Task<HydraCollection<Asset>> RetrieveAllImages(int customerId, List<string> assets,
+        CancellationToken cancellationToken = default)
+    {
+        logger.LogTrace("performing an all images call for customer {CustomerId}", customerId);
+        var allImagesPath = $"/customers/{customerId}/allImages";
+
+        var allImagesRequest = new AllImages(assets);
+
+        var asset =
+            await CallDlcsApi<HydraCollection<Asset>>(HttpMethod.Post, allImagesPath, allImagesRequest, cancellationToken);
+
+        return asset ?? throw new DlcsException("Failed to retrieve all images");
+    }
+
     private async Task<T?> CallDlcsApi<T>(HttpMethod httpMethod, string path, object payload,
         CancellationToken cancellationToken)
     {

src/IIIFPresentation/DLCS/DlcsSettings.cs

@@ -8,6 +8,11 @@ public class DlcsSettings
     /// URL root of DLCS API 
     /// </summary>
     public required Uri ApiUri { get; set; }
+
+    /// <summary>
+    /// URL root of DLCS API 
+    /// </summary>
+    public Uri? OrchestratorUri { get; set; }
 
     /// <summary>
     /// Default timeout (in ms) use for HttpClient.Timeout.
@@ -18,4 +23,9 @@ public class DlcsSettings
     /// The maximum size of an individual batch request
     /// </summary>
     public int MaxBatchSize { get; set; } = 100;
+
+    /// <summary>
+    /// Used to authenticate requests that do not go via the HttpContextAccessor
+    /// </summary>
+    public string ApiLocalAuth { get; set; } //Todo: is this right? is there a better way of setting auth
 }

src/IIIFPresentation/DLCS/Handlers/AmbientAuthHandler.cs

@@ -25,4 +25,4 @@ protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage reques
         request.Headers.Authorization = authHeader;
         return base.SendAsync(request, cancellationToken);
     }
-}
+}

src/IIIFPresentation/DLCS/Handlers/AmbientAuthLocalHandler.cs

@@ -0,0 +1,21 @@
+using System.Net.Http.Headers;
+using Microsoft.Extensions.Options;
+
+namespace DLCS.Handlers;
+
+public class AmbientAuthLocalHandler() : DelegatingHandler
+{
+    private readonly DlcsSettings dlcsSettings;
+
+    public AmbientAuthLocalHandler(IOptions<DlcsSettings> dlcsOptions) : this()
+    {
+        dlcsSettings = dlcsOptions.Value;
+    }
+
+    protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request,
+        CancellationToken cancellationToken)
+    {
+        request.Headers.Authorization = new AuthenticationHeaderValue("Basic", dlcsSettings.ApiLocalAuth);
+        return base.SendAsync(request, cancellationToken);
+    }
+}

src/IIIFPresentation/DLCS/Models/AllImages.cs

@@ -0,0 +1,23 @@
+using System.Text.Json.Serialization;
+
+namespace DLCS.Models;
+
+public class AllImages : JsonLdBase
+{
+    public AllImages(List<string> members)
+    {
+        Members = members.Select(m => new AllImagesMember(m)).ToList();
+        Type = "Collection" ;
+    }
+
+    [JsonPropertyOrder(3)]
+    [JsonPropertyName("member")]
+    public List<AllImagesMember> Members { get; set; }
+}
+
+public class AllImagesMember(string id)
+{
+    public string Id { get; set; } = id;
+}
+
+