Skip to content

Commit

Permalink
removing unused methods from BucketWriter
Browse files Browse the repository at this point in the history
  • Loading branch information
@JackLewis-digirati
JackLewis-digirati committed Sep 30, 2024
1 parent 04832b7 commit f23ac6e
Show file tree
Hide file tree
Showing 2 changed files with 0 additions and 204 deletions.
31 changes: 0 additions & 31 deletions src/IIIFPresentation/AWS/S3/IBucketWriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,43 +7,12 @@ namespace AWS.S3;
/// </summary>
public interface IBucketWriter
{
/// <summary>
/// Copy bucket object from source to destination
/// </summary>
Task CopyObject(ObjectInBucket source, ObjectInBucket destination);

/// <summary>
/// Copy a large file between buckets using multi part upload.
/// This should always be used for files >5GiB
/// </summary>
/// <param name="source">Bucket where object is currently stored.</param>
/// <param name="destination">Target bucket where object is to be stored.</param>
/// <param name="verifySize">Function to verify objectSize prior to copying. Not copied if false returned.</param>
/// <param name="contentType">ContentType to set on uploaded object</param>
/// <param name="token">Cancellation token</param>
/// <returns>ResultStatus signifying success or failure alongside ContentSize</returns>
/// <remarks>See https://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsUsingLLNetMPUapi.html </remarks>
public Task<LargeObjectCopyResult> CopyLargeObject(ObjectInBucket source, ObjectInBucket destination,
Func<long, Task<bool>>? verifySize = null, string? contentType = null,
CancellationToken token = default);

/// <summary>
/// Write content from provided string to S3
/// </summary>
/// <returns></returns>
Task WriteToBucket(ObjectInBucket dest, string content, string contentType,
CancellationToken cancellationToken = default);

/// <summary>
/// Write content from provided stream to S3
/// </summary>
Task<bool> WriteToBucket(ObjectInBucket dest, Stream content, string? contentType = null);

/// <summary>
/// Write file to S3
/// </summary>
Task<bool> WriteFileToBucket(ObjectInBucket dest, string filePath, string? contentType = null,
CancellationToken token = default);

/// <summary>
/// Delete specified objects underlying storage.
Expand Down
173 changes: 0 additions & 173 deletions src/IIIFPresentation/AWS/S3/S3BucketWriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,156 +19,6 @@ public S3BucketWriter(IAmazonS3 s3Client, ILogger<S3BucketWriter> logger)
this.logger = logger;
}

public async Task CopyObject(ObjectInBucket source, ObjectInBucket destination)
{
logger.LogDebug("Copying {Source} to {Destination}", source, destination);
try
{
var request = new CopyObjectRequest
{
SourceBucket = source.Bucket,
SourceKey = source.Key,
DestinationBucket = destination.Bucket,
DestinationKey = destination.Key
};
var response = await s3Client.CopyObjectAsync(request);
}
catch (AmazonS3Exception e)
{
logger.LogWarning(e, "Error encountered on server. Message:'{Message}' when writing an object",
e.Message);
}
catch (Exception e)
{
logger.LogWarning(e, "Unknown encountered on server. Message:'{Message}' when writing an object",
e.Message);
}
}

/// <summary>
/// Copy a large file between buckets using multi part upload.
/// This should always be used for files >5GiB
/// </summary>
/// <param name="source">Bucket where object is currently stored.</param>
/// <param name="destination">Target bucket where object is to be stored.</param>
/// <param name="verifySize">Function to verify objectSize prior to copying. Not copied if false returned.</param>
/// <param name="contentType">ContentType to set on uploaded object</param>
/// <param name="token">Cancellation token</param>
/// <returns>ResultStatus signifying success or failure alongside ContentSize</returns>
/// <remarks>See https://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsUsingLLNetMPUapi.html </remarks>
public async Task<LargeObjectCopyResult> CopyLargeObject(ObjectInBucket source, ObjectInBucket destination,
Func<long, Task<bool>>? verifySize = null, string? contentType = null,
CancellationToken token = default)
{
long? objectSize = null;
var partSize = 5 * (long)Math.Pow(2, 20); // 5 MB
var success = false;
var timer = Stopwatch.StartNew();

try
{
var sourceMetadata = await GetObjectMetadata(source, token);
if (sourceMetadata == null)
{
var notFoundResponse = new LargeObjectCopyResult(LargeObjectStatus.SourceNotFound);
var destinationMetadata = await GetObjectMetadata(destination, token);
notFoundResponse.DestinationExists = destinationMetadata != null;
return notFoundResponse;
}

objectSize = sourceMetadata.ContentLength;

if (verifySize != null)
{
if (!await verifySize.Invoke(objectSize.Value))
{
logger.LogInformation("Aborting multipart upload for {Target} as size verification failed",
destination);
return new LargeObjectCopyResult(LargeObjectStatus.FileTooLarge, objectSize);
}
}

var numberOfParts = Convert.ToInt32(objectSize / partSize);
var copyResponses = new List<CopyPartResponse>(numberOfParts);

var uploadId = await InitiateMultipartUpload(destination, contentType);

long bytePosition = 0;
for (int i = 1; bytePosition < objectSize; i++)
{
if (token.IsCancellationRequested)
{
logger.LogInformation("Cancellation requested, aborting multipart upload for {Target}",
destination);
await s3Client.AbortMultipartUploadAsync(destination.Bucket, destination.Key, uploadId, token);
return new LargeObjectCopyResult(LargeObjectStatus.Cancelled, objectSize);
}

var copyRequest = new CopyPartRequest
{
DestinationBucket = destination.Bucket,
DestinationKey = destination.Key,
SourceBucket = source.Bucket,
SourceKey = source.Key,
UploadId = uploadId,
FirstByte = bytePosition,
LastByte = bytePosition + partSize - 1 >= objectSize.Value
? objectSize.Value - 1
: bytePosition + partSize - 1,
PartNumber = i
};

copyResponses.Add(await s3Client.CopyPartAsync(copyRequest, token));
bytePosition += partSize;
}

// Complete the request
var completeRequest = new CompleteMultipartUploadRequest
{
Key = destination.Key,
BucketName = destination.Bucket,
UploadId = uploadId,
};
completeRequest.AddPartETags(copyResponses);
await s3Client.CompleteMultipartUploadAsync(completeRequest, token);
success = true;
return new LargeObjectCopyResult(LargeObjectStatus.Success, objectSize);
}
catch (OverflowException e)
{
logger.LogError(e,
"Error getting number of parts to copy. From '{Source}' to '{Destination}'. Size {Size}", source,
destination, objectSize);
}
catch (AmazonS3Exception e)
{
logger.LogError(e,
"S3 Error encountered copying bucket-bucket item. From '{Source}' to '{Destination}'",
source, destination);
}
catch (Exception e)
{
logger.LogError(e,
"Error during multipart bucket-bucket copy. From '{Source}' to '{Destination}'", source, destination);
}
finally
{
timer.Stop();
if (success)
{
logger.LogInformation("Copied large file to '{Target}' in {Elapsed}ms", destination,
timer.ElapsedMilliseconds);
}
else
{
logger.LogInformation("Failed to copy large file to '{Target}'. Failed after {Elapsed}ms", destination,
timer.ElapsedMilliseconds);
}
}

return new LargeObjectCopyResult(LargeObjectStatus.Error, objectSize);
}

public async Task WriteToBucket(ObjectInBucket dest, string content, string contentType,
CancellationToken cancellationToken = default)
{
Expand Down Expand Up @@ -326,27 +176,4 @@ public async Task DeleteFolder(ObjectInBucket root, bool deleteRoot)
return null;
}
}

private async Task<string> InitiateMultipartUpload(ObjectInBucket destination, string? contentType)
{
var request = new InitiateMultipartUploadRequest
{ BucketName = destination.Bucket, Key = destination.Key, ContentType = contentType };

var response = await s3Client.InitiateMultipartUploadAsync(request);
return response.UploadId;
}

private async Task<GetObjectMetadataResponse?> GetObjectMetadata(ObjectInBucket resource,
CancellationToken cancellationToken)
{
try
{
var request = resource.AsObjectMetadataRequest();
return await s3Client.GetObjectMetadataAsync(request, cancellationToken);
}
catch (AmazonS3Exception e) when (e.StatusCode == HttpStatusCode.NotFound)
{
return null;
}
}
}

0 comments on commit f23ac6e

Please sign in to comment.