Add support for uploading non-seekable streams

cloudinary · Dec 25, 2023 · fc556bc · fc556bc
1 parent 7290a86
commit fc556bc
Show file tree

Hide file tree

Showing 4 changed files with 98 additions and 12 deletions.
diff --git a/CloudinaryDotNet.IntegrationTests/UploadApi/UploadMethodsTest.cs b/CloudinaryDotNet.IntegrationTests/UploadApi/UploadMethodsTest.cs
@@ -23,6 +23,7 @@ public class UploadMethodsTest : IntegrationTestBase
         private const string MODERATION_WEBPURIFY = "webpurify";
         private const string TEST_REMOTE_IMG = "http://cloudinary.com/images/old_logo.png";
         private const string TEST_REMOTE_VIDEO = "http://res.cloudinary.com/demo/video/upload/v1496743637/dog.mp4";
+        private const int TEST_CHUNK_SIZE = 5 * 1024 * 1024; // 5 MB
 
         private Transformation m_implicitTransformation;
 
@@ -546,6 +547,37 @@ public void TestUploadStream()
             }
         }
 
+        private class NonSeekableStream : MemoryStream
+        {
+            public NonSeekableStream(byte[] buffer) : base(buffer) { }
+
+            public override bool CanSeek => false;
+
+            public override long Seek(long offset, SeekOrigin loc) => throw new NotSupportedException();
+
+            public override long Length => throw new NotSupportedException();
+        }
+
+        [Test, RetryWithDelay]
+        public void TestUploadLargeNonSeekableStream()
+        {
+            byte[] bytes = File.ReadAllBytes(m_testLargeImagePath);
+            const string streamed = "stream_non_seekable";
+
+            using (var memoryStream = new NonSeekableStream(bytes))
+            {
+                var uploadParams = new ImageUploadParams()
+                {
+                    File = new FileDescription(streamed, memoryStream),
+                    Tags = $"{m_apiTag},{streamed}"
+                };
+
+                var result = m_cloudinary.UploadLarge(uploadParams, TEST_CHUNK_SIZE);
+
+                AssertUploadLarge(result, bytes.Length);
+            }
+        }
+
         [Test, RetryWithDelay]
         public void TestUploadLargeRawFiles()
         {
@@ -555,7 +587,7 @@ public void TestUploadLargeRawFiles()
 
             var uploadParams = GetUploadLargeRawParams(largeFilePath);
 
-            var result = m_cloudinary.UploadLarge(uploadParams, 5 * 1024 * 1024);
+            var result = m_cloudinary.UploadLarge(uploadParams, TEST_CHUNK_SIZE);
 
             AssertUploadLarge(result, largeFileLength);
         }
@@ -569,7 +601,7 @@ public async Task TestUploadLargeRawFilesAsync()
 
             var uploadParams = GetUploadLargeRawParams(largeFilePath);
 
-            var result = await m_cloudinary.UploadLargeAsync(uploadParams, 5 * 1024 * 1024);
+            var result = await m_cloudinary.UploadLargeAsync(uploadParams, TEST_CHUNK_SIZE);
 
             AssertUploadLarge(result, largeFileLength);
         }
@@ -599,7 +631,7 @@ public void TestUploadLarge()
             {
                 File = new FileDescription(largeFilePath),
                 Tags = m_apiTag
-            }, 5 * 1024 * 1024);
+            }, TEST_CHUNK_SIZE);
 
             Assert.AreEqual(fileLength, result.Bytes, result.Error?.Message);
         }
@@ -617,7 +649,7 @@ public async Task TestUploadLargeAutoFilesAsync()
                 Tags = m_apiTag
             };
 
-            var result = await m_cloudinary.UploadLargeAsync(uploadParams, 5 * 1024 * 1024);
+            var result = await m_cloudinary.UploadLargeAsync(uploadParams, TEST_CHUNK_SIZE);
 
             AssertUploadLarge(result, largeFileLength);
 
@@ -679,7 +711,7 @@ public void TestUploadLargeVideoFromWeb()
             {
                 File = new FileDescription(TEST_REMOTE_VIDEO),
                 Tags = m_apiTag
-            }, 5 * 1024 * 1024);
+            }, TEST_CHUNK_SIZE);
 
             Assert.AreEqual(result.StatusCode, HttpStatusCode.OK, result.Error?.Message);
             Assert.AreEqual(result.Format, FILE_FORMAT_MP4);

diff --git a/CloudinaryDotNet/ApiShared.Internal.cs b/CloudinaryDotNet/ApiShared.Internal.cs
@@ -342,6 +342,18 @@ private static bool ShouldPrepareContent(HttpMethod method, object parameters) =
         private static bool IsContentRange(Dictionary<string, string> extraHeaders) =>
             extraHeaders != null && extraHeaders.ContainsKey("Content-Range");
 
+        private static void UpdateContentRange(IDictionary<string, string> extraHeaders, FileDescription file)
+        {
+            if (!file.Eof || file.GetFileLength() > 0)
+            {
+                return; // no need to update the header, all good.
+            }
+
+            var startOffset = file.BytesSent - file.CurrChunkSize;
+
+            extraHeaders["Content-Range"] = $"bytes {startOffset}-{file.BytesSent - 1}/{file.BytesSent}";
+        }
+
         private static Stream GetFileStream(FileDescription file) =>
             file.Stream ?? File.OpenRead(file.FilePath);
 
@@ -385,7 +397,11 @@ private static StreamWriter SetStreamToStartAndCreateWriter(FileDescription file
             var memStream = new MemoryStream();
             var writer = new StreamWriter(memStream) { AutoFlush = true };
 
-            stream.Seek(file.BytesSent, SeekOrigin.Begin);
+            if (stream.CanSeek)
+            {
+                stream.Seek(file.BytesSent, SeekOrigin.Begin);
+            }
+
             return writer;
         }
 
@@ -446,6 +462,7 @@ private async Task<HttpContent> CreateMultipartContentAsync(
                             // Unfortunately we don't have ByteRangeStreamContent here,
                             // let's create another stream from the original one
                             stream = await GetRangeFromFileAsync(file, stream, cancellationToken).ConfigureAwait(false);
+                            UpdateContentRange(extraHeaders, file);
                         }
 
                         SetStreamContent(param.Key, file, stream, content);
@@ -492,6 +509,7 @@ private HttpContent CreateMultipartContent(
                             // Unfortunately we don't have ByteRangeStreamContent here,
                             // let's create another stream from the original one
                             stream = GetRangeFromFile(file, stream);
+                            UpdateContentRange(extraHeaders, file);
                         }
 
                         SetStreamContent(param.Key, file, stream, content);
@@ -594,14 +612,26 @@ private void PrepareRequestContent(
         private async Task<Stream> GetRangeFromFileAsync(FileDescription file, Stream stream, CancellationToken? cancellationToken = null)
         {
             var writer = SetStreamToStartAndCreateWriter(file, stream);
-            file.BytesSent += await ReadBytesAsync(writer, stream, file.BufferLength, cancellationToken).ConfigureAwait(false);
+            file.CurrChunkSize = await ReadBytesAsync(writer, stream, file.BufferLength, cancellationToken).ConfigureAwait(false);
+            file.BytesSent += file.CurrChunkSize;
+            if (file.CurrChunkSize < file.BufferLength)
+            {
+                file.Eof = true; // last chunk
+            }
+
             return WriterStreamFromBegin(writer);
         }
 
         private Stream GetRangeFromFile(FileDescription file, Stream stream)
         {
             var writer = SetStreamToStartAndCreateWriter(file, stream);
-            file.BytesSent += ReadBytes(writer, stream, file.BufferLength);
+            file.CurrChunkSize = ReadBytes(writer, stream, file.BufferLength);
+            file.BytesSent += file.CurrChunkSize;
+            if (file.CurrChunkSize < file.BufferLength)
+            {
+                file.Eof = true; // last chunk
+            }
+
             return WriterStreamFromBegin(writer);
         }
 

diff --git a/CloudinaryDotNet/Cloudinary.UploadApi.cs b/CloudinaryDotNet/Cloudinary.UploadApi.cs
@@ -969,7 +969,10 @@ private static void UpdateContentRange(UploadLargeParams internalParams)
             var fileDescription = internalParams.Parameters.File;
             var fileLength = fileDescription.GetFileLength();
             var startOffset = fileDescription.BytesSent;
-            var endOffset = startOffset + Math.Min(internalParams.BufferSize, fileLength - startOffset) - 1;
+            var buffSize = fileLength > 0
+                ? Math.Min(internalParams.BufferSize, fileLength - startOffset)
+                : internalParams.BufferSize;
+            var endOffset = startOffset + buffSize - 1;
 
             internalParams.Headers["Content-Range"] = $"bytes {startOffset}-{endOffset}/{fileLength}";
         }

diff --git a/CloudinaryDotNet/FileDescription.cs b/CloudinaryDotNet/FileDescription.cs
@@ -17,6 +17,13 @@ public class FileDescription
         /// </summary>
         internal long BytesSent;
 
+        /// <summary>
+        /// Current chunk size.
+        /// </summary>
+        internal long CurrChunkSize;
+
+        private bool isEof;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="FileDescription"/> class.
         /// Constructor to upload file from stream.
@@ -75,17 +82,31 @@ public FileDescription(string filePath)
         public bool IsRemote { get; }
 
         /// <summary>
-        /// Gets a value indicating whether the pointer is at the end of file.
+        /// Gets or sets a value indicating whether the pointer is at the end of file.
         /// </summary>
-        internal bool Eof => BytesSent == GetFileLength();
+        internal bool Eof
+        {
+            get => isEof ? isEof : GetFileLength() != -1 && BytesSent == GetFileLength();
+            set => isEof = value;
+        }
 
         /// <summary>
         /// Get file length.
         /// </summary>
         /// <returns>The length of file.</returns>
         internal long GetFileLength()
         {
-            return Stream?.Length ?? new FileInfo(FilePath).Length;
+            if (Stream == null)
+            {
+                return new FileInfo(FilePath).Length;
+            }
+
+            if (Stream?.CanSeek ?? false)
+            {
+                return Stream?.Length ?? -1;
+            }
+
+            return -1; // unknown length
         }
 
         /// <summary>