feat: Add concurrent integration test & utils (#1596)

* Add concurrent tests

* Add missing 100x to a test's name

* Remove 100MiB file and generate file instead.

* See if using UUID for filename fixes this possibly-concurrency-issue.

* Try temporaryDirectory property.

* Fix concurrency issue

* Comment fixes & remove STS getCallerIdentity concurrent calls.

* Remove unused import

* Remove concurrent test for event stream output

* Refactor tests a bit

* Use FileStream to write to file

* Swap out filemanager with writing emtpy data approach

* Tweak a bit

* Simplify; no need to safe generated dummy data to a file, just use 50MB memory.

* Change from 10x50MB to 100x5MB

* Reduce payload from 5MB to 3MB

* Reduce payload from 3MB to 1MB.

* Add test for 1.5 MB to test aws chunked & flexible checksum flow also.

---------

Co-authored-by: Sichan Yoo <[email protected]>
Co-authored-by: Josh Elkins <[email protected]>

IntegrationTests/AWSIntegrationTestUtils/ConcurrentTestHelper.swift

@@ -0,0 +1,49 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/// Runs a test concurrently for the number of times specified
+///
+/// The test is run repeatedly using a Swift task group; each test run is performed as a "child task".
+/// The function returns when all test runs have completed, and rethrows if a test run throws.
+/// - Parameters:
+///   - count: The number of test runs
+///   - test: The function pointer for the test to run
+/// - Throws: Any error thrown by one of the test runs.
+public func repeatConcurrently(count: Int, test: @escaping () async throws -> Void) async throws {
+    try await withThrowingTaskGroup(of: Void.self) { taskGroup in
+        for _ in 0..<count {
+            taskGroup.addTask {
+                try await test()
+            }
+        }
+        try await taskGroup.waitForAll()
+    }
+}
+
+/// Runs a test concurrently for the number of times specified with args
+///
+/// The test is run repeatedly using a Swift task group; each test run is performed as a "child task".
+/// The function returns when all test runs have completed, and rethrows if a test run throws.
+/// - Parameters:
+///   - count: The number of test runs
+///   - test: The function pointer for the test to run
+///   - args: Any values to pass along to test function
+/// - Throws: Any error thrown by one of the test runs.
+public func repeatConcurrentlyWithArgs(
+    count: Int,
+    test: @escaping (Any...) async throws -> Void,
+    args: Any...
+) async throws {
+    try await withThrowingTaskGroup(of: Void.self) { taskGroup in
+        for _ in 0..<count {
+            taskGroup.addTask {
+                try await test(args)
+            }
+        }
+        try await taskGroup.waitForAll()
+    }
+}

IntegrationTests/Package.swift

@@ -13,6 +13,9 @@ import PackageDescription
 // MARK: - Target dependencies
 
 extension Target.Dependency {
+    // Test utility module
+    static var awsIntegrationTestUtils: Self { "AWSIntegrationTestUtils" }
+
     // AWS modules
     static var awsClientRuntime: Self { .product(name: "AWSClientRuntime", package: "aws-sdk-swift") }
     static var awsSDKCommon: Self { .product(name: "AWSSDKCommon", package: "aws-sdk-swift") }
@@ -36,6 +39,12 @@ let package = Package(
         .iOS(.v13),
         .tvOS(.v13),
         .watchOS(.v6)
+    ],
+    targets: [
+        .target(
+            name: "AWSIntegrationTestUtils",
+            path: "./AWSIntegrationTestUtils"
+        )
     ]
 )
 
@@ -123,6 +132,7 @@ func addIntegrationTestTarget(_ name: String) {
                 .awsSDKIdentity,
                 .smithyIdentity,
                 .awsSDKCommon,
+                .awsIntegrationTestUtils,
                 .product(name: name, package: "aws-sdk-swift")
             ] + additionalDependencies.map {
                 Target.Dependency.product(name: $0, package: "aws-sdk-swift", condition: nil)

IntegrationTests/Services/AWSS3IntegrationTests/S3ConcurrentTests.swift

@@ -0,0 +1,57 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import Foundation
+import Smithy
+import SmithyStreams
+import XCTest
+import AWSS3
+import AWSIntegrationTestUtils
+
+class S3ConcurrentTests: S3XCTestCase {
+    public var fileData: Data!
+    let MEGABYTE: Double = 1_000_000
+
+    // Payload below 1,048,576 bytes; sends as simple data payload
+    func test_100x_1MB_getObject() async throws {
+        fileData = try generateDummyTextData(numMegabytes: MEGABYTE)
+        try await repeatConcurrentlyWithArgs(count: 100, test: getObject, args: fileData!)
+    }
+
+    // Payload over 1,048,576 bytes; uses aws chunked encoding & flexible checksum
+    func test_100x_1_5MB_getObject() async throws {
+        fileData = try generateDummyTextData(numMegabytes: MEGABYTE * 1.5)
+        try await repeatConcurrentlyWithArgs(count: 100, test: getObject, args: fileData!)
+    }
+
+    /* Helper functions */
+
+    // Generates text data in increments of 10 bytes
+    func generateDummyTextData(numMegabytes: Double) throws -> Data {
+        let segmentData = Data("1234567890".utf8)
+        var wholeData = Data()
+        for _ in 0..<(Int(numMegabytes)/10) {
+            wholeData.append(segmentData)
+        }
+        return wholeData
+    }
+
+    // Puts data to S3, gets the uploaded file, then asserts retrieved data equals original data
+    func getObject(args: Any...) async throws {
+        guard let data = args[0] as? Data else {
+            throw ClientError.dataNotFound("Failed to retrieve dummy data.")
+        }
+        let file = ByteStream.data(data)
+        let objectKey = UUID().uuidString.split(separator: "-").first!.lowercased()
+        let putObjectInput = PutObjectInput(body: file, bucket: bucketName, key: objectKey)
+        _ = try await client.putObject(input: putObjectInput)
+        let retrievedData = try await client.getObject(input: GetObjectInput(
+            bucket: bucketName, key: objectKey
+        )).body?.readData()
+        XCTAssertEqual(data, retrievedData)
+    }
+}