Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add concurrent integration test & utils #1596

Merged
merged 27 commits into from
Sep 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
268afa9
Add concurrent tests
Jun 27, 2024
13732b5
Add missing 100x to a test's name
Jun 27, 2024
fb34981
Merge branch 'main' into feat/concurrent-tests
Jul 5, 2024
25f7f1c
Merge branch 'main' into feat/concurrent-tests
Jul 8, 2024
a1a4e25
Remove 100MiB file and generate file instead.
Jul 8, 2024
648215c
See if using UUID for filename fixes this possibly-concurrency-issue.
Jul 8, 2024
7fa4672
Try temporaryDirectory property.
Jul 9, 2024
d4cf57b
Fix concurrency issue
Jul 9, 2024
390e7a4
Merge branch 'main' into feat/concurrent-tests
Jul 10, 2024
e667d1f
Merge branch 'main' into feat/concurrent-tests
Sep 18, 2024
8930eb4
Comment fixes & remove STS getCallerIdentity concurrent calls.
Sep 18, 2024
2850fdd
Remove unused import
Sep 18, 2024
ec40d7d
Remove concurrent test for event stream output
Sep 18, 2024
324caf0
Refactor tests a bit
Sep 18, 2024
3ed5c58
Use FileStream to write to file
Sep 18, 2024
8871d6b
Swap out filemanager with writing emtpy data approach
Sep 18, 2024
b0ebe87
Tweak a bit
Sep 18, 2024
bb49389
Merge branch 'main' into feat/concurrent-tests
Sep 18, 2024
2857855
Simplify; no need to safe generated dummy data to a file, just use 50…
Sep 18, 2024
3ee9a59
Merge branch 'main' into feat/concurrent-tests
Sep 19, 2024
55add5d
Change from 10x50MB to 100x5MB
Sep 19, 2024
8194103
Merge branch 'main' into feat/concurrent-tests
jbelkins Sep 20, 2024
b5f97f6
Merge branch 'main' into feat/concurrent-tests
Sep 20, 2024
19935c5
Merge branch 'feat/concurrent-tests' of github.com:awslabs/aws-sdk-sw…
Sep 20, 2024
fed7e45
Reduce payload from 5MB to 3MB
Sep 20, 2024
58e1ca2
Reduce payload from 3MB to 1MB.
Sep 20, 2024
a8a31f5
Add test for 1.5 MB to test aws chunked & flexible checksum flow also.
Sep 20, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
//
// Copyright Amazon.com Inc. or its affiliates.
// All Rights Reserved.
//
// SPDX-License-Identifier: Apache-2.0
//

/// Runs a test concurrently for the number of times specified
///
/// The test is run repeatedly using a Swift task group; each test run is performed as a "child task".
/// The function returns when all test runs have completed, and rethrows if a test run throws.
/// - Parameters:
/// - count: The number of test runs
/// - test: The function pointer for the test to run
/// - Throws: Any error thrown by one of the test runs.
public func repeatConcurrently(count: Int, test: @escaping () async throws -> Void) async throws {
try await withThrowingTaskGroup(of: Void.self) { taskGroup in
for _ in 0..<count {
taskGroup.addTask {
try await test()
}
}
try await taskGroup.waitForAll()
}
}

/// Runs a test concurrently for the number of times specified with args
///
/// The test is run repeatedly using a Swift task group; each test run is performed as a "child task".
/// The function returns when all test runs have completed, and rethrows if a test run throws.
/// - Parameters:
/// - count: The number of test runs
/// - test: The function pointer for the test to run
/// - args: Any values to pass along to test function
/// - Throws: Any error thrown by one of the test runs.
public func repeatConcurrentlyWithArgs(
jbelkins marked this conversation as resolved.
Show resolved Hide resolved
count: Int,
test: @escaping (Any...) async throws -> Void,
args: Any...
) async throws {
try await withThrowingTaskGroup(of: Void.self) { taskGroup in
for _ in 0..<count {
taskGroup.addTask {
try await test(args)
}
}
try await taskGroup.waitForAll()
}
}
10 changes: 10 additions & 0 deletions IntegrationTests/Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ import PackageDescription
// MARK: - Target dependencies

extension Target.Dependency {
// Test utility module
static var awsIntegrationTestUtils: Self { "AWSIntegrationTestUtils" }

// AWS modules
static var awsClientRuntime: Self { .product(name: "AWSClientRuntime", package: "aws-sdk-swift") }
static var awsSDKCommon: Self { .product(name: "AWSSDKCommon", package: "aws-sdk-swift") }
Expand All @@ -36,6 +39,12 @@ let package = Package(
.iOS(.v13),
.tvOS(.v13),
.watchOS(.v6)
],
targets: [
.target(
name: "AWSIntegrationTestUtils",
path: "./AWSIntegrationTestUtils"
)
]
)

Expand Down Expand Up @@ -123,6 +132,7 @@ func addIntegrationTestTarget(_ name: String) {
.awsSDKIdentity,
.smithyIdentity,
.awsSDKCommon,
.awsIntegrationTestUtils,
.product(name: name, package: "aws-sdk-swift")
] + additionalDependencies.map {
Target.Dependency.product(name: $0, package: "aws-sdk-swift", condition: nil)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
//
// Copyright Amazon.com Inc. or its affiliates.
// All Rights Reserved.
//
// SPDX-License-Identifier: Apache-2.0
//

import Foundation
import Smithy
import SmithyStreams
import XCTest
import AWSS3
import AWSIntegrationTestUtils

class S3ConcurrentTests: S3XCTestCase {
public var fileData: Data!
let MEGABYTE: Double = 1_000_000

// Payload below 1,048,576 bytes; sends as simple data payload
func test_100x_1MB_getObject() async throws {
fileData = try generateDummyTextData(numMegabytes: MEGABYTE)
try await repeatConcurrentlyWithArgs(count: 100, test: getObject, args: fileData!)
}

// Payload over 1,048,576 bytes; uses aws chunked encoding & flexible checksum
func test_100x_1_5MB_getObject() async throws {
fileData = try generateDummyTextData(numMegabytes: MEGABYTE * 1.5)
try await repeatConcurrentlyWithArgs(count: 100, test: getObject, args: fileData!)
}

/* Helper functions */

// Generates text data in increments of 10 bytes
func generateDummyTextData(numMegabytes: Double) throws -> Data {
let segmentData = Data("1234567890".utf8)
var wholeData = Data()
jbelkins marked this conversation as resolved.
Show resolved Hide resolved
for _ in 0..<(Int(numMegabytes)/10) {
wholeData.append(segmentData)
}
return wholeData
}

// Puts data to S3, gets the uploaded file, then asserts retrieved data equals original data
func getObject(args: Any...) async throws {
guard let data = args[0] as? Data else {
throw ClientError.dataNotFound("Failed to retrieve dummy data.")
}
let file = ByteStream.data(data)
let objectKey = UUID().uuidString.split(separator: "-").first!.lowercased()
let putObjectInput = PutObjectInput(body: file, bucket: bucketName, key: objectKey)
_ = try await client.putObject(input: putObjectInput)
let retrievedData = try await client.getObject(input: GetObjectInput(
bucket: bucketName, key: objectKey
)).body?.readData()
XCTAssertEqual(data, retrievedData)
}
}
Loading