Merge branch 'master' into hr/constructor-s3path

CHANGELOG.md

@@ -9,9 +9,11 @@
 - v0.11.0: The `s3_exists` and `isdir(::S3Path)` calls no longer encounter HTTP 403 (Access Denied) errors when attempting to list resources which requiring an `s3:prefix` to be specified ([#289]).
 - v0.11.1: The new keyword argument `returns` for `Base.write(fp::S3Path, ...)` determines the output returned from `write`, which can now be the raw `AWS.Response` (`returns=:response`) or the `S3Path` (`returns=:path`); this latter option returns an `S3Path` populated with the version ID of the written object (when versioning is enabled on the bucket) ([#293]).
 - v0.11.2: `s3_copy` supports the `parse_response` keyword allowing for access to the unparsed AWS API response ([#300]).
-- v0.11.2: Add `S3Path` copy constructor for allowing updating `version`, `config`, and/or `isdirectory` ([#297]).
+- v0.11.2: Added `s3_nuke_object` function to delete all versions of an object ([#299]).
+- v0.11.2: Added `S3Path` copy constructor for allowing updating `version`, `config`, and/or `isdirectory` ([#297]).
 
 [#289]: https://github.com/JuliaCloud/AWSS3.jl/pull/289
 [#293]: https://github.com/JuliaCloud/AWSS3.jl/pull/293
 [#297]: https://github.com/JuliaCloud/AWSS3.jl/pull/297
+[#299]: https://github.com/JuliaCloud/AWSS3.jl/pull/299
 [#300]: https://github.com/JuliaCloud/AWSS3.jl/pull/300

src/AWSS3.jl

@@ -24,6 +24,7 @@ export S3Path,
     s3_list_objects,
     s3_list_keys,
     s3_list_versions,
+    s3_nuke_object,
     s3_get_meta,
     s3_directory_stat,
     s3_purge_versions,
@@ -410,6 +411,47 @@ end
 
 s3_delete(a...; b...) = s3_delete(global_aws_config(), a...; b...)
 
+"""
+    s3_nuke_object([::AbstractAWSConfig], bucket, path; kwargs...)
+
+Deletes all versions of object `path` from `bucket`. All provided `kwargs` are forwarded to
+[`s3_delete`](@ref). In the event an error occurs any object versions already deleted by
+`s3_nuke_object` will be lost.
+
+To only delete one specific version, use [`s3_delete`](@ref); to delete all versions
+EXCEPT the latest version, use [`s3_purge_versions`](@ref); to delete all versions
+in an entire bucket, use [`AWSS3.s3_nuke_bucket`](@ref).
+
+# API Calls
+
+- [`DeleteObject`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html)
+- [`ListObjectVersions`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html)
+
+# Permissions
+
+- [`s3:DeleteObject`](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazons3.html#amazons3-DeleteObject)
+- [`s3:ListBucketVersions`](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazons3.html#amazons3-ListBucketVersions)
+"""
+function s3_nuke_object(aws::AbstractAWSConfig, bucket, path; kwargs...)
+    # Because list_versions returns ALL keys with the given _prefix_, we need to
+    # restrict the results to ones with the _exact same_ key.
+    for object in s3_list_versions(aws, bucket, path)
+        object["Key"] == path || continue
+        version = object["VersionId"]
+        try
+            s3_delete(aws, bucket, path; version, kwargs...)
+        catch e
+            @warn "Failed to delete version $(version) of $(path)"
+            rethrow(e)
+        end
+    end
+    return nothing
+end
+
+function s3_nuke_object(bucket, path; kwargs...)
+    return s3_nuke_object(global_aws_config(), bucket, path; kwargs...)
+end
+
 """
     s3_copy([::AbstractAWSConfig], bucket, path; acl::AbstractString="",
             to_bucket=bucket, to_path=path, metadata::AbstractDict=SSDict(),

src/s3path.jl

@@ -468,7 +468,18 @@ function Base.rm(fp::S3Path; recursive=false, kwargs...)
     end
 
     @debug "delete: $fp"
-    return s3_delete(get_config(fp), fp.bucket, fp.key; fp.version)
+    return s3_delete(fp)
+end
+
+s3_delete(fp::S3Path) = s3_delete(get_config(fp), fp.bucket, fp.key; fp.version)
+
+"""
+    s3_nuke_object(fp::S3Path)
+
+Delete all versions of an object `fp`.
+"""
+function s3_nuke_object(fp::S3Path)
+    return s3_nuke_object(get_config(fp), fp.bucket, fp.key)
 end
 
 # We need to special case sync with S3Paths because of how directories

test/awss3.jl

@@ -371,6 +371,33 @@ function awss3_tests(base_config)
         @test s3_get(config, bucket_name, "key3") == b"data3.v3"
     end
 
+    is_aws(base_config) && @testset "Delete All Versions" begin
+        config = assume_testset_role("NukeObjectTestset"; base_config)
+        key_to_delete = "NukeObjectTestset_key"
+        # Test that object that starts with the same prefix as `key_to_delete` is
+        # not _also_ deleted
+        key_not_to_delete = "NukeObjectTestset_key/rad"
+
+        function _s3_object_versions(config, bucket, key)
+            return filter!(x -> x["Key"] == key, s3_list_versions(config, bucket, key))
+        end
+
+        s3_put(config, bucket_name, key_to_delete, "foo.v1")
+        s3_put(config, bucket_name, key_to_delete, "foo.v2")
+        s3_put(config, bucket_name, key_to_delete, "foo.v3")
+        s3_put(config, bucket_name, key_not_to_delete, "rad.v1")
+        s3_put(config, bucket_name, key_not_to_delete, "rad.v2")
+
+        @test length(_s3_object_versions(config, bucket_name, key_to_delete)) == 3
+        @test length(_s3_object_versions(config, bucket_name, key_not_to_delete)) == 2
+
+        s3_nuke_object(config, bucket_name, key_to_delete)
+        @test length(_s3_object_versions(config, bucket_name, key_to_delete)) == 0
+
+        # Test that _only_ specific path was deleted---not paths at the same prefix
+        @test length(_s3_object_versions(config, bucket_name, key_not_to_delete)) == 2
+    end
+
     if is_aws(base_config)
         @testset "Empty and Delete Bucket" begin
             config = assume_testset_role("EmptyAndDeleteBucketTestset"; base_config)

test/resources/awss3_jl_test.yaml

@@ -236,6 +236,38 @@ Resources:
               - s3:DeleteObjectVersion
             Resource: !Sub arn:aws:s3:::${BucketPrefix}*/*
 
+  NukeObjectTestsetRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub ${GitHubRepo}-NukeObjectTestset
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS: !GetAtt PublicCIRole.Arn
+            Action: sts:AssumeRole
+
+  NukeObjectTestsetPolicy:
+    Type: AWS::IAM::Policy
+    Properties:
+      PolicyName: !Sub ${GitHubRepo}-NukeObjectTestset
+      Roles:
+        - !Ref NukeObjectTestsetRole
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Action:
+              - s3:ListBucketVersions
+            Resource: !Sub arn:aws:s3:::${BucketPrefix}*
+          - Effect: Allow
+            Action:
+              - s3:GetObject
+              - s3:DeleteObjectVersion
+              - s3:PutObject
+            Resource: !Sub arn:aws:s3:::${BucketPrefix}*/*
+
   RestrictedPrefixTestsetRole:
     Type: AWS::IAM::Role
     Properties:

test/s3path.jl

@@ -731,6 +731,13 @@ function s3path_tests(base_config)
             rm(versioned_path)
             @test !exists(versioned_path)
             @test length(s3_list_versions(config, bucket_name, key)) == 1
+
+            fp = S3Path(bucket_name, "test_versions_nukeobject"; config)
+            foreach(_ -> write(fp, "foo"), 1:6)
+            @test length(s3_list_versions(fp.config, fp.bucket, fp.key)) == 6
+            s3_nuke_object(fp)
+            @test length(s3_list_versions(fp.config, fp.bucket, fp.key)) == 0
+            @test !exists(fp)
         end
 
         @testset "S3Path null version" begin