Merge pull request #19 from lilab-bcb/remove-boto3

Use s3fs for strato exists on AWS folders
lilab-bcb · Mar 21, 2023 · 524fa2e · 524fa2e
2 parents fa81922 + 2e4a1dc
commit 524fa2e
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 12 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
+.DS_Store
 __pycache__/
 *.egg-info/
 version.py

diff --git a/pyproject.toml b/pyproject.toml
@@ -31,10 +31,12 @@ classifiers = [
 ]
 
 requires-python = '>=3'
-dependencies = []
+dependencies = [
+    's3fs',
+    'importlib_metadata >= 0.7; python_version < "3.8"',
+]
 
 [project.optional-dependencies]
-aws = ['boto3']
 test = ['pytest']
 
 [tool.black]

diff --git a/requirements.txt b/requirements.txt
diff --git a/strato/backends/_aws.py b/strato/backends/_aws.py
@@ -108,17 +108,14 @@ def stat(self, filename, profile):
         is_folder = True if filename[-1] == "/" else False
 
         if is_folder:
-            import boto3
-
             fn_list = filename[5:].split("/")
             bucket = fn_list[0]
             folder = "/".join(fn_list[1:]) if len(fn_list) > 1 else ""
-            session = boto3.Session(profile_name=profile)
-            resp = session.client("s3").list_objects_v2(
-                Bucket=bucket,
-                Prefix=folder,
-            )
-            if resp["KeyCount"] == 0:
+
+            from s3fs import S3FileSystem
+
+            s3 = S3FileSystem(anon=False, profile=profile)
+            if len(s3.ls(f"s3://{bucket}/{folder}")) == 0:
                 raise CalledProcessError(
                     returncode=1,
                     cmd=["strato command"],