From 9d25922328cc3e4f097f323e7e920d993f89d581 Mon Sep 17 00:00:00 2001
From: Simon Clematide <simon.clematide@uzh.ch>
Date: Tue, 12 Mar 2024 12:20:05 +0100
Subject: [PATCH] Add dotenv functionality  - reading SE_ACCESS_KEY and
 SE_SECRET_KEY from local .env file

---
 README.md                    | 3 +++
 impresso_commons/utils/s3.py | 8 ++++++++
 pyproject.toml               | 3 ++-
 requirements.txt             | 1 +
 4 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 38eedd5..f419e8c 100644
--- a/README.md
+++ b/README.md
@@ -16,6 +16,9 @@ With `pip`:
 pip install impresso-commons
 ```
 
+## Notes
+The library supports configuration of s3 credentials via project-specific local .env files. 
+
 ## License
 
 The second project 'impresso - Media Monitoring of the Past II. Beyond Borders: Connecting Historical Newspapers and Radio' is funded by the Swiss National Science Foundation (SNSF) under grant number [CRSII5_213585](https://data.snf.ch/grants/grant/213585) and the Luxembourg National Research Fund under grant No. 17498891.
diff --git a/impresso_commons/utils/s3.py b/impresso_commons/utils/s3.py
index c2796f1..8a95742 100644
--- a/impresso_commons/utils/s3.py
+++ b/impresso_commons/utils/s3.py
@@ -12,6 +12,7 @@
 import bz2
 from smart_open.s3 import iter_bucket
 from smart_open import open as s_open
+from dotenv import load_dotenv
 
 from impresso_commons.utils import _get_cores
 
@@ -20,6 +21,8 @@
 
 
 def get_storage_options():
+    # load environment variables from local .env files
+    load_dotenv()
     return {
         "client_kwargs": {"endpoint_url": "https://os.zhdk.cloud.switch.ch"},
         "key": os.environ["SE_ACCESS_KEY"],
@@ -42,6 +45,9 @@ def get_storage_options():
 
 
 def get_s3_client(host_url="https://os.zhdk.cloud.switch.ch/"):
+
+    # load environment variables from local .env files
+    load_dotenv()
     if host_url is None:
         try:
             host_url = os.environ["SE_HOST_URL"]
@@ -77,6 +83,8 @@ def get_s3_resource(host_url="https://os.zhdk.cloud.switch.ch/"):
     :rtype: `boto3.resources.factory.s3.ServiceResource`
     """
 
+    # load environment variables from local .env files
+    load_dotenv()
     if host_url is None:
         try:
             host_url = os.environ["SE_HOST_URL"]
diff --git a/pyproject.toml b/pyproject.toml
index fe92be1..7bda584 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -38,7 +38,8 @@ dependencies = [
     "numpy",
     "smart_open",
     "jsonlines",
-    "s3fs"
+    "s3fs",
+    "python-dotenv"
 ]
 
 [project.urls]
diff --git a/requirements.txt b/requirements.txt
index 26a4b8d..c77e4fc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -148,6 +148,7 @@ pyrsistent>=0.19.3
 pytest>=7.4.2
 python-box>=7.1.1
 python-dateutil>=2.8.2
+python-dotenv>=1.0.1
 python-json-logger>=2.0.7
 python-jsonpath>=0.7.1
 python-jsonschema-objects>=0.5.0