Merge pull request #85 from impresso/feature_dotenv

Add dotenv functionality

README.md

@@ -16,6 +16,9 @@ With `pip`:
 pip install impresso-commons
 ```
 
+## Notes
+The library supports configuration of s3 credentials via project-specific local .env files. 
+
 ## License
 
 The second project 'impresso - Media Monitoring of the Past II. Beyond Borders: Connecting Historical Newspapers and Radio' is funded by the Swiss National Science Foundation (SNSF) under grant number [CRSII5_213585](https://data.snf.ch/grants/grant/213585) and the Luxembourg National Research Fund under grant No. 17498891.

impresso_commons/utils/s3.py

@@ -12,6 +12,7 @@
 import bz2
 from smart_open.s3 import iter_bucket
 from smart_open import open as s_open
+from dotenv import load_dotenv
 
 from impresso_commons.utils import _get_cores
 
@@ -20,6 +21,8 @@
 
 
 def get_storage_options():
+    # load environment variables from local .env files
+    load_dotenv()
     return {
         "client_kwargs": {"endpoint_url": "https://os.zhdk.cloud.switch.ch"},
         "key": os.environ["SE_ACCESS_KEY"],
@@ -42,6 +45,9 @@ def get_storage_options():
 
 
 def get_s3_client(host_url="https://os.zhdk.cloud.switch.ch/"):
+
+    # load environment variables from local .env files
+    load_dotenv()
     if host_url is None:
         try:
             host_url = os.environ["SE_HOST_URL"]
@@ -77,6 +83,8 @@ def get_s3_resource(host_url="https://os.zhdk.cloud.switch.ch/"):
     :rtype: `boto3.resources.factory.s3.ServiceResource`
     """
 
+    # load environment variables from local .env files
+    load_dotenv()
     if host_url is None:
         try:
             host_url = os.environ["SE_HOST_URL"]

pyproject.toml

@@ -38,7 +38,8 @@ dependencies = [
     "numpy",
     "smart_open",
     "jsonlines",
-    "s3fs"
+    "s3fs",
+    "python-dotenv"
 ]
 
 [project.urls]

requirements.txt

@@ -148,6 +148,7 @@ pyrsistent>=0.19.3
 pytest>=7.4.2
 python-box>=7.1.1
 python-dateutil>=2.8.2
+python-dotenv>=1.0.1
 python-json-logger>=2.0.7
 python-jsonpath>=0.7.1
 python-jsonschema-objects>=0.5.0