project housekeeping (#51)

* Revert "fixed versioning and packaging, which resulted in an uninstallable/empty pandahub package"

This reverts commit 1985879.

* re-apply packaging fix after revert

* fix .env defaults

* fix docker (compose) setup

* set default setting on empty env var

* move test directory

* persist data for mongodb service

* remove outdated files

* revert e84d1aa and related commits (_id handling on project activation)

.env.example

@@ -1,10 +1,10 @@
-MONGODB_URL="mongodb://localhost:27017"
-MONGODB_USER=None
-MONGODB_PASSWORD=None
+MONGODB_URL=""
+MONGODB_USER=""
+MONGODB_PASSWORD=""
 
-MONGODB_GLOBAL_DATABASE_URL=None
-MONGODB_GLOBAL_DATABASE_USER=None
-MONGODB_GLOBAL_DATABASE_PASSWORD=None
+MONGODB_GLOBAL_DATABASE_URL=""
+MONGODB_GLOBAL_DATABASE_USER=""
+MONGODB_GLOBAL_DATABASE_PASSWORD=""
 
 EMAIL_VERIFICATION_REQUIRED=False
 

Dockerfile

@@ -1,19 +1,18 @@
-FROM python:3.10 as dev
+# dev stage
+FROM python:3.11 AS dev
+ENV PYTHONUNBUFFERED=1
+ENV PATH="/opt/venv/bin:$PATH"
+WORKDIR /src
+COPY requirements.txt .
+RUN python -m venv /opt/venv && \
+    python -m pip install -r requirements.txt
+CMD ["python",  "-m", "pandahub.api.main"]
 
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONPATH /code/pandahub
-
-ENV DEBUG False
-ENV PANDAHUB_SERVER_URL 0.0.0.0
-ENV PANDAHUB_SERVER_PORT 8002
-ENV WORKERS 2
-
-COPY ./ /code/pandahub/
-WORKDIR /code/pandahub
-
-RUN python -m pip install --upgrade pip
-RUN python -m pip install .["all"]
-RUN python -m pip install watchdog pyyaml
-
-# CMD uvicorn --host "0.0.0.0" --port "8002" "pandahub.api.main:app" --reload
-ENTRYPOINT ["python", "pandahub/api/main.py"]
+# production stage
+FROM python:3.11-slim AS production-stage
+ENV PATH="/opt/venv/bin:$PATH"
+WORKDIR /src
+# copy files from build stage
+COPY --from=dev /opt/venv /opt/venv
+COPY . .
+CMD ["uvicorn", "--host", "127.0.0.1", "--port", "8080", "pandahub.api.main:app"]

README.md

@@ -6,48 +6,11 @@ pandahub is a data hub for pandapower and pandapipes networks based on MongoDB.
 pandapipes networks as well as timeseries in a MongoDB. pandahub allows you to access the database directly through the PandaHub class,
 but also provides a REST-API based on FastAPI. Access through the API is managed with a user management implementation based on FastAPI Users.
 
-## Setup a local pandahub api
+## Development
+`docker compose up -d` runs a mongodb container alongside a pandahub api instance with live reload available
+at http://localhost:8002. To connect to an existing database instead, set `MONGODB_URL` to the connection string through an environment variable / in you `.env` file.
 
-Steps to test the client/server structure locally:
+Swagger UI is available at http://localhost:8002/docs.
 
-1. Start a MongoDB on localhost:27017 (or another custom port)
-
-2. Start the uvicorn server that exposes the API by navigating to "pandahub/api" and running:
-
-   ```
-   # windows
-   set SECRET=secret & python main.py
-
-   # linux
-   SECRET=secret python main.py
-   ```
-
-   or if you don't run mongodb on the default port (27017)
-
-   ```
-   # windows
-   set MONGODB_URL=mongodb://localhost:[mongo-port] & set SECRET=secret & python main.py
-
-   # linux
-   MONGODB_URL=mongodb://localhost:[mongo-port] SECRET=secret python main.py
-   ```
-
-   The API should now run on http://localhost:8002
-
-   >**Note**
-   >A full documentation of all api endpoints can be seen at http://localhost:8002/docs
-
-   >**Note 2**
-   >You can avoid always setting the environment variables for SECRET and MONGODB_URL by creating an `.env` file in `pandahub/api/` with the following content:
-   >```
-   >SECRET=secret
-   >MONGODB_URL=mongodb://localhost:[mongo-port]
-   >```
-
-## Develop with Docker
-
-`docker compose up` starts a mongodb container alongside pandahub with live reload available at http://localhost:8002.
-
-If you want to connect to a running database, set the database url and specify only docker-compose.yml:
-
-    MONGODB_URL=mongodb://localhost:[mongo-port] docker compose -f docker-compose.yml up
+If you develop on the library and do not need the fastapi app, `docker compose up db -d` starts only the mongodb
+container.

docker-compose.yml

@@ -1,34 +1,39 @@
-version: '3'
-
 networks:
   test:
     external: false
 
 services:
   db:
-    user: ${CURRENT_UID}
     image: mongo:latest
-    container_name: mongodb
     restart: always
+    volumes:
+        - mongodb_pandahub:/data/db
     networks:
         - test
     ports:
-        - "27017:27017"
+        - "127.0.0.1:27017:27017"
 
-  pandahub_server:
-    user: ${CURRENT_UID}
-    container_name: pandahub
+  pandahub-dev:
+    image: dev/pandahub
     build:
         context: .
         dockerfile: Dockerfile
+        target: dev
+    env_file:
+        - .env
     environment:
         - SECRET=devonly!
         - MONGODB_URL=${MONGODB_URL:-mongodb://db:27017}
+        - PYTHONPYCACHEPREFIX=/pycache
     ports:
-        - "8002:8002"
-#    volumes:
-#        - ./:/code/pandahub
+        - "127.0.0.1:8002:8002"
+    volumes:
+        - ./:/src/
+        - pycache:/pycache
     networks:
         - test
     depends_on:
         - db
+volumes:
+  pycache:
+  mongodb_pandahub:

pandahub/api/internal/db.py

@@ -9,13 +9,12 @@
     BeanieBaseAccessToken,
 )
 
-from pandahub.api.internal.settings import REGISTRATION_ADMIN_APPROVAL
-from pandahub.api.internal import settings
+from pandahub.api.internal.settings import REGISTRATION_ADMIN_APPROVAL, MONGODB_URL, MONGODB_USER, MONGODB_PASSWORD
 from pydantic import Field
 
-mongo_client_args = {"host": settings.MONGODB_URL, "uuidRepresentation": "standard", "connect": False}
-if settings.MONGODB_USER:
-    mongo_client_args |= {"username": settings.MONGODB_USER, "password": settings.MONGODB_PASSWORD}
+mongo_client_args = {"host": MONGODB_URL, "uuidRepresentation": "standard", "connect": False}
+if MONGODB_USER:
+    mongo_client_args |= {"username": MONGODB_USER, "password": MONGODB_PASSWORD}
 
 client = motor.motor_asyncio.AsyncIOMotorClient(**mongo_client_args)
 client.get_io_loop = asyncio.get_event_loop

pandahub/api/internal/settings.py

@@ -1,8 +1,13 @@
 from dotenv import load_dotenv
 import os
 
+def get_os_env(key, default=None):
+    value = os.getenv(key, default)
+    value = default if value == "" else value
+    return value
+
 def settings_bool(var_name, default=None):
-    var = os.getenv(var_name)
+    var = get_os_env(var_name)
     if var is None:
         return default
     if isinstance(var, str) and var.lower() == "true":
@@ -12,7 +17,7 @@ def settings_bool(var_name, default=None):
     return False
 
 def get_secret(key, default=None):
-    secret = os.getenv(key, default)
+    secret = get_os_env(key, default)
     if secret and os.path.isfile(secret):
         with open(secret) as f:
             secret = f.read()
@@ -21,35 +26,35 @@ def get_secret(key, default=None):
 # load variables from .env to environment variables
 load_dotenv()
 
-MONGODB_URL = get_secret("MONGODB_URL") or "mongodb://localhost:27017"
-MONGODB_USER = get_secret("MONGODB_USER") or None
-MONGODB_PASSWORD = get_secret("MONGODB_PASSWORD") or None
+MONGODB_URL = get_secret("MONGODB_URL", "mongodb://localhost:27017")
+MONGODB_USER = get_secret("MONGODB_USER")
+MONGODB_PASSWORD = get_secret("MONGODB_PASSWORD")
 
-MONGODB_GLOBAL_DATABASE_URL = get_secret("MONGODB_GLOBAL_DATABASE_URL") or None
-MONGODB_GLOBAL_DATABASE_USER = get_secret("MONGODB_GLOBAL_DATABASE_USER") or None
-MONGODB_GLOBAL_DATABASE_PASSWORD = get_secret("MONGODB_GLOBAL_DATABASE_PASSWORD") or None
+MONGODB_GLOBAL_DATABASE_URL = get_secret("MONGODB_GLOBAL_DATABASE_URL")
+MONGODB_GLOBAL_DATABASE_USER = get_secret("MONGODB_GLOBAL_DATABASE_USER")
+MONGODB_GLOBAL_DATABASE_PASSWORD = get_secret("MONGODB_GLOBAL_DATABASE_PASSWORD")
 if not MONGODB_GLOBAL_DATABASE_URL:
-    MONGODB_GLOBAL_DATABASE_URL = os.getenv("MONGODB_URL_GLOBAL_DATABASE") or None
+    MONGODB_GLOBAL_DATABASE_URL = get_os_env("MONGODB_URL_GLOBAL_DATABASE")
 
 EMAIL_VERIFICATION_REQUIRED = settings_bool("EMAIL_VERIFICATION_REQUIRED", default=False)
 
-MAIL_USERNAME = os.getenv("MAIL_USERNAME") or "[email protected]"
-MAIL_PASSWORD = os.getenv("MAIL_PASSWORD") or ""
-MAIL_PORT = os.getenv("MAIL_PORT") or 587
-MAIL_SMTP_SERVER = os.getenv("MAIL_SMTP_SERVER") or ""
+MAIL_USERNAME = get_os_env("MAIL_USERNAME", "[email protected]")
+MAIL_PASSWORD = get_os_env("MAIL_PASSWORD", "")
+MAIL_PORT = get_os_env("MAIL_PORT", 587)
+MAIL_SMTP_SERVER = get_os_env("MAIL_SMTP_SERVER", "")
 MAIL_STARTTLS = settings_bool("MAIL_STARTTLS", default=True)
 MAIL_SSL_TLS = settings_bool("MAIL_SSL_TLS", default=False)
 
-PASSWORD_RESET_URL = os.getenv("PASSWORD_RESET_URL") or ""
-EMAIL_VERIFY_URL = os.getenv("EMAIL_VERIFY_URL") or ""
-SECRET = get_secret("SECRET") or None
+PASSWORD_RESET_URL = get_os_env("PASSWORD_RESET_URL", "")
+EMAIL_VERIFY_URL = get_os_env("EMAIL_VERIFY_URL", "")
+SECRET = get_secret("SECRET")
 
 REGISTRATION_ENABLED = settings_bool("REGISTRATION_ENABLED", default=True)
 REGISTRATION_ADMIN_APPROVAL = settings_bool("REGISTRATION_ADMIN_APPROVAL", default=False)
 
 CREATE_INDEXES_WITH_PROJECT = settings_bool("CREATE_INDEXES_WITH_PROJECT", default=True)
 
 DEBUG = settings_bool("DEBUG", default=False)
-PANDAHUB_SERVER_URL = os.getenv("PANDAHUB_SERVER_URL", "0.0.0.0")
-PANDAHUB_SERVER_PORT = int(os.getenv('PANDAHUB_SERVER_PORT', 8002))
-WORKERS = int(os.getenv('WORKER', 2))
+PANDAHUB_SERVER_URL = get_os_env("PANDAHUB_SERVER_URL", "0.0.0.0")
+PANDAHUB_SERVER_PORT = int(get_os_env('PANDAHUB_SERVER_PORT', 8002))
+WORKERS = int(get_os_env('WORKER', 2))

pandahub/api/main.py

@@ -61,7 +61,7 @@ async def ready():
 
 
 if __name__ == "__main__":
-    uvicorn.run("main:app",
+    uvicorn.run("pandahub.api.main:app",
                 host=settings.PANDAHUB_SERVER_URL,
                 port=settings.PANDAHUB_SERVER_PORT,
                 log_level="info",

pandahub/lib/PandaHub.py

@@ -234,14 +234,11 @@ def create_project(
             project_data["_id"] = project_id
         if self.user_id is not None:
             project_data["users"] = {self.user_id: "owner"}
-        id = self.mongo_client["user_management"]["projects"].insert_one(project_data).inserted_id
+        self.mongo_client["user_management"]["projects"].insert_one(project_data)
         if CREATE_INDEXES_WITH_PROJECT:
             self._create_mongodb_indexes(project_data["_id"])
         if activate:
-            if "_id" in project_data.keys():
-                id = project_data["_id"]
-
-            self.set_active_project_by_id(id)
+            self.set_active_project_by_id(project_data["_id"])
         return project_data
 
     def delete_project(self, i_know_this_action_is_final=False, project_id=None):

pyproject.toml

@@ -35,24 +35,6 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12"
 ]
-dependencies = [
-    "uvicorn>=0.24.0",
-    "fastapi-users[beanie]>=12.0",
-    "fastapi>=0.104.0",
-    "fastapi-mail>=1.4.1",
-    "pandapower~=2.14",
-    "pandapipes>=0.7.0",
-    "pymongo",
-    "pydantic",
-    "simplejson",
-    "requests<2.28.0",
-    "python-dotenv",
-    "pymongoarrow",
-    "blosc",
-    "motor<3.0.0",
-    "hypothesis-graphql<0.10.0",
-    "attrs>=22.2.0"
-]
 keywords = [
     "network", "analysis", "optimization", "automation", "grid", "energy", "engineering", "simulation",
 ]
@@ -66,12 +48,15 @@ Issues = "https://github.com/e2nIEE/pandahub/issues"
 Download = "https://pypi.org/project/pandahub/#files"
 Changelog = "https://github.com/e2nIEE/pandahub/blob/develop/CHANGELOG.md"
 
+[tool.setuptools.dynamic]
+dependencies = { file = ["requirements.txt"] }
+
 [project.optional-dependencies]
 docs = ["numpydoc", "sphinx", "sphinx_rtd_theme", "sphinxcontrib.bibtex", "sphinx-pyproject"]
 test = ["pytest", "pytest-xdist", "nbmake", "simbench", "line_profiler"]
 all = [
     "numpydoc", "sphinx", "sphinx_rtd_theme", "sphinxcontrib.bibtex", "sphinx-pyproject",
-    "pytest<8.0.0", "pytest-xdist", "nbmake", "simbench", "line_profiler",
+    "pytest", "pytest-xdist", "nbmake", "simbench", "line_profiler",
 ]
 
 [tool.setuptools.packages]

requirements.txt

@@ -0,0 +1,13 @@
+uvicorn>=0.24.0
+fastapi-users[beanie]>=12.0
+fastapi>=0.104.0
+fastapi-mail>=1.4.1
+pandapower~=2.14
+pandapipes>=0.7.0
+pymongo
+pydantic
+simplejson
+requests
+python-dotenv
+pymongoarrow
+blosc

pandahub/test/conftest.py → test/conftest.py

@@ -12,8 +12,7 @@ def ph():
         ph.set_active_project(project_name)
         ph.delete_project(i_know_this_action_is_final=True)
 
-    ph.create_project(name=project_name, activate=False)
-    ph.set_active_project(project_name)
+    ph.create_project(name=project_name)
 
     yield ph