Feat(extension): Enable Async Gunicorn workers for Flask and Django e…

docs/.wordlist.txt

@@ -3,6 +3,7 @@ AMD
 amd
 ARGS
 ASGI
+async
 Autotools
 autotools
 boolean
@@ -29,6 +30,7 @@ Dockerfiles
 DockerHub
 dotnet
 ECR
+eg
 emacs
 entrypoint
 entrypoints
@@ -43,10 +45,12 @@ filesystem
 filesystems
 fs
 gc
+gevent
 GiB
 GID
 github
 GPG
+gunicorn
 Gunicorn
 gzipped
 hardcoded
@@ -143,6 +147,7 @@ triaged
 ubuntu
 unbuilt
 UID
+uncomment
 usrmerge
 Uvicorn
 VENV

docs/reference/extensions/django-framework.rst

@@ -17,6 +17,15 @@ server metrics.
     The Django extension is compatible with the ``bare``, ``[email protected]``
     and ``[email protected]`` bases.
 
+Django framework supports both synchronous and asynchronous. If you want
+asynchronous you have to add ``django-framework/async-dependencies``
+read more :ref:`django-framework/dependencies <django-sync-deps>`.
+
+If you define
+``django-framework/async-dependencies`` you can not use
+``django-framework/dependencies``. ``rockcraft pack`` will error if you try to
+use both at the same time.
+
 Project requirements
 ====================
 
@@ -29,6 +38,8 @@ There are 2 requirements to be able to use the ``django-framework`` extension:
    ``./<Rock name with - replaced by _>/<Rock name with - replaced by _>/manage.py``
    relative to the ``rockcraft.yaml`` file.
 
+.. _django-sync-deps:
+
 ``parts`` > ``django-framework/dependencies:`` > ``stage-packages``
 ===================================================================
 
@@ -43,6 +54,43 @@ application. In the following example we use it to specify ``libpq-dev``:
         # list required packages or slices for your Django application below.
         - libpq-dev
 
+.. _django-async-deps:
+
+``parts`` > ``django-framework/async-dependencies``
+===================================================
+
+In order to be able to use async Gunicorn workers you need to use
+``django-framework/async-dependencies`` part instead of
+``django-framework/dependencies`` part.
+
+To use this just uncomment the following lines:
+
+.. code-block:: yaml
+
+  parts:
+    django-framework/async-dependencies:
+      python-packages:
+        - gunicorn[gevent]
+
+If your project needs additional debs to run, you can add them to
+``stage-packages`` just like it is done in :ref:`django-framework/dependencies <django-sync-deps>`:
+
+.. code-block:: yaml
+
+  parts:
+    django-framework/async-dependencies:
+      python-packages:
+        - gunicorn[gevent]
+      stage-packages:
+        # list required packages or slices for your Django application below.
+        - libpq-dev
+
+.. warning::
+  You can only use 1 of the dependencies parts at a time.
+  (eg. either ``django-framework/async-dependencies`` or
+  ``django-framework/dependencies``, to read more about synchronous dependencies
+  see :ref:`django-framework/dependencies <django-sync-deps>`)
+
 Useful links
 ============
 

docs/reference/extensions/flask-framework.rst

@@ -16,6 +16,13 @@ server metrics.
     The Flask extension is compatible with the ``bare``, ``[email protected]``
     and ``[email protected]`` bases.
 
+Flask framework supports both synchronous and asynchronous. If you want
+asynchronous you have to add ``flask-framework/async-dependencies``
+read more :ref:`flask-framework/dependencies <flask-sync-deps>`. If you define
+``flask-framework/async-dependencies`` you can not use
+``flask-framework/dependencies``. ``rockcraft pack`` will error if you try to
+use both at the same time.
+
 Project requirements
 ====================
 
@@ -27,6 +34,8 @@ There are 2 requirements to be able to use the ``flask-framework`` extension:
    there must be an ``app.py`` file at the root of the project with the name
    of the Flask object is set to ``app``
 
+.. _flask-sync-deps:
+
 ``parts`` > ``flask-framework/dependencies`` > ``stage-packages``
 =================================================================
 
@@ -41,6 +50,43 @@ application. In the following example we use it to specify ``libpq-dev``:
         # list required packages or slices for your flask app below.
         - libpq-dev
 
+.. _flask-async-deps:
+
+``parts`` > ``flask-framework/async-dependencies``
+=================================================================
+
+In order to be able to use async Gunicorn workers you need to use
+``flask-framework/async-dependencies`` part instead of
+``flask-framework/dependencies`` part.
+
+To use this just uncomment the following lines:
+
+.. code-block:: yaml
+
+  parts:
+    flask-framework/async-dependencies:
+      python-packages:
+        - gunicorn[gevent]
+
+If your project needs additional debs to run, you can add them to
+``stage-packages`` just like it is done in :ref:`flask-framework/dependencies <flask-sync-deps>`:
+
+.. code-block:: yaml
+
+  parts:
+    flask-framework/async-dependencies:
+      python-packages:
+        - gunicorn[gevent]
+      stage-packages:
+        # list required packages or slices for your Flask application below.
+        - libpq-dev
+
+.. warning::
+  You can only use 1 of the dependencies parts at a time.
+  (eg. either ``flask-framework/async-dependencies`` or
+  ``flask-framework/dependencies``, to read more about synchronous dependencies
+  see :ref:`flask-framework/dependencies <flask-sync-deps>`)
+
 ``parts`` > ``flask-framework/install-app`` > ``prime``
 =======================================================
 

extensions/django-framework/async-gunicorn.conf.py

@@ -0,0 +1,6 @@
+"""gunicorn configuration."""
+
+bind = ["0.0.0.0:8000"]
+chdir = "/django/app"
+statsd_host = "localhost:9125"
+worker_class = "gevent"

extensions/flask-framework/async-gunicorn.conf.py

@@ -0,0 +1,6 @@
+"""gunicorn configuration."""
+
+bind = ["0.0.0.0:8000"]
+chdir = "/flask/app"
+statsd_host = "localhost:9125"
+worker_class = "gevent"

rockcraft/commands/init.py

@@ -101,18 +101,18 @@ class InitCommand(AppCommand):
                 # the platforms this rock should be built on and run on.
                 # you can check your architecture with `dpkg --print-architecture`
                 platforms:
-                    amd64:
-                    # arm64:
-                    # ppc64el:
-                    # s390x:
+                  amd64:
+                  # arm64:
+                  # ppc64el:
+                  # s390x:
 
                 # to ensure the flask-framework extension works properly, your Flask application
                 # should have an `app.py` file with an `app` object as the WSGI entrypoint.
                 # a `requirements.txt` file with at least the flask package should also exist.
                 # see {versioned_url}/reference/extensions/flask-framework
                 # for more information.
                 extensions:
-                    - flask-framework
+                  - flask-framework
 
                 # uncomment the sections you need and adjust according to your requirements.
                 # parts:  # you need to uncomment this line to add or update any part.
@@ -130,6 +130,11 @@ class InitCommand(AppCommand):
                 #       - flask/app/templates
                 #       - flask/app/static
 
+                # uncomment this section to enable the async workers for Gunicorn.
+                #   flask-framework/async-dependencies:
+                #     python-packages:
+                #       - gunicorn[gevent]
+
                 # you may need Ubuntu packages to build a python dependency. Add them here if necessary.
                 #   flask-framework/dependencies:
                 #     build-packages:
@@ -179,10 +184,10 @@ class InitCommand(AppCommand):
                 # the platforms this rock should be built on and run on.
                 # you can check your architecture with `dpkg --print-architecture`
                 platforms:
-                    amd64:
-                    # arm64:
-                    # ppc64el:
-                    # s390x:
+                  amd64:
+                  # arm64:
+                  # ppc64el:
+                  # s390x:
 
                 # to ensure the django-framework extension functions properly, your Django project
                 # should have a structure similar to the following with ./{snake_name}/{snake_name}/wsgi.py
@@ -200,14 +205,19 @@ class InitCommand(AppCommand):
                 # +-- rockcraft.yaml
 
                 extensions:
-                    - django-framework
+                  - django-framework
 
                 # uncomment the sections you need and adjust according to your requirements.
                 # parts:
                 #   django-framework/dependencies:
                 #     stage-packages:
                 #       # list required packages or slices for your Django application below.
                 #       - libpq-dev
+                # uncomment this section to enable the async workers for Gunicorn.
+                #   django-framework/async-dependencies:
+                #     python-packages:
+                #       - gunicorn[gevent]
+
             """
             )
         ),

rockcraft/extensions/gunicorn.py

@@ -71,30 +71,62 @@ def _gen_parts(self) -> dict:
             stage_packages = ["python3.10-venv_ensurepip"]
             build_environment = [{"PARTS_PYTHON_INTERPRETER": "python3.10"}]
 
-        parts: dict[str, Any] = {
+        sync_dependencies: dict[str, Any] = {
             f"{self.framework}-framework/dependencies": {
                 "plugin": "python",
                 "stage-packages": stage_packages,
                 "source": ".",
                 "python-packages": ["gunicorn"],
                 "python-requirements": ["requirements.txt"],
                 "build-environment": build_environment,
+            }
+        }
+        async_dependencies: dict[str, Any] = {
+            f"{self.framework}-framework/async-dependencies": {
+                "plugin": "python",
+                "stage-packages": stage_packages,
+                "source": ".",
+                "python-packages": ["gunicorn[gevent]"],
+                "python-requirements": ["requirements.txt"],
+                "build-environment": build_environment,
+            }
+        }
+        sync_config_file: dict[str, Any] = {
+            f"{self.framework}-framework/config-files": {
+                "plugin": "dump",
+                "source": str(data_dir / f"{self.framework}-framework"),
+                "organize": {
+                    "gunicorn.conf.py": f"{self.framework}/sync-gunicorn.conf.py",
+                },
             },
-            f"{self.framework}-framework/install-app": self.gen_install_app_part(),
+        }
+        async_config_file: dict[str, Any] = {
             f"{self.framework}-framework/config-files": {
                 "plugin": "dump",
                 "source": str(data_dir / f"{self.framework}-framework"),
                 "organize": {
-                    "gunicorn.conf.py": f"{self.framework}/gunicorn.conf.py",
+                    "gunicorn.conf.py": f"{self.framework}/async-gunicorn.conf.py",
                 },
             },
+        }
+        parts: dict[str, Any] = {
+            f"{self.framework}-framework/install-app": self.gen_install_app_part(),
             f"{self.framework}-framework/statsd-exporter": {
                 "build-snaps": ["go"],
                 "source-tag": "v0.26.0",
                 "plugin": "go",
                 "source": "https://github.com/prometheus/statsd_exporter.git",
             },
         }
+        if f"{self.framework}-framework/async-dependencies" in self.yaml_data.get(
+            "parts", {}
+        ):
+            parts = dict(async_dependencies, **parts)
+            parts = dict(async_config_file, **parts)
+        else:
+            parts = dict(sync_dependencies, **parts)
+            parts = dict(sync_config_file, **parts)
+
         if self.yaml_data["base"] == "bare":
             parts[f"{self.framework}-framework/runtime"] = {
                 "plugin": "nil",
@@ -273,12 +305,25 @@ def _requirements_txt_error_messages(self) -> list[str]:
 
         return []
 
+    def _dependencies_error_messages(self) -> list[str]:
+        """Ensure only 1 of the dependencies parts is defined."""
+        yaml_parts = self.yaml_data.get("parts", {})
+
+        if yaml_parts.get(
+            f"{self.framework}-framework/async-dependencies", None
+        ) and yaml_parts.get(f"{self.framework}-framework/dependencies", None):
+            return [
+                f"Cannot have both sync and async dependencies. https://bit.ly/{self.framework}-async-doc"
+            ]
+        return []
+
     @override
     def check_project(self) -> None:
         """Ensure this extension can apply to the current rockcraft project."""
         error_messages = self._requirements_txt_error_messages()
         if not self.yaml_data.get("services", {}).get("flask", {}).get("command"):
             error_messages += self._wsgi_path_error_messages()
+        error_messages += self._dependencies_error_messages()
         if error_messages:
             raise ExtensionError(
                 "\n".join("- " + message for message in error_messages),

rockcraft/models/project.py

@@ -483,7 +483,7 @@ def load_project(filename: Path) -> dict[str, Any]:
         msg = err.strerror or "unknown"
         if err.filename:
             msg = f"{msg}: {err.filename!r}."
-        raise ProjectLoadError(msg) from err
+        raise ProjectLoadError(str(msg)) from err
 
     return transform_yaml(filename.parent, yaml_data)