Generate, cache and pass valid fernet keys for local deployment (#196) (

#207)

*Issue #, if available:* #196

*Description of changes:* 
- Modified run.sh to generate, cache and pass valid fernet key to the
docker-compose file.
- Added a script to be able to use pip install in run.sh to temporarily
install dependencies needed before the bootstrap steps, like in this
use-case.
- Updated pip_install_check.py and run_all.py quality_check files'
shebang for better portability.
- Updated lint_bash check to exclude .venv's generated scripts.

*Description of testing:* 
- Built and ran image locally with the run.sh script.
- Used log statements to verify the fernet key was valid and being
passed correctly.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Co-authored-by: Kashyap Kannan <[email protected]>

images/airflow/2.9.2/docker-compose.yaml

@@ -20,7 +20,7 @@ x-airflow-common: &airflow-common
     # Additional Airflow configuration can be passed here in JSON form.
     MWAA__CORE__CREATED_AT: "Tue Sep 18 23:05:58 UTC 2024"
     MWAA__CORE__CUSTOM_AIRFLOW_CONFIGS: "{}"
-    MWAA__CORE__FERNET_KEY: '{"FernetKey": "fake-key-nNge+lks3RBeGVrnZ1Dq5GjKerbZKmb7dXNnsNsGy3E="}'
+    MWAA__CORE__FERNET_KEY: ${FERNET_KEY}
     MWAA__WEBSERVER__SECRET: '{"secret_key": "fake-key-aYDdF6d+Fjznai5yBW63CUAi0IipJqDHlNSWIun6y8o="}'
     # Use this enviornment variable to enable encryption with KMS.
     MWAA__CORE__KMS_KEY_ARN: ${MWAA__CORE__KMS_KEY_ARN}

images/airflow/2.9.2/generate_fernet_key.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+"""
+This Module generates Fernet keys, which are used by Airflow for connection encryption
+"""
+
+from cryptography.fernet import Fernet
+import json
+
+def generate_fernet_key():
+    """
+    Generate a Fernet key and return it as a JSON string.
+
+    :returns A JSON string containing the generated Fernet key in the format {"FernetKey": "<key>"}
+    """
+    key = Fernet.generate_key().decode()
+    return json.dumps({"FernetKey": key})
+
+if __name__ == "__main__":
+    print(generate_fernet_key())

images/airflow/2.9.2/run.sh

@@ -10,6 +10,37 @@ else
     CONTAINER_RUNTIME="docker"
 fi
 
+# Generate valid Fernet key as json
+generate_fernet_key() {
+
+    # Install cryptography package quietly
+    chmod +x temporary-pip-install generate_fernet_key.py
+    ./temporary-pip-install cryptography >/dev/null 2>&1
+
+    # Generate the key and format as JSON
+    KEY=$(python3 generate_fernet_key.py)
+
+    # Uninstall cryptography package quietly
+    python3 -m pip uninstall -y cryptography cryptography-vectors &>/dev/null 2>&1
+
+    echo "$KEY"
+}
+
+# Set up cache directory ; generate if it dosen't exist
+CACHE_DIR="${HOME}/.cache/mwaa-local"
+FERNET_KEY_FILE="${CACHE_DIR}/fernet.key"
+mkdir -p "${CACHE_DIR}"
+
+# Check if we have a cached Fernet key, if not generate and cache it
+if [ ! -f "${FERNET_KEY_FILE}" ]; then
+    generate_fernet_key > "${FERNET_KEY_FILE}"
+    chmod 600 "${FERNET_KEY_FILE}"
+fi
+
+# Read the Fernet key from cache
+FERNET_KEY=$(cat "${FERNET_KEY_FILE}")
+export FERNET_KEY
+
 # Build the Docker image
 ./build.sh $CONTAINER_RUNTIME
 

images/airflow/2.9.2/temporary-pip-install

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# This script is specifically designed for temporarily installing packages needed ONLY before bootstrap steps.
+# It intentionally bypasses constraint checks, since it is intended that the packages will be used for setup/configuration 
+# and then UNINSTALLED before the bootstrap steps, during local setup.
+#
+# NOTE: This script should NOT be used for installing production Airflow/MWAA dependencies.
+# For those, use 'safe-pip-install' which properly handles Airflow/MWAA constraints.
+
+pip3 install "$@"

quality-checks/lint_bash.sh

@@ -9,9 +9,9 @@ if [[ "$PWD" != "$REPO_ROOT" ]]; then
     exit 1
 fi
 
-# Lint all Bash files
+# Lint all Bash files, excluding .venv directory
 echo "Running ShellCheck on Bash scripts..."
-if ! find . -type f -name "*.sh" -exec shellcheck {} +; then
+if ! find . -type f -name "*.sh" -not -path "./.venv/*" -exec shellcheck {} +; then
     echo "ShellCheck linting failed."
     exit 1
 else

quality-checks/pip_install_check.py

@@ -1,4 +1,4 @@
-#!/bin/python3
+#!/usr/bin/env python3
 """
 This module verifies there are no direct use of "pip install" in the code.
 

quality-checks/run_all.py

@@ -1,4 +1,4 @@
-#!/bin/python3
+#!/usr/bin/env python3
 """Run all quality check scripts under the quality-checks/ folder."""
 
 import os