Merge pull request #196 from bcgov/dev

chore: switchover agent prod release

.tool-versions

@@ -1,5 +1,5 @@
 python 3.11.0
-poetry 1.2.0
+poetry 1.8.3
 helm 3.2.4
 shellcheck 0.8.0
 oc 4.7.5

Dockerfile

@@ -1,10 +1,10 @@
-FROM python:3.8.6-alpine
+FROM python:3.12.6-alpine3.19
 
 WORKDIR /app
 
 RUN python -m pip install --upgrade pip
 
-ARG POETRY_VERSION=1.2.0
+ARG POETRY_VERSION=1.8.3
 
 RUN apk add --no-cache \
         curl \

Dockerfile-test

@@ -1,10 +1,10 @@
-FROM python:3.8.6-alpine
+FROM python:3.12.6-alpine3.19
 
 WORKDIR /app
 
 RUN python -m pip install --upgrade pip
 
-ARG POETRY_VERSION=1.2.0
+ARG POETRY_VERSION=1.8.3
 
 RUN apk add --no-cache \
         curl \

docs/environment-variables.md

@@ -71,3 +71,16 @@ Then every minute a log will be output describing the state of the failover coun
 ```
 
 Pairing this with a couple of DNS checking Uptime alerts will give the team the confidence to schedule a preemptive failover for the loginproxy app.
+
+## CHES log integration
+
+To persist the events the switchover agent records, we have integrated it with the CHES email server.  There are four environment varibles that must be set for this to work:
+
+```
+CHES_TOKEN_ENDPOINT
+CHES_USERNAME
+CHES_PASSWORD
+LOG_EMAIL
+```
+
+The credentials are stored with the team secrets and the `LOG_EMAIL` is a comma separated list for the accounts we want these logs sent to.

pyproject.toml

@@ -1,20 +1,20 @@
 [tool.poetry]
 name = "sso-switchover-agent"
-version = "0.1.0"
+version = "0.1.1"
 description = ""
 authors = ["SSO Team"]
 readme = "README.md"
 packages = [{include = "sso_switchover_agent"}]
 
 [tool.poetry.dependencies]
-python = "3.8.6"
-fastapi = "0.103.1"
-python-dotenv = "^0.20.0"
+python = "3.12.6"
+fastapi = "0.115.0"
+python-dotenv = "^1.0.1"
 asyncio = "^3.4.3"
-requests = "^2.28.0"
-uvicorn = "^0.17.6"
-python-hosts = "^1.0.3"
-pytz = "^2024.1"
+requests = "^2.32.3"
+uvicorn = "^0.30.6"
+python-hosts = "^1.0.7"
+pytz = "^2024.2"
 
 [tool.poetry.dev-dependencies]
 autopep8 = "^1.6.0"

requirements.txt

@@ -3,5 +3,5 @@ gitlint==0.15.1
 autopep8==1.6.0
 pytest
 python-hosts
-requests==2.28.0
+requests==2.32.3
 pytz

src/config.py

@@ -29,5 +29,10 @@
     enable_gold_route_workflow_id=os.environ.get("ENABLE_GOLD_ROUTE_WORKFLOW_ID", "turn-off-gold-routing.yml"),
     uptime_status_api="https://uptime.com/api/v1/statuspages/",
     uptime_status_page_id=os.environ.get("UPTIME_STATUS_PAGE_ID", ""),
-    uptime_status_token=os.environ.get("UPTIME_STATUS_TOKEN", "")
+    uptime_status_token=os.environ.get("UPTIME_STATUS_TOKEN", ""),
+    ches_api_endpoint="https://ches.api.gov.bc.ca/api/v1/email",
+    ches_token_endpoint=os.environ.get("CHES_TOKEN_ENDPOINT", ""),
+    ches_username=os.environ.get("CHES_USERNAME", ""),
+    ches_password=os.environ.get("CHES_PASSWORD", ""),
+    log_email=os.environ.get("LOG_EMAIL", "")
 )

src/logic.py

@@ -18,6 +18,12 @@
 css_maintenance_workflow_id = config.get('css_maintenance_workflow_id')
 css_gh_token = config.get('css_gh_token')
 
+ches_api_endpoint = config.get('ches_api_endpoint')
+ches_token_endpoint = config.get('ches_token_endpoint')
+ches_username = config.get('ches_username')
+ches_password = config.get('ches_password')
+log_email = config.get('log_email')
+
 incident_id = None
 
 
@@ -54,11 +60,61 @@ def handle_queues(queue: Queue, processes: list):
                 dispatch_action_by_id(config.get("enable_gold_route_workflow_id"))
                 logger.debug(item['message'])
 
+            log_events(item)
+
         except Exception as ex:
             logger.error('Unknown error in logic. %s' % ex)
             traceback.print_exc(file=sys.stdout)
 
 
+def log_events(queu_item: object):
+    if ches_token_endpoint != "" and ches_username != "" and ches_password != "":
+        body = f"<p>The following change was detected by the switchover agent:</p> \
+        <p>{json.dumps(queu_item)}</p> \
+        <br> <p>{datetime.datetime.now()}</p>"
+
+        sendChesEmail(log_email, body)
+
+
+def sendChesEmail(to: str, body: str):
+    try:
+        namespace = config.get('namespace')
+        access_token = fetchChesToken()
+
+        if access_token is None:
+            logger.error('No Access Token Created')
+            return
+
+        payload = {
+            "bodyType": "html",
+            "body": body,
+            "encoding": "utf-8",
+            "from": "[email protected]",
+            "priority": "normal",
+            "to": to.split(','),
+            "subject": f"The {namespace} switchover agent detected a change.",
+        }
+        headers = {"Authorization": f"Bearer {access_token}"}
+        response = requests.post(ches_api_endpoint, json=payload, headers=headers)
+    except Exception as ex:
+        logger.error('Log failed to send log to team email: %s' % ex)
+
+        traceback.print_exc(file=sys.stdout)
+
+
+def fetchChesToken():
+    auth_url = ches_token_endpoint
+    payload = {"grant_type": "client_credentials"}
+    headers = {"content-type": "application/x-www-form-urlencoded"}
+    try:
+        response = requests.post(auth_url, auth=(ches_username, ches_password), data=payload, headers=headers)
+        return response.json()['access_token']
+    except Exception as ex:
+        logger.error('Failed to get a ches token: %s' % ex)
+        traceback.print_exc(file=sys.stdout)
+    return None
+
+
 def action_dispatcher(ip: str, prev_ip: str, active_ip: str, passive_ip: str):
     if (ip == active_ip and prev_ip == passive_ip):
         css_maintenance_to_active = True