Send failure notifications on CI/CD workflows

.github/workflows/cd-app.yml

@@ -1,3 +1,6 @@
+# This workflow handles the deployment of the application to different
+# environments.
+
 name: Deploy app
 # Need to set a default value for when the workflow is triggered from a git push
 # which bypasses the default configuration for inputs
@@ -35,3 +38,12 @@ jobs:
       app_name: "app"
       environment: ${{ inputs.environment || 'dev' }}
       version: ${{ inputs.version || 'main' }}
+  notify:
+    name: Notify
+    needs: deploy
+    if: failure() && github.event_name == 'push'
+    uses: ./.github/workflows/send-system-notification.yml
+    with:
+      channel: "workflow-failures"
+      message: "❌ [Deploy of app to ${{ inputs.environment || 'dev' }} failed](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})"
+    secrets: inherit

.github/workflows/check-infra-deploy-status.yml

@@ -72,7 +72,7 @@ jobs:
   notify:
     name: Notify
     needs: check
-    if: failure()
+    if: failure() && github.event_name == 'schedule'
     uses: ./.github/workflows/send-system-notification.yml
     with:
       channel: "workflow-failures"

.github/workflows/ci-app-infra-service.yml

@@ -1,3 +1,6 @@
+# This workflow runs end to end infrastructure tests for the service layer
+# of the app using Terratest.
+
 name: CI Infra Service Checks - app
 
 on:
@@ -45,3 +48,13 @@ jobs:
 
       - name: Run Terratest
         run: make infra-test-service APP_NAME=app
+
+  notify:
+    name: Notify
+    needs: infra-test-e2e
+    if: failure() && github.event_name == 'push'
+    uses: ./.github/workflows/send-system-notification.yml
+    with:
+      channel: "workflow-failures"
+      message: "❌ [Service layer infra tests for app failed](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})"
+    secrets: inherit

.github/workflows/ci-app-vulnerability-scans.yml

@@ -1,3 +1,5 @@
+# This workflow runs vulnerability scans on the app using several tools.
+
 name: CI Vulnerability Scans - app
 
 on:
@@ -19,10 +21,20 @@ on:
       - .trivyignore
       - .github/workflows/vulnerability-scans.yml
       - .github/workflows/ci-app-vulnerability-scans.yml
+  workflow_dispatch:
 
 jobs:
   vulnerability-scans:
     name: Vulnerability Scans
     uses: ./.github/workflows/vulnerability-scans.yml
     with:
       app_name: "app"
+  notify:
+    name: Notify
+    needs: vulnerability-scans
+    if: failure() && github.event_name == 'push'
+    uses: ./.github/workflows/send-system-notification.yml
+    with:
+      channel: "workflow-failures"
+      message: "⚠️ [Vulnerabilities found in app](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})"
+    secrets: inherit

.github/workflows/ci-docs.yml

@@ -1,10 +1,13 @@
+# This workflow runs CI checks for documentation, including linting markdown files.
+
 name: CI Documentation Checks
 
 on:
   push:
     branches:
       - main
   pull_request:
+  workflow_dispatch:
 
 jobs:
   lint-markdown:
@@ -17,3 +20,13 @@ jobs:
         with:
           use-quiet-mode: "yes" # errors only.
           config-file: ".github/workflows/markdownlint-config.json"
+
+  notify:
+    name: Notify
+    needs: lint-markdown
+    if: failure() && github.event_name == 'push'
+    uses: ./.github/workflows/send-system-notification.yml
+    with:
+      channel: "workflow-failures"
+      message: "⚠️ [Documentation checks failed](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})"
+    secrets: inherit

.github/workflows/ci-infra.yml

@@ -1,3 +1,6 @@
+# This workflow runs various checks for infrastructure code, including
+# linting, Terraform format checks, and compliance checks.
+
 name: CI Infra Checks
 
 on:
@@ -13,6 +16,7 @@ on:
       - bin/**
       - infra/**
       - .github/workflows/**
+  workflow_dispatch:
 
 jobs:
   lint-github-actions:
@@ -93,3 +97,12 @@ jobs:
         uses: aquasecurity/[email protected]
         with:
           github_token: ${{ github.token }}
+  notify:
+    name: Notify
+    needs: [lint-github-actions, lint-scripts, check-terraform-format, validate-terraform, check-compliance-with-checkov, check-compliance-with-tfsec]
+    if: failure() && github.event_name == 'push'
+    uses: ./.github/workflows/send-system-notification.yml
+    with:
+      channel: "workflow-failures"
+      message: "❌ [Infra checks failed](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})"
+    secrets: inherit