Alert for Cloud Run deployments that bypass Binary Authorization (#1533)

docs/playbooks/alerts/CloudRunBreakglass.md

@@ -0,0 +1,25 @@
+# CloudRunBreakglass
+
+This alert fires when a Cloud Run service is deployed that bypassed Binary
+Authorization using breakglass.
+
+## Triage Steps
+
+Check with your team. There may have been a legitimate reason for a breakglass
+deployment. However, you should try to get on a non-breakglass deployment as
+quickly as possible.
+
+To identify the incident(s), go to Logs Explorer and use the following filter:
+
+```text
+protoPayload.@type="type.googleapis.com/google.cloud.audit.AuditLog"
+protoPayload.serviceName="run.googleapis.com"
+protoPayload.status.message:"breakglass"
+resource.labels.revision_name!=""
+```
+
+The principal that did the breakglass deploy can be found at:
+
+```text
+protoPayload.response.metadata.annotations."serving.knative.dev/creator"
+```

terraform/alerting/alerts.tf

@@ -418,11 +418,76 @@ EOT
     }
   }
 
-
-
   label_extractors = {
     "path"     = "REGEXP_EXTRACT(httpRequest.requestUrl, \"https?://.+/(.+/.+)\\\\.zip\")"
     "platform" = "REGEXP_EXTRACT(httpRequest.userAgent, \"(Android|Darwin)\")"
   }
 }
 
+resource "google_logging_metric" "cloud_run_breakglass" {
+  name    = "cloud_run_breakglass"
+  project = var.project
+
+  filter = <<EOT
+protoPayload.@type="type.googleapis.com/google.cloud.audit.AuditLog"
+protoPayload.serviceName="run.googleapis.com"
+protoPayload.status.message:"breakglass"
+resource.labels.revision_name!=""
+EOT
+
+  metric_descriptor {
+    metric_kind = "DELTA"
+    value_type  = "INT64"
+
+    labels {
+      key         = "revision"
+      value_type  = "STRING"
+      description = "Name of the revision which was deployed with breakglass"
+    }
+  }
+
+  label_extractors = {
+    "revision" = "EXTRACT(resource.labels.revision_name)"
+  }
+}
+
+resource "google_monitoring_alert_policy" "CloudRunBreakglass" {
+  count = var.alert_on_cloud_run_breakglass ? 1 : 0
+
+  project      = var.project
+  display_name = "CloudRunBreakglass"
+  combiner     = "OR"
+
+  conditions {
+    display_name = "A service was deployed that bypassed Binary Authorization"
+
+    condition_monitoring_query_language {
+      duration = "0s"
+
+      query = <<-EOT
+      fetch global
+      | metric 'logging.googleapis.com/user/${google_logging_metric.cloud_run_breakglass.name}'
+      | align rate(5m)
+      | every 1m
+      | group_by [resource.project_id],
+          [val: aggregate(value.cloud_run_breakglass)]
+      | condition val > 0
+      EOT
+
+      trigger {
+        count = 1
+      }
+    }
+  }
+
+  documentation {
+    content   = "${local.playbook_prefix}/CloudRunBreakglass.md"
+    mime_type = "text/markdown"
+  }
+
+  notification_channels = [for x in values(google_monitoring_notification_channel.paging) : x.id]
+
+  depends_on = [
+    null_resource.manual-step-to-enable-workspace,
+  ]
+}

terraform/alerting/variables.tf

@@ -73,6 +73,13 @@ variable "alert_on_human_decrypted_value" {
   description = "Alert when a human accesses a secret. You must enable DATA_READ audit logs for Cloud KMS."
 }
 
+variable "alert_on_cloud_run_breakglass" {
+  type    = bool
+  default = true
+
+  description = "Alert when a service is deployed that bypassed Binary Authorization."
+}
+
 variable "capture_export_file_downloads" {
   type    = bool
   default = true