Use structured logging in db maintenance lambda

data_services/db_maintenance/main.py

@@ -1,8 +1,27 @@
 from pg8000.native import Connection
+import json
 
 def handler(event, _context):
-  config = event['connection']
+  max_age = event.get('max_age', '1 WEEK')
+  config = event.get('connection', {})
+  database = config['database']
+
+  log('Beginning database maintenance', { 'database': database })
   conn = Connection(**config)
+
   for table in event.get('tables', []):
-    conn.run(f"DELETE FROM {table} WHERE updated_at < NOW() - interval '1 WEEK'")
-    conn.run(f"VACUUM {table}")
+    log('Deleting stale entries', { 'database': database, 'max_age': max_age, 'table': table })
+    conn.run(f"DELETE FROM {table} WHERE updated_at < NOW() - interval '{max_age}'")
+    log('Deleted', { 'database': database, 'table': table, 'rows': conn.row_count })
+
+    log('Size before vacuuming', { 'database': database, 'table': table, 'bytes': table_size(conn, table) })
+    conn.run(f"VACUUM FULL {table}")
+    log('Size after vacuuming', { 'database': database, 'table': table, 'bytes': table_size(conn, table) })
+
+    log('Maintenance complete', { 'database': database })
+
+def log(message, detail = {}):
+  print(json.dumps({ 'message': message, 'detail': detail }))
+
+def table_size(conn, table):
+  return conn.run(f"SELECT pg_total_relation_size('{table}')")[0][0];

data_services/postgres.tf

@@ -76,7 +76,7 @@ resource "aws_db_parameter_group" "db_parameter_group" {
 }
 
 resource "aws_db_instance" "db" {
-  allocated_storage         = 100
+  allocated_storage         = 125
   apply_immediately         = true
   engine                    = "postgres"
   engine_version            = var.postgres_version
@@ -103,7 +103,7 @@ module "maintenance_lambda" {
   handler                = "main.handler"
   runtime                = "python3.10"
   source_path            = "${path.module}/db_maintenance"
-  timeout                = 120
+  timeout                = 600
 
   vpc_subnet_ids         = module.core.outputs.vpc.public_subnets.ids
   vpc_security_group_ids = [aws_security_group.db_client.id]