S3 to Azure Blob Storage backups

* Conditionally creates resources to run a scheduled task that backs up
  given S3 backups to an Azure Blob Storage Container
* CodeBuild is used to pull the dxw/s3-to-azure repo, build the image
  and push it to ECR
* Scheduled tasks (Fargate conatiners using the s3-to-azure image) are
  then created for each given S3 bucket, to backup to Azure
* Each S3 bucket can be given a different Azure Storage Account /
  Container target, and also be ran at different chosen cron expressions
* Permissions for the S3 buckets are directly added to the Fargate task
  role. The Azure permissions need to be provided (eg. Tenant ID, App ID
  and Secret)

buildspecs/dalmatian-s3-azure-blob-storage-backups.yml

@@ -0,0 +1,24 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - echo "Build started on $(date)"
+      - echo "Logging in to Amazon ECR..."
+      - aws ecr get-login-password --region "$AWS_DEFAULT_REGION" | docker login --username AWS --password-stdin "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com"
+      - |
+        if [ -n "$DOCKERHUB_USERNAME" ] && [ -n "DOCKERHUB_TOKEN" ];
+        then
+          echo "Logging into Dockerhub ...";
+          echo "$DOCKERHUB_TOKEN" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin;
+        fi;
+      - echo Building s3-azure-blob-storage-backups docker image ...
+      - docker build -t s3-azure-blob-storage-backups:latest .
+  build:
+    commands:
+      - echo Adding ECR repo tag...
+      - docker tag s3-azure-blob-storage-backups:latest "$REPOSITORY_URI:latest"
+  post_build:
+    commands:
+      - echo Pushing the Docker image...
+      - docker push "$REPOSITORY_URI:latest"

locals.tf

@@ -266,6 +266,14 @@ locals {
     "redis" = 6379
   }
 
+  enable_s3_backup_to_azure_blob_storage                           = var.enable_s3_backup_to_azure_blob_storage
+  s3_backup_to_azure_blob_storage_login_type                       = var.s3_backup_to_azure_blob_storage_login_type
+  s3_backup_to_azure_blob_storage_spa_application_id               = var.s3_backup_to_azure_blob_storage_spa_application_id
+  s3_backup_to_azure_blob_storage_spa_client_secret                = var.s3_backup_to_azure_blob_storage_spa_client_secret
+  s3_backup_to_azure_blob_storage_tenant_id                        = var.s3_backup_to_azure_blob_storage_tenant_id
+  s3_backup_to_azure_blob_storage_source_and_targets               = var.s3_backup_to_azure_blob_storage_source_and_targets
+  s3_backup_to_azure_blob_storage_backups_tooling_ecs_cluster_name = "${local.resource_prefix}-s3-azure-blob-storage-backups-tooling"
+
   custom_route53_hosted_zones = var.custom_route53_hosted_zones
 
   custom_s3_buckets = var.custom_s3_buckets

s3-azure-blob-storage-backups-cloudwatch-logs.tf

@@ -0,0 +1,8 @@
+resource "aws_cloudwatch_log_group" "s3_azure_blob_storage_backups" {
+  for_each = local.enable_s3_backup_to_azure_blob_storage ? local.s3_backup_to_azure_blob_storage_source_and_targets : {}
+
+  name              = "${local.resource_prefix}-s3-azure-blob-storage-backups-${each.key}"
+  retention_in_days = 30
+  kms_key_id        = local.infrastructure_kms_encryption ? aws_kms_key.infrastructure[0].arn : null
+  skip_destroy      = true
+}

s3-azure-blob-storage-backups-ecr.tf

@@ -0,0 +1,17 @@
+resource "aws_ecr_repository" "s3_azure_blob_storage_backups" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  name = "${local.resource_prefix}-s3-azure-blob-storage-backups"
+
+  #tfsec:ignore:aws-ecr-enforce-immutable-repository
+  image_tag_mutability = "MUTABLE"
+
+  encryption_configuration {
+    encryption_type = local.infrastructure_kms_encryption ? "KMS" : "AES256"
+    kms_key         = local.infrastructure_kms_encryption ? aws_kms_key.infrastructure[0].arn : null
+  }
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+}

s3-azure-blob-storage-backups-image-codebuild.tf

@@ -0,0 +1,144 @@
+resource "aws_iam_role" "s3_azure_blob_storage_backups_image_codebuild" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  name        = "${local.resource_prefix}-${substr(sha512("s3-azure-blob-storage-backups-image-codebuild"), 0, 6)}"
+  description = "${local.resource_prefix}-s3-azure-blob-storage-backups-image-codebuild"
+  assume_role_policy = templatefile(
+    "${path.root}/policies/assume-roles/service-principle-standard.json.tpl",
+    { services = jsonencode(["codebuild.amazonaws.com", "events.amazonaws.com"]) }
+  )
+}
+
+resource "aws_iam_policy" "s3_azure_blob_storage_backups_image_codebuild_cloudwatch_rw" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  name        = "${local.resource_prefix}-${substr(sha512("s3-azure-blob-storage-backups-image-codebuild-cloudwatch-rw"), 0, 6)}"
+  description = "${local.resource_prefix}-s3-azure-blob-storage-backups-image-codebuild-cloudwatch-rw"
+  policy      = templatefile("${path.root}/policies/cloudwatch-logs-rw.json.tpl", {})
+}
+
+resource "aws_iam_role_policy_attachment" "s3_azure_blob_storage_backups_image_codebuild_cloudwatch_rw" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  role       = aws_iam_role.s3_azure_blob_storage_backups_image_codebuild[0].name
+  policy_arn = aws_iam_policy.s3_azure_blob_storage_backups_image_codebuild_cloudwatch_rw[0].arn
+}
+
+resource "aws_iam_policy" "s3_azure_blob_storage_backups_image_codebuild_allow_builds" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  name        = "${local.resource_prefix}-${substr(sha512("s3-azure-blob-storage-backups-image-codebuild-allow-builds"), 0, 6)}"
+  description = "${local.resource_prefix}-s3-azure-blob-storage-backups-image-codebuild-allow-builds"
+  policy      = templatefile("${path.root}/policies/codebuild-allow-builds.json.tpl", {})
+}
+
+resource "aws_iam_role_policy_attachment" "s3_azure_blob_storage_backups_image_codebuild_allow_builds" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  role       = aws_iam_role.s3_azure_blob_storage_backups_image_codebuild[0].name
+  policy_arn = aws_iam_policy.s3_azure_blob_storage_backups_image_codebuild_allow_builds[0].arn
+}
+
+resource "aws_iam_policy" "s3_azure_blob_storage_backups_image_codebuild_ecr_push" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  name        = "${local.resource_prefix}-${substr(sha512("s3-azure-blob-storage-backups-image-codebuild-ecr-push"), 0, 6)}"
+  description = "${local.resource_prefix}-s3-azure-blob-storage-backups-image-codebuild-ecr-push"
+  policy = templatefile(
+    "${path.root}/policies/ecr-push.json.tpl",
+    { ecr_repository_arn = aws_ecr_repository.s3_azure_blob_storage_backups[0].arn }
+  )
+}
+
+resource "aws_iam_role_policy_attachment" "s3_azure_blob_storage_backups_image_codebuild_ecr_push" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  role       = aws_iam_role.s3_azure_blob_storage_backups_image_codebuild[0].name
+  policy_arn = aws_iam_policy.s3_azure_blob_storage_backups_image_codebuild_ecr_push[0].arn
+}
+
+resource "aws_codebuild_project" "s3_azure_blob_storage_backups_image_build" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  name          = "${local.resource_prefix}-s3-azure-blob-storage-backups-image-build"
+  description   = "${local.resource_prefix} S3 Azure Blob Storage Backups Image Build"
+  build_timeout = "20"
+  service_role  = aws_iam_role.s3_azure_blob_storage_backups_image_codebuild[0].arn
+
+  artifacts {
+    type = "NO_ARTIFACTS"
+  }
+
+  environment {
+    compute_type    = "BUILD_GENERAL1_SMALL"
+    image           = "aws/codebuild/standard:7.0"
+    type            = "LINUX_CONTAINER"
+    privileged_mode = true
+
+    environment_variable {
+      name  = "AWS_ACCOUNT_ID"
+      value = local.aws_account_id
+    }
+
+    environment_variable {
+      name  = "REPOSITORY_URI"
+      value = aws_ecr_repository.s3_azure_blob_storage_backups[0].repository_url
+    }
+
+    environment_variable {
+      name  = "DOCKERHUB_USERNAME"
+      value = local.infrastructure_dockerhub_username
+    }
+
+    environment_variable {
+      name  = "DOCKERHUB_TOKEN"
+      value = local.infrastructure_dockerhub_token
+    }
+  }
+
+  source {
+    type            = "GITHUB"
+    location        = "https://github.com/dxw/s3-to-azure"
+    git_clone_depth = 1
+    buildspec       = templatefile("${path.root}/buildspecs/dalmatian-sql-backup.yml", {})
+  }
+
+  depends_on = [
+    aws_iam_role_policy_attachment.s3_azure_blob_storage_backups_image_codebuild_cloudwatch_rw,
+    aws_iam_role_policy_attachment.s3_azure_blob_storage_backups_image_codebuild_allow_builds,
+    aws_iam_role_policy_attachment.s3_azure_blob_storage_backups_image_codebuild_ecr_push,
+  ]
+}
+
+resource "terraform_data" "s3_azure_blob_storage_backups_image_build_trigger_codebuild" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  triggers_replace = [
+    md5(templatefile("${path.root}/buildspecs/dalmatian-s3-azure-blob-storage-backups.yml", {})),
+  ]
+
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
+      ${path.root}/local-exec-scripts/trigger-codedeploy-project.sh \
+      -n "${aws_codebuild_project.s3_azure_blob_storage_backups_image_build[0].name}"
+    EOF
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "s3_azure_blob_storage_backups_image_build_trigger_codebuild" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  name                = "${local.resource_prefix_hash}-s3-azure-blob-storage-bkps-img-build-trigger-codebuild"
+  description         = "${local.resource_prefix} S3 Azure blob storage backups Image Build Trigger CodeBuild"
+  schedule_expression = "rate(24 hours)"
+}
+
+resource "aws_cloudwatch_event_target" "s3_azure_blob_storage_backups_image_build_trigger_codebuild" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  target_id = "${local.resource_prefix_hash}-s3-azure-blob-storage-bkps-img-build-trigger-codebuild"
+  rule      = aws_cloudwatch_event_rule.s3_azure_blob_storage_backups_image_build_trigger_codebuild[0].name
+  arn       = aws_codebuild_project.s3_azure_blob_storage_backups_image_build[0].id
+  role_arn  = aws_iam_role.s3_azure_blob_storage_backups_image_codebuild[0].arn
+}

s3-azure-blob-storage-backups-scheduled-task.tf

@@ -0,0 +1,92 @@
+resource "aws_iam_role" "s3_azure_blob_storage_backups_cloudwatch_schedule" {
+  for_each = local.enable_s3_backup_to_azure_blob_storage ? local.s3_backup_to_azure_blob_storage_source_and_targets : {}
+
+  name        = "${local.resource_prefix}-${substr(sha512("s3-azure-blob-storage-backups-cloudwatch-schedule-${each.key}"), 0, 6)}"
+  description = "${local.resource_prefix}-s3-azure-blob-storage-backups-${each.key}"
+  assume_role_policy = templatefile(
+    "${path.root}/policies/assume-roles/service-principle-standard.json.tpl",
+    { services = jsonencode(["events.amazonaws.com"]) }
+  )
+}
+
+resource "aws_iam_policy" "s3_azure_blob_storage_backups_cloudwatch_schedule_ecs_run_task" {
+  for_each = local.enable_s3_backup_to_azure_blob_storage ? local.s3_backup_to_azure_blob_storage_source_and_targets : {}
+
+  name        = "${local.resource_prefix}-${substr(sha512("s3-azure-blob-storage-backups-cloudwatch-schedule-${each.key}-ecs-run-task"), 0, 6)}"
+  description = "${local.resource_prefix}-s3-azure-blob-storage-backups-cloudwatch-schedule-${each.key}-ecs-run-task"
+  policy      = templatefile("${path.root}/policies/ecs-run-task.json.tpl", {})
+}
+
+resource "aws_iam_role_policy_attachment" "s3_azure_blob_storage_backups_cloudwatch_schedule_ecs_run_task" {
+  for_each = local.enable_s3_backup_to_azure_blob_storage ? local.s3_backup_to_azure_blob_storage_source_and_targets : {}
+
+  role       = aws_iam_role.s3_azure_blob_storage_backups_cloudwatch_schedule[each.key].name
+  policy_arn = aws_iam_policy.s3_azure_blob_storage_backups_cloudwatch_schedule_ecs_run_task[each.key].arn
+}
+
+resource "aws_iam_policy" "s3_azure_blob_storage_backups_cloudwatch_schedule_pass_role" {
+  for_each = local.enable_s3_backup_to_azure_blob_storage ? local.s3_backup_to_azure_blob_storage_source_and_targets : {}
+
+  name        = "${local.resource_prefix}-${substr(sha512("s3-azure-blob-storage-backups-cloudwatch-schedule-${each.key}-pass-role-execution-role"), 0, 6)}"
+  description = "${local.resource_prefix}-s3-azure-blob-storage-backups-cloudwatch-schedule-${each.key}-pass-role-execution-role"
+  policy = templatefile(
+    "${path.root}/policies/pass-role.json.tpl",
+    {
+      role_arns = jsonencode([
+        aws_iam_role.s3_azure_blob_storage_backups_task_execution[each.key].arn,
+        aws_iam_role.s3_azure_blob_storage_backups_task[each.key].arn,
+      ])
+      services = jsonencode(["ecs-tasks.amazonaws.com"])
+    }
+  )
+}
+
+resource "aws_iam_role_policy_attachment" "s3_azure_blob_storage_backups_cloudwatch_schedule_pass_role" {
+  for_each = local.enable_s3_backup_to_azure_blob_storage ? local.s3_backup_to_azure_blob_storage_source_and_targets : {}
+
+  role       = aws_iam_role.s3_azure_blob_storage_backups_cloudwatch_schedule[each.key].name
+  policy_arn = aws_iam_policy.s3_azure_blob_storage_backups_cloudwatch_schedule_pass_role[each.key].arn
+}
+
+resource "aws_cloudwatch_event_rule" "s3_azure_blob_storage_backups_scheduled_task" {
+  for_each = local.enable_s3_backup_to_azure_blob_storage ? local.s3_backup_to_azure_blob_storage_source_and_targets : {}
+
+  name                = "${local.resource_prefix}-s3-azure-blob-storage-backups-${each.key}"
+  description         = "Run ${local.resource_prefix}-s3-azure-blob-storage-backups-cloudwatch-schedule-${each.key} task at a scheduled time (${each.value["cron_expression"]})"
+  schedule_expression = each.value["cron_expression"]
+}
+
+resource "aws_cloudwatch_event_target" "s3_azure_blob_storage_backups_scheduled_task" {
+  for_each = local.enable_s3_backup_to_azure_blob_storage ? local.s3_backup_to_azure_blob_storage_source_and_targets : {}
+
+  target_id = "${local.resource_prefix}-s3-azure-blob-storage-backups-${each.key}"
+  rule      = aws_cloudwatch_event_rule.s3_azure_blob_storage_backups_scheduled_task[each.key].name
+  arn       = aws_ecs_cluster.s3_azure_blob_storage_backups_tooling[0].arn
+  role_arn  = aws_iam_role.s3_azure_blob_storage_backups_cloudwatch_schedule[each.key].arn
+  input     = jsonencode({})
+
+  ecs_target {
+    task_count          = 1
+    task_definition_arn = aws_ecs_task_definition.s3_azure_blob_storage_backups_scheduled_task[each.key].arn
+    launch_type         = "FARGATE"
+    platform_version    = "1.4.0"
+    propagate_tags      = "TASK_DEFINITION"
+
+    network_configuration {
+      subnets = local.infrastructure_vpc_network_enable_private ? [
+        for subnet in aws_subnet.infrastructure_private : subnet.id
+        ] : local.infrastructure_vpc_network_enable_public ? [
+        for subnet in aws_subnet.infrastructure_public : subnet.id
+      ] : null
+      assign_public_ip = local.infrastructure_vpc_network_enable_private ? false : local.infrastructure_vpc_network_enable_public ? true : false
+      security_groups = [
+        aws_security_group.s3_azure_blob_storage_backups_scheduled_task[0].id,
+      ]
+    }
+  }
+
+  depends_on = [
+    aws_iam_role_policy_attachment.s3_azure_blob_storage_backups_cloudwatch_schedule_ecs_run_task,
+    aws_iam_role_policy_attachment.s3_azure_blob_storage_backups_cloudwatch_schedule_pass_role,
+  ]
+}

s3-azure-blob-storage-backups-security-group.tf

@@ -0,0 +1,65 @@
+resource "aws_security_group" "s3_azure_blob_storage_backups_scheduled_task" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  name        = "${local.resource_prefix}-s3-azure-blob-storage-backups-scheduled-task"
+  description = "S3 Azure blob storage backups scheduled task"
+  vpc_id      = aws_vpc.infrastructure[0].id
+}
+
+resource "aws_security_group_rule" "s3_azure_blob_storage_backups_scheduled_task_egress_https_tcp" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  description = "Allow HTTPS tcp outbound"
+  type        = "egress"
+  from_port   = 443
+  to_port     = 443
+  protocol    = "tcp"
+  # tfsec:ignore:aws-ec2-no-public-egress-sgr
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = aws_security_group.s3_azure_blob_storage_backups_scheduled_task[0].id
+}
+
+resource "aws_security_group_rule" "s3_azure_blob_storage_backups_scheduled_task_egress_https_udp" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  description = "Allow HTTPS udp outbound"
+  type        = "egress"
+  from_port   = 443
+  to_port     = 443
+  protocol    = "udp"
+  # tfsec:ignore:aws-ec2-no-public-egress-sgr
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = aws_security_group.s3_azure_blob_storage_backups_scheduled_task[0].id
+}
+
+resource "aws_security_group_rule" "s3_azure_blob_storage_backups_scheduled_task_egress_dns_tcp" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  description = "Allow DNS tcp outbound to AWS"
+  type        = "egress"
+  from_port   = 53
+  to_port     = 53
+  protocol    = "tcp"
+  cidr_blocks = local.infrastructure_ecs_cluster_publicly_avaialble ? [
+    for subnet in aws_subnet.infrastructure_public : subnet.cidr_block
+    ] : [
+    for subnet in aws_subnet.infrastructure_private : subnet.cidr_block
+  ]
+  security_group_id = aws_security_group.s3_azure_blob_storage_backups_scheduled_task[0].id
+}
+
+resource "aws_security_group_rule" "s3_azure_blob_storage_backups_scheduled_task_egress_dns_udp" {
+  count = local.enable_s3_backup_to_azure_blob_storage ? 1 : 0
+
+  description = "Allow DNS udp outbound to AWS"
+  type        = "egress"
+  from_port   = 53
+  to_port     = 53
+  protocol    = "udp"
+  cidr_blocks = local.infrastructure_ecs_cluster_publicly_avaialble ? [
+    for subnet in aws_subnet.infrastructure_public : subnet.cidr_block
+    ] : [
+    for subnet in aws_subnet.infrastructure_private : subnet.cidr_block
+  ]
+  security_group_id = aws_security_group.s3_azure_blob_storage_backups_scheduled_task[0].id
+}