Merge pull request #1 from silinternational/develop

Develop

.dockerignore

@@ -0,0 +1,5 @@
+*.aes
+codeship-services.yml
+codeship-steps.yml
+dockercfg
+*.env

.gitignore

@@ -9,3 +9,9 @@
 node_modules/
 vendor/
 .terraform/
+
+# tfc-dump creates .json files that don't belong in version control
+*.json
+*.env
+*.aes
+dockercfg

Dockerfile

@@ -0,0 +1,30 @@
+FROM alpine:3
+
+# Variables set with ARG can be overridden at image build time with
+# "--build-arg var=value".  They are not available in the running container.
+ARG restic_ver=0.16.0
+ARG tfc_ops_ver=3.5.1
+ARG tfc_ops_distrib=tfc-ops_${tfc_ops_ver}_Linux_x86_64.tar.gz
+
+# Install Restic, tfc-ops, perl, and jq
+RUN cd /tmp \
+ && wget -O /tmp/restic.bz2 \
+    https://github.com/restic/restic/releases/download/v${restic_ver}/restic_${restic_ver}_linux_amd64.bz2 \
+ && bunzip2 /tmp/restic.bz2 \
+ && chmod +x /tmp/restic \
+ && mv /tmp/restic /usr/local/bin/restic \
+ && wget https://github.com/silinternational/tfc-ops/releases/download/v${tfc_ops_ver}/${tfc_ops_distrib} \
+ && tar zxf ${tfc_ops_distrib} \
+ && rm LICENSE README.md ${tfc_ops_distrib} \
+ && mv tfc-ops /usr/local/bin \
+ && apk update \
+ && apk add --no-cache perl jq curl \
+ && rm -rf /var/cache/apk/*
+
+COPY ./tfc-backup-b2.sh  /usr/local/bin/tfc-backup-b2.sh
+COPY ./tfc-dump.pl       /usr/local/bin/tfc-dump.pl
+COPY application/        /data/
+
+WORKDIR /data
+
+CMD [ "/usr/local/bin/tfc-backup-b2.sh" ]

README.md

@@ -1,2 +1,63 @@
-# template-public
-general template for public repositories
+# tfc-backup-b2
+Docker image to export variables from Terraform Cloud and back them up to a
+Restic repository on Backblaze B2.
+The image can also initialize the Restic repository on the existing
+Backblaze B2 bucket.
+
+## Description
+During the review of a disaster recovery plan, we realized that we didn't have
+a record of the values we set for variables in Terraform Cloud workspaces.
+It would be difficult to recover from the accidental deletion of a Terraform
+Cloud workspace.
+A Perl script exports workspaces, variables, and variable sets to JSON files
+using the Terraform Cloud API.
+The JSON files are then backed up using Restic to a repository on a Backblaze
+B2 bucket.
+
+Two files are created for each Terraform Cloud workspace:
+
+- _workspace-name_-attributes.json
+- _workspace-name_-variables.json
+
+Two files are created for each Terraform Cloud Variable Set:
+
+- varset-_variable-set-name_-attributes.json
+- varset-_variable-set-name_-variables.json
+
+Spaces in the variable set name are replaced with hyphens (`-`).
+
+## How to use it
+
+1. Copy `local.env.dist` to `local.env`.
+1. Set the values for the variables contained in `local.env`.
+1. Obtain a Terraform Cloud access token. Go to [https://app.terraform.io/app/settings/tokens](https://app.terraform.io/app/settings/tokens) to create an API token.
+1. Add the access token as the value for `ATLAS_TOKEN` in `local.env`.
+1. Create a Backblaze B2 bucket. Set the `File Lifecycle` to `Keep only the last version`.
+1. Add the B2 bucket name to `RESTIC_REPOSITORY` in `local.env`.
+1. Obtain a Backblaze Application Key. Restrict its access to the B2 bucket you just created. Ensure the application key has these capabilities: `deleteFiles`, `listBuckets`, `listFiles`, `readBuckets`, `readFiles`, `writeBuckets`, `writeFiles`.
+1. Add the application key and secret to `local.env` as the values of `B2_ACCOUNT_ID` and `B2_ACCOUNT_KEY` respectively.
+1. Initialize the Restic repository (one time only):  `docker run --env-file=local.env --env BACKUP_MODE=init silintl/tfc-backup-b2:latest`
+1. Run the Docker image:  `docker run --env-file=local.env silintl/tfc-backup-b2:latest`
+
+### Variables
+
+* `ATLAS_TOKEN`        - Terraform Cloud access token
+* `B2_ACCOUNT_ID`      - Backblaze keyID
+* `B2_ACCOUNT_KEY`     - Backblaze applicationKey
+* `FSBACKUP_MODE`      - `init` initializes the Restic repository at `$RESTIC_REPOSITORY` (only do this once), `backup` performs a backup
+* `ORGANIZATION`       - Name of the Terraform Cloud organization to be backed up
+* `RESTIC_BACKUP_ARGS` - additional arguments to pass to `restic backup` command
+* `RESTIC_FORGET_ARGS` - additional arguments to pass to `restic forget --prune` command (e.g., `--keep-daily 7 --keep-weekly 5  --keep-monthly 3 --keep-yearly 2`)
+* `RESTIC_HOST`        - hostname to be used for the backup
+* `RESTIC_PASSWORD`    - password for the Restic repository
+* `RESTIC_REPOSITORY`  - Restic repository location (e.g., `b2:bucketname:restic`)
+* `RESTIC_TAG`         - tag to apply to the backup
+* `SOURCE_PATH`        - Full path to the directory to be backed up
+
+## Restrictions
+The code assumes that all of the Terraform Cloud Variable Sets are contained
+within the first result page of 20 entries.
+
+## Docker Hub
+This image is built automatically on Docker Hub as [silintl/tfc-backup-b2](https://hub.docker.com/r/silintl/tfc-backup-b2/)
+

application/backup.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env sh
+
+STATUS=0
+myname="tfc-backup-b2"
+
+echo "${myname}: Backing up ${SOURCE_PATH}"
+
+start=$(date +%s)
+/usr/local/bin/restic backup --host ${RESTIC_HOST} --tag ${RESTIC_TAG} ${RESTIC_BACKUP_ARGS} ${SOURCE_PATH} || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+	echo "${myname}: FATAL: Backup returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+	exit $STATUS
+else
+	echo "${myname}: Backup completed in $(expr ${end} - ${start}) seconds."
+fi
+
+start=$(date +%s)
+/usr/local/bin/restic forget --host ${RESTIC_HOST} ${RESTIC_FORGET_ARGS} --prune || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+	echo "${myname}: FATAL: Backup pruning returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+	exit $STATUS
+else
+	echo "${myname}: Backup pruning completed in $(expr ${end} - ${start}) seconds."
+fi
+
+start=$(date +%s)
+/usr/local/bin/restic check || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+	echo "${myname}: FATAL: Repository check returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+	exit $STATUS
+else
+	echo "${myname}: Repository check completed in $(expr ${end} - ${start}) seconds."
+fi
+
+start=$(date +%s)
+/usr/local/bin/restic unlock || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+	echo "${myname}: FATAL: Repository unlock returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+	exit $STATUS
+else
+	echo "${myname}: Repository unlock completed in $(expr ${end} - ${start}) seconds."
+fi
+
+exit $STATUS

application/init.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env sh
+
+STATUS=0
+myname="tfc-backup-b2"
+
+echo "${myname}: init: Started"
+
+start=$(date +%s)
+/usr/local/bin/restic init || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+	echo "${myname}: FATAL: Repository initialization returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+	exit $STATUS
+else
+	echo "${myname}: Repository initialization completed in $(expr ${end} - ${start}) seconds."
+fi
+
+echo "${myname}: init: Completed"
+exit $STATUS

codeship-services.yml

@@ -0,0 +1,5 @@
+app:
+  build:
+    image: silintl/tfc-backup-b2
+    dockerfile: ./Dockerfile
+  cached: true

codeship-steps.yml

@@ -0,0 +1,22 @@
+- name: push_branch
+  service: app
+  type: push
+  image_name: silintl/tfc-backup-b2
+  image_tag: "{{.Branch}}"
+  exclude: (main)
+  registry: https://index.docker.io/v1/
+  encrypted_dockercfg_path: dockercfg.encrypted
+
+- name: push_latest
+  service: app
+  type: push
+  image_name: silintl/tfc-backup-b2
+  image_tag: "latest"
+  tag: main
+  registry: https://index.docker.io/v1/
+  encrypted_dockercfg_path: dockercfg.encrypted
+
+#- name: test
+#  service: app
+#  command: echo "Image was tested"
+

dockercfg.encrypted

@@ -0,0 +1,2 @@
+codeship:v2
+7dIJy1+PbnBmTeT07EE1uGyRLKXSVolFSBdCuppOIGWBGFaINyq82M6ggw37zZJewKvnjuMkgGFH2uEoDd7sSDFZnvNqgsAKvU9NGhl2hQqfPOgE2LZe4UbFuH/GGWXL/mQ24pDdxkE90GL6Zma1Is7O3yxybeffNEdMqIeEAwMR4IqB6Zp0I6RwcpMN+PrrpS3IYXUx78VHHMk9vYh2mC4+XI6mIKDZSKw/ozZTqGy4iJtY2HJuRuOGM8UanJAB+5s0f1NV5WAu0YW9Mdj3F4K8zPNdEVaC+lpM0hQpDxCOkurU1VuqKRylXUPWuHG/xoY5jB3yX7vkhJcsdckIoXXIB+S1Eki4U/+hOQ1Sc9Ftx38O4r9l4pu9G3k0dUVaGM7to3NikFWcWpLRAsbR4BOHN8eL34g+VWT715DK0hcDysBqdXBv3w==

local.env.dist

@@ -0,0 +1,27 @@
+# ATLAS_TOKEN        - Terraform Cloud access token
+# B2_ACCOUNT_ID      - Backblaze keyID
+# B2_ACCOUNT_KEY     - Backblaze applicationKey
+# FSBACKUP_MODE      - `init` initializes the Restic repository at `$RESTIC_REPOSITORY` (only do this once)
+#                      `backup` performs a backup
+# ORGANIZATION       - Name of the Terraform Cloud organization to be backed up
+# RESTIC_BACKUP_ARGS - additional arguments to pass to 'restic backup' command
+# RESTIC_FORGET_ARGS - additional arguments to pass to 'restic forget --prune' command
+#                      (e.g., --keep-daily 7 --keep-weekly 5  --keep-monthly 3 --keep-yearly 2)
+# RESTIC_HOST        - hostname to be used for the backup
+# RESTIC_PASSWORD    - password for the Restic repository
+# RESTIC_REPOSITORY  - Restic repository location (e.g., 'b2:bucketname:restic')
+# RESTIC_TAG         - tag to apply to the backup
+# SOURCE_PATH        - Full path to the directory to be backed up
+
+ATLAS_TOKEN=
+B2_ACCOUNT_ID=
+B2_ACCOUNT_KEY=
+FSBACKUP_MODE=backup
+ORGANIZATION=
+RESTIC_BACKUP_ARGS=
+RESTIC_FORGET_ARGS=
+RESTIC_HOST=
+RESTIC_PASSWORD=
+RESTIC_REPOSITORY=b2:backblaze-bucket-name-goes-here:restic
+RESTIC_TAG=
+SOURCE_PATH=/tmp/tfc-backup-b2

tfc-backup-b2.sh

@@ -0,0 +1,76 @@
+#!/bin/sh
+
+# tfc-backup-b2.sh - Back up Terraform Cloud workspaces and variables to a Restic repository on Backblaze B2
+#
+# Dale Newby
+# SIL International
+# July 20, 2023
+
+# Required environment variables:
+# ATLAS_TOKEN        - Terraform Cloud access token
+# B2_ACCOUNT_ID      - Backblaze keyID
+# B2_ACCOUNT_KEY     - Backblaze applicationKey
+# FSBACKUP_MODE      - `init` initializes the Restic repository at `$RESTIC_REPOSITORY` (only do this once)
+#                      `backup` performs a backup
+# ORGANIZATION       - Name of the Terraform Cloud organization to be backed up
+# RESTIC_BACKUP_ARGS - additional arguments to pass to 'restic backup' command
+# RESTIC_FORGET_ARGS - additional arguments to pass to 'restic forget --prune' command
+#                      (e.g., --keep-daily 7 --keep-weekly 5  --keep-monthly 3 --keep-yearly 2)
+# RESTIC_HOST        - hostname to be used for the backup
+# RESTIC_PASSWORD    - password for the Restic repository
+# RESTIC_REPOSITORY  - Restic repository location (e.g., 'b2:bucketname:restic')
+# RESTIC_TAG         - tag to apply to the backup
+# SOURCE_PATH        - Full path to the directory to be backed up
+
+STATUS=0
+myname="tfc-backup-b2"
+
+case "${FSBACKUP_MODE}" in
+	init)
+		/data/${FSBACKUP_MODE}.sh || STATUS=$?
+		;;
+	backup)
+		echo "${myname}: backup: Started"
+		echo "${myname}: Exporting Terraform Cloud data to ${SOURCE_PATH}"
+
+		mkdir -p ${SOURCE_PATH} && cd ${SOURCE_PATH} && rm -rf *
+		if [ $STATUS -ne 0 ]; then
+			echo "${myname}: FATAL: Cannot create directory ${SOURCE_PATH}: $STATUS"
+			exit $STATUS
+		fi
+
+		start=$(date +%s)
+		/usr/local/bin/tfc-dump.pl --org ${ORGANIZATION} --all --quiet
+		end=$(date +%s)
+
+		if [ $STATUS -ne 0 ]; then
+			echo "${myname}: FATAL: Terraform Cloud export returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+        		exit $STATUS
+		else
+        		echo "${myname}: Terraform Cloud export completed in $(expr ${end} - ${start}) seconds."
+		fi
+
+		/data/${FSBACKUP_MODE}.sh || STATUS=$?
+		if [ $STATUS -ne 0 ]; then
+			echo "${myname}: FATAL: backup failed: $STATUS"
+			exit $STATUS
+		fi
+
+		cd .. &&  rm -rf ${SOURCE_PATH} || STATUS=$?
+		if [ $STATUS -ne 0 ]; then
+			echo "${myname}: FATAL: Cannot remove directory ${SOURCE_PATH}: $STATUS"
+			exit $STATUS
+		fi
+
+		echo "${myname}: backup: Completed"
+		;;
+	*)
+		echo "${myname}: FATAL: Unknown FSBACKUP_MODE: ${FSBACKUP_MODE}"
+		exit 1
+esac
+
+if [ $STATUS -ne 0 ]; then
+	echo "${myname}: Non-zero exit: $STATUS"
+fi
+
+exit $STATUS