Merge pull request #19 from swisscom/redis

add Redis support

.circleci/config.yml

@@ -25,6 +25,8 @@ jobs:
         MONGO_INITDB_ROOT_USERNAME: mongoadmin
         MONGO_INITDB_ROOT_PASSWORD: super-secret
 
+    - image: circleci/redis:5.0
+
     branches:
       ignore:
       - develop
@@ -86,3 +88,9 @@ jobs:
           name: install Postgres client
           command: sudo apt install -y postgresql-client
       - run: make postgres-test
+
+      # redis integration tests
+      - run:
+          name: install Redis client
+          command: sudo apt install -y redis-tools
+      - run: make redis-test

.dockerenv

@@ -6,4 +6,4 @@ BACKMAN_CONFIG={"log_level":"debug","logging_timestamp":true,"username":"john","
 CF_INSTANCE_GUID=e5d1bf0f-7b51-4ff8-7621-9f20
 VCAP_APPLICATION={"application_id":"fa05c1a9-0fc1-4fbd-bae1-139850dec7a3","application_name":"backman","application_uris":["appcloud-backman-app.example.com","backman-app.example.com","backman.example.com"],"application_version":"fb8fbcc6-8d58-479e-bcc7-3b4ce5a7f0ca","cf_api":"https://api.example.com","limits":{"disk":1024,"fds":16384,"mem":256},"name":"backman","organization_id":"c0134bad-97a9-468d-ab9d-e97547e3aed5","organization_name":"my-org","space_id":"06450c72-4669-4dc6-8096-45f9777db68a","space_name":"my-space","uris":["appcloud-backman-app.example.com","backman-app.example.com","backman.example.com"],"version":"fb8fbcc6-8d58-479e-bcc7-3b4ce5a7f0ca"}
 
-VCAP_SERVICES={"elasticsearch":[{"binding_name":null,"credentials":{"full_access_password":"yolo","full_access_username":"yolo","host":"https://yolo.elasticsearch.lyra-836.appcloud.swisscom.com","kibana_read_only_password":"yolo","kibana_read_only_username":"yolo","kibana_system_password":"yolo","kibana_system_username":"yolo","logstash_system_password":"yolo","logstash_system_username":"yolo"},"instance_name":"my-elasticsearch","label":"elasticsearch","name":"my-elasticsearch","plan":"medium","provider":null,"syslog_drain_url":null,"tags":["elasticsearch"]}],"mongodb":[{"credentials":{"uri":"mongodb://mongoadmin:[email protected]:27017","host":"127.0.0.1","port":27017,"name":"my_mongodb","database":"my-db","username":"mongoadmin","password":"super-secret"},"syslog_drain_url":null,"volume_mounts":[],"label":"mongodb","provider":null,"plan":"usage","name":"my_mongodb","tags":["mangodb","mongodb"]},{"credentials":{"uri":"mongodb://mongoadmin:[email protected]:27017","host":"127.0.0.1","port":27017,"name":"my_mongodb","database":"my-db","username":"mongoadmin","password":"super-secret"},"syslog_drain_url":null,"volume_mounts":[],"label":"mongodbent","provider":null,"plan":"usage","name":"some other mangodb!","tags":["mangodb","mongodb"]}],"mysql":[{"credentials":{"host":"127.0.0.1","port":3306,"name":"my_mysql_db","database":"mysql","username":"root","password":"my-secret-pw"},"syslog_drain_url":null,"volume_mounts":[],"label":"mysql","provider":null,"plan":"usage","name":"my_mysql_db","tags":["mysql","mariadb"]}],"postgres":[{"credentials":{"host":"127.0.0.1","hostname":"127.0.0.1","port":5432,"name":"my_postgres_db","database":"my_postgres_db","username":"dev-user","password":"dev-secret","database_uri":"postgres://dev-user:[email protected]:5432/my_postgres_db?sslmode=disable","uri":"postgres://dev-user:[email protected]:5432/my_postgres_db?sslmode=disable","jdbcUrl":"jdbc:postgres://127.0.0.1:5432/my_postgres_db?sslmode=disable"},"syslog_drain_url":null,"volume_mounts":[],"label":"postgres","provider":null,"plan":"usage","name":"my_postgres_db","tags":[]},{"credentials":{"host":"127.0.0.1","hostname":"127.0.0.1","port":5432,"name":"other_postgres_db","database":"other_postgres_db","username":"dev-user","password":"dev-secret","database_uri":"postgres://dev-user:[email protected]:5432/other_postgres_db?sslmode=disable","uri":"postgres://dev-user:[email protected]:5432/other_postgres_db?sslmode=disable","jdbcUrl":"jdbc:postgres://127.0.0.1:5432/other_postgres_db?sslmode=disable"},"syslog_drain_url":null,"volume_mounts":[],"label":"postgres","provider":null,"plan":"usage","name":"other_postgres_db","tags":[]}],"dynstrg":[{"binding_name":null,"credentials":{"accessHost":"127.0.0.1:9000","accessKey":"6d611e2d-330b-4e52-a27c-59064d6e8a62","namespace":"deadbeef","sharedSecret":"eW9sbywgeW91IGhhdmUganVzdCBiZWVuIHRyb2xsZWQh"},"instance_name":"my-database-backups","syslog_drain_url":null,"volume_mounts":[],"label":"dynstrg","provider":null,"plan":"usage","name":"my-database-backups","tags":[]}]}
+VCAP_SERVICES={"elasticsearch":[{"binding_name":null,"credentials":{"full_access_password":"yolo","full_access_username":"yolo","host":"https://yolo.elasticsearch.lyra-836.appcloud.swisscom.com","kibana_read_only_password":"yolo","kibana_read_only_username":"yolo","kibana_system_password":"yolo","kibana_system_username":"yolo","logstash_system_password":"yolo","logstash_system_username":"yolo"},"instance_name":"my-elasticsearch","label":"elasticsearch","name":"my-elasticsearch","plan":"medium","provider":null,"syslog_drain_url":null,"tags":["elasticsearch"]}],"redis-2":[{"binding_name":null,"credentials":{"host":"127.0.0.1","master_port":6379,"password":"very-secret","port":6379},"instance_name":"my-redis","label":"redis-2","name":"my-redis","plan":"small","provider":null,"syslog_drain_url":null,"tags":["redis"],"volume_mounts":[]}],"mongodb":[{"credentials":{"uri":"mongodb://mongoadmin:[email protected]:27017","host":"127.0.0.1","port":27017,"name":"my_mongodb","database":"my-db","username":"mongoadmin","password":"super-secret"},"syslog_drain_url":null,"volume_mounts":[],"label":"mongodb","provider":null,"plan":"usage","name":"my_mongodb","tags":["mangodb","mongodb"]},{"credentials":{"uri":"mongodb://mongoadmin:[email protected]:27017","host":"127.0.0.1","port":27017,"name":"my_mongodb","database":"my-db","username":"mongoadmin","password":"super-secret"},"syslog_drain_url":null,"volume_mounts":[],"label":"mongodbent","provider":null,"plan":"usage","name":"some other mangodb!","tags":["mangodb","mongodb"]}],"mysql":[{"credentials":{"host":"127.0.0.1","port":3306,"name":"my_mysql_db","database":"mysql","username":"root","password":"my-secret-pw"},"syslog_drain_url":null,"volume_mounts":[],"label":"mysql","provider":null,"plan":"usage","name":"my_mysql_db","tags":["mysql","mariadb"]}],"postgres":[{"credentials":{"host":"127.0.0.1","hostname":"127.0.0.1","port":5432,"name":"my_postgres_db","database":"my_postgres_db","username":"dev-user","password":"dev-secret","database_uri":"postgres://dev-user:[email protected]:5432/my_postgres_db?sslmode=disable","uri":"postgres://dev-user:[email protected]:5432/my_postgres_db?sslmode=disable","jdbcUrl":"jdbc:postgres://127.0.0.1:5432/my_postgres_db?sslmode=disable"},"syslog_drain_url":null,"volume_mounts":[],"label":"postgres","provider":null,"plan":"usage","name":"my_postgres_db","tags":[]},{"credentials":{"host":"127.0.0.1","hostname":"127.0.0.1","port":5432,"name":"other_postgres_db","database":"other_postgres_db","username":"dev-user","password":"dev-secret","database_uri":"postgres://dev-user:[email protected]:5432/other_postgres_db?sslmode=disable","uri":"postgres://dev-user:[email protected]:5432/other_postgres_db?sslmode=disable","jdbcUrl":"jdbc:postgres://127.0.0.1:5432/other_postgres_db?sslmode=disable"},"syslog_drain_url":null,"volume_mounts":[],"label":"postgres","provider":null,"plan":"usage","name":"other_postgres_db","tags":[]}],"dynstrg":[{"binding_name":null,"credentials":{"accessHost":"127.0.0.1:9000","accessKey":"6d611e2d-330b-4e52-a27c-59064d6e8a62","namespace":"deadbeef","sharedSecret":"eW9sbywgeW91IGhhdmUganVzdCBiZWVuIHRyb2xsZWQh"},"instance_name":"my-database-backups","syslog_drain_url":null,"volume_mounts":[],"label":"dynstrg","provider":null,"plan":"usage","name":"my-database-backups","tags":[]}]}

.env

@@ -184,6 +184,25 @@ export VCAP_SERVICES='{
   "name": "other_postgres_db",
   "tags": []
 }],
+"redis-2": [{
+  "binding_name": null,
+  "credentials": {
+   "host": "127.0.0.1",
+   "master_port": 6379,
+   "password": "very-secret",
+   "port": 6379
+  },
+  "instance_name": "my-redis",
+  "label": "redis-2",
+  "name": "my-redis",
+  "plan": "small",
+  "provider": null,
+  "syslog_drain_url": null,
+  "tags": [
+   "redis"
+  ],
+  "volume_mounts": []
+}],
 "dynstrg":[{
   "binding_name": null,
   "credentials": {

.gitignore

@@ -40,3 +40,4 @@ appcloud-backup-app
 appcloud-backman-app
 backman-app
 backman
+/tmp

Dockerfile

@@ -15,7 +15,7 @@ RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
   echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" > /etc/apt/sources.list.d/mongodb-org-4.2.list
 RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
 RUN apt-get -y $package_args update && \
-  apt-get -y $package_args install mysql-client postgresql-client-11 mongodb-org-tools=4.2.0 nodejs openssh-server bash vim && \
+  apt-get -y $package_args install mysql-client postgresql-client-11 mongodb-org-tools=4.2.0 redis-tools nodejs openssh-server bash vim && \
   apt-get clean && \
   find /usr/share/doc/*/* ! -name copyright | xargs rm -rf && \
   rm -rf \

Makefile

@@ -1,4 +1,4 @@
-.PHONY: run gin build test docker-build docker-push docker-run docker-exec swagger elasticsearch elasticsearch-network elasticsearch-stop elasticsearch-start elasticsearch-data minio minio-stop minio-start mysql mysql-network mysql-stop mysql-start mysql-client mysql-test postgres postgres-network postgres-stop postgres-start postgres-client postgres-test mongodb mongodb-network mongodb-stop mongodb-start mongodb-client mongodb-test cleanup
+.PHONY: run gin build test docker-build docker-push docker-run docker-exec swagger elasticsearch elasticsearch-network elasticsearch-stop elasticsearch-start elasticsearch-data minio minio-stop minio-start mysql mysql-network mysql-stop mysql-start mysql-client mysql-test postgres postgres-network postgres-stop postgres-start postgres-client postgres-test mongodb mongodb-network mongodb-stop mongodb-start mongodb-client mongodb-test redis redis-network redis-stop redis-start redis-client redis-test cleanup
 SHELL := /bin/bash
 
 all: run
@@ -152,5 +152,30 @@ mongodb-client:
 mongodb-test: build
 	scripts/mongodb.sh
 
+redis: redis-network redis-stop redis-start
+	docker logs redis -f
+
+redis-network:
+	docker network create redis-network --driver bridge || true
+
+redis-stop:
+	docker rm -f redis || true
+
+redis-start:
+	docker run --name redis \
+		--network redis-network \
+		-p 6379:6379 \
+		-d redis \
+			redis-server --requirepass 'very-secret'
+
+redis-client:
+	docker run -it --rm \
+		--network redis-network \
+		--name redis-cli redis redis-cli \
+			-h redis -a 'very-secret'
+
+redis-test: build
+	scripts/redis.sh
+
 cleanup:
 	docker system prune --volumes -a

README.md

@@ -12,6 +12,7 @@ a backup-manager app for [Cloud Foundry](https://www.cloudfoundry.org/)
 - PostgreSQL
 - MongoDB
 - Elasticsearch
+- Redis
 
 ## Usage
 

apt.yml

@@ -9,3 +9,4 @@ packages:
 - mysql-client
 - postgresql-client-11
 - mongodb-org-tools=4.2.0
+- redis-tools

manifest.yml

@@ -19,10 +19,11 @@ applications:
   # - my_postgres_db
   # - my_mongodb
   # - my_elasticsearch
+  # - my_redis
 
   # ### push either as docker image
   docker:
-    image: jamesclonk/backman:1.9.0 # choose version from https://hub.docker.com/r/jamesclonk/backman/tags, or 'latest'
+    image: jamesclonk/backman:1.10.0 # choose version from https://hub.docker.com/r/jamesclonk/backman/tags, or 'latest'
   # ### or as buildpack/src
   # buildpacks:
   # - https://github.com/cloudfoundry/apt-buildpack

public/service.html

@@ -100,10 +100,10 @@ <h3 class="table__title">Backups</h3>
 										<i class="icon icon-024-download-cloud" aria-hidden="true"></i>
 										<span class="toolbar__label"><small>Download</small></span>
 									</button>
-									<button class="toolbar__item item--show padding-v-0 restore-button" aria-label="Restore" v-on:click="showRestoreDialog(file.Filename)">
+									{{ if not (eq (ServiceType .Service.Label).String "Redis") }}<button class="toolbar__item item--show padding-v-0 restore-button" aria-label="Restore" v-on:click="showRestoreDialog(file.Filename)">
 										<i class="icon icon-delivery-2" aria-hidden="true"></i>
 										<span class="toolbar__label"><small>Restore</small></span>
-									</button>
+									</button>{{ end }}
 									<button class="toolbar__item item--show padding-v-0 delete-button" aria-label="Delete" v-on:click="showDeleteDialog(file.Filename)">
 										<i class="icon icon-008-bin" aria-hidden="true"></i>
 										<span class="toolbar__label"><small>Delete</small></span>

scripts/redis.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+
+# fail on error
+set -e
+
+# =============================================================================================
+if [[ "$(basename $PWD)" == "scripts" ]]; then
+	cd ..
+fi
+echo $PWD
+
+# =============================================================================================
+source .env
+
+# =============================================================================================
+retry() {
+    local -r -i max_attempts="$1"; shift
+    local -r cmd="$@"
+    local -i attempt_num=1
+
+    until $cmd
+    do
+        if (( attempt_num == max_attempts ))
+        then
+            echo "Attempt $attempt_num failed and there are no more attempts left!"
+            return 1
+        else
+            echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+            sleep $(( attempt_num++ ))
+        fi
+    done
+}
+
+# =============================================================================================
+echo "waiting on redis ..."
+retry 10 redis-cli -h 127.0.0.1 ping
+echo "redis is up!"
+
+echo "configuring redis auth password ..."
+redis-cli -h 127.0.0.1 CONFIG SET requirepass "very-secret" || true
+
+# =============================================================================================
+echo "testing redis integration ..."
+
+sleep 5
+# starting backman
+killall backman || true
+./backman 2>&1 &
+sleep 5
+
+set -x
+if [ $(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:9990) != "401" ]; then
+	echo "Should be Unauthorized"
+	exit 1
+fi
+
+if [ $(curl -s -o /dev/null -w "%{http_code}" http://john:[email protected]:9990) != "200" ]; then
+	echo "Should be authorized"
+	exit 1
+fi
+
+if [ $(curl -s -o /dev/null -w "%{http_code}" http://john:[email protected]:9990/api/v1/state/redis-2/my-redis) != "200" ]; then
+	echo "Failed to query state"
+	exit 1
+fi
+curl -s http://john:[email protected]:9990/api/v1/state/redis-2/my-redis | grep '"Status":"idle"'
+
+# write to redis
+redis-cli -h 127.0.0.1 -a 'very-secret' SET blubb 123
+redis-cli -h 127.0.0.1 -a 'very-secret' SET blabb hello
+sleep 2
+
+# trigger new backup
+curl -X POST http://john:[email protected]:9990/api/v1/backup/redis-2/my-redis
+curl -s http://john:[email protected]:9990/api/v1/state/redis-2/my-redis | grep '"Operation":"backup"' | grep '"Status":"running"'
+sleep 15
+curl -s http://john:[email protected]:9990/api/v1/state/redis-2/my-redis | grep '"Operation":"backup"' | grep '"Status":"success"'
+
+# read from redis
+redis-cli -h 127.0.0.1 -a 'very-secret' GET blubb | grep 123
+redis-cli -h 127.0.0.1 -a 'very-secret' GET blabb | grep hello
+
+# download backup and check for completeness
+FILENAME=$(curl -s http://john:[email protected]:9990/api/v1/backup/redis-2/my-redis | jq -r .Files[0].Filename)
+curl -s http://john:[email protected]:9990/api/v1/backup/redis-2/my-redis/${FILENAME}/download | zgrep 'blubb'
+
+# delete from redis
+redis-cli -h 127.0.0.1 -a 'very-secret' DEL blubb
+redis-cli -h 127.0.0.1 -a 'very-secret' SET blibb howdy
+sleep 2
+redis-cli -h 127.0.0.1 -a 'very-secret' GET blubb | grep -v 123
+redis-cli -h 127.0.0.1 -a 'very-secret' GET blibb | grep howdy
+
+## restore is unsupported for redis
+# # trigger restore
+# FILENAME=$(curl -s http://john:[email protected]:9990/api/v1/backup/redis-2/my-redis | jq -r .Files[0].Filename)
+# curl -X POST http://john:[email protected]:9990/api/v1/restore/redis-2/my-redis/${FILENAME}
+# curl -s http://john:[email protected]:9990/api/v1/state/redis-2/my-redis | grep '"Operation":"restore"' | grep '"Status":"running"'
+# sleep 15
+# curl -s http://john:[email protected]:9990/api/v1/state/redis-2/my-redis | grep '"Operation":"restore"' | grep '"Status":"success"'
+
+# # read from redis
+# redis-cli -h 127.0.0.1 -a 'very-secret' GET blibb | grep -v howdy
+# redis-cli -h 127.0.0.1 -a 'very-secret' GET blubb | grep 123
+# redis-cli -h 127.0.0.1 -a 'very-secret' GET blabb | grep hello
+
+# delete backup
+curl -X DELETE http://john:[email protected]:9990/api/v1/backup/redis-2/my-redis/${FILENAME}
+sleep 10
+curl -s http://john:[email protected]:9990/api/v1/backup/redis-2/my-redis | grep -v 'Filename'

service/backup.go

@@ -17,6 +17,7 @@ import (
 	"github.com/swisscom/backman/service/mongodb"
 	"github.com/swisscom/backman/service/mysql"
 	"github.com/swisscom/backman/service/postgres"
+	"github.com/swisscom/backman/service/redis"
 	"github.com/swisscom/backman/service/util"
 )
 
@@ -68,6 +69,8 @@ func (s *Service) Backup(service util.Service) error {
 	switch service.Type() {
 	case util.MongoDB:
 		err = mongodb.Backup(ctx, s.S3, service, envService, filename)
+	case util.Redis:
+		err = redis.Backup(ctx, s.S3, service, envService, filename)
 	case util.MySQL:
 		err = mysql.Backup(ctx, s.S3, service, envService, filename)
 	case util.Postgres:

service/elasticsearch/backup.go

@@ -94,6 +94,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 		err = s3.UploadWithContext(uploadCtx, objectPath, pr, -1)
 		if err != nil {
 			log.Errorf("could not upload service backup [%s] to S3: %v", service.Name, err)
+			state.BackupFailure(service)
 		}
 	}()
 	time.Sleep(2 * time.Second) // wait for upload goroutine to be ready

service/mongodb/backup.go

@@ -75,6 +75,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 		err = s3.UploadWithContext(uploadCtx, objectPath, pr, -1)
 		if err != nil {
 			log.Errorf("could not upload service backup [%s] to S3: %v", service.Name, err)
+			state.BackupFailure(service)
 		}
 	}()
 	time.Sleep(2 * time.Second) // wait for upload goroutine to be ready

service/mysql/backup.go

@@ -97,6 +97,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 		err = s3.UploadWithContext(uploadCtx, objectPath, pr, -1)
 		if err != nil {
 			log.Errorf("could not upload service backup [%s] to S3: %v", service.Name, err)
+			state.BackupFailure(service)
 		}
 	}()
 	time.Sleep(2 * time.Second) // wait for upload goroutine to be ready

service/postgres/backup.go

@@ -93,6 +93,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 		err = s3.UploadWithContext(uploadCtx, objectPath, pr, -1)
 		if err != nil {
 			log.Errorf("could not upload service backup [%s] to S3: %v", service.Name, err)
+			state.BackupFailure(service)
 		}
 	}()
 	time.Sleep(2 * time.Second) // wait for upload goroutine to be ready