wip -- draft pg wrapper script

Makefile

@@ -46,9 +46,9 @@ help:
 	} \
 	{ lastLine = $$0 }' $(MAKEFILE_LIST)
 
-.PHONY: build-and-push-fuse-sidecar
-## build sidecar container used in Argo workflows
-build-and-push-fuse-sidecar: build-datamon-binaries
+.PHONY: build-and-push-fuse-sidecar-img
+## build FUSE sidecar container used in Argo workflows
+build-and-push-fuse-sidecar-img:
 	@echo 'building fuse sidecar container'
 	docker build \
 		--progress plain \
@@ -60,6 +60,27 @@ build-and-push-fuse-sidecar: build-datamon-binaries
 		.
 	docker push gcr.io/onec-co/datamon-fuse-sidecar
 
+.PHONY: build-and-push-fuse-sidecar
+## build FUSE sidecar container used in Argo workflows
+build-and-push-fuse-sidecar: build-datamon-binaries build-and-push-fuse-sidecar-img
+
+.PHONY: build-and-push-pg-sidecar-img
+## build postgres sidecar container used in Argo workflows
+build-and-push-pg-sidecar-img:
+	@echo 'building pg sidecar container'
+	docker build \
+		--progress plain \
+		-t gcr.io/onec-co/datamon-pg-sidecar \
+		-t gcr.io/onec-co/datamon-pg-sidecar:${GITHUB_USER}-$$(date '+%Y%m%d') \
+		-t gcr.io/onec-co/datamon-pg-sidecar:$(subst /,_,$(GIT_BRANCH)) \
+		--ssh default \
+		-f sidecar-pg.Dockerfile \
+		.
+	docker push gcr.io/onec-co/datamon-pg-sidecar
+
+.PHONY: build-and-push-pg-sidecar
+## build postgres sidecar container used in Argo workflows
+build-and-push-pg-sidecar: build-datamon-binaries build-and-push-pg-sidecar-img
 
 .PHONY: build-and-push-datamover
 ## build sidecar container used in Argo workflows
@@ -245,7 +266,7 @@ fuse-demo-ro: fuse-demo-build-shell fuse-demo-build-sidecar
 .PHONY: fuse-demo-coord-build-app
 ## build shell container used in fuse demo
 fuse-demo-coord-build-app:
-	@echo 'building fuse demo container'
+	@echo 'building fuse demo application container'
 	docker build \
 		--progress plain \
 		-t gcr.io/onec-co/datamon-fuse-demo-coord-app \
@@ -257,7 +278,7 @@ fuse-demo-coord-build-app:
 .PHONY: fuse-demo-coord-build-datamon
 ## build shell container used in fuse demo
 fuse-demo-coord-build-datamon:
-	@echo 'building fuse demo container'
+	@echo 'building fuse demo sidecar container'
 	docker build \
 		--progress plain \
 		-t gcr.io/onec-co/datamon-fuse-demo-coord-datamon \
@@ -266,6 +287,18 @@ fuse-demo-coord-build-datamon:
 		.
 	docker push gcr.io/onec-co/datamon-fuse-demo-coord-datamon
 
+.PHONY: pg-demo-coord-build-app
+## build shell container used in fuse demo
+pg-demo-coord-build-app:
+	@echo 'building pg demo application container'
+	docker build \
+		--progress plain \
+		-t gcr.io/onec-co/datamon-pg-demo-coord-app \
+		--ssh default \
+		-f ./hack/fuse-demo/coord-app-pg.Dockerfile \
+		.
+	docker push gcr.io/onec-co/datamon-pg-demo-coord-app
+
 .PHONY: profile-metrics
 ## Build the metrics collection binary and write output
 profile-metrics:

hack/fuse-demo/coord-app-pg.Dockerfile

@@ -0,0 +1,60 @@
+FROM debian
+
+RUN apt-get update &&\
+    curl -sL https://deb.nodesource.com/setup_10.x  | bash &&\
+    apt-get install -y \
+        curl \
+        postgresql \
+        ca-certificates \
+        gnupg \
+        zsh \
+        vim \
+        &&\
+    apt-get autoremove -yqq &&\
+    apt-get clean -y &&\
+    apt-get autoclean -yqq &&\
+    rm -rf \
+        /tmp/* \
+        /var/tmp/* \
+        /var/lib/apt/lists/* \
+        /usr/share/doc/* \
+        /usr/share/locale/* \
+        /var/cache/debconf/*-old
+
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+
+## BEGIN tini
+
+ENV TINI_VERSION v0.18.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-amd64 /tmp/tini-static-amd64
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-amd64.asc /tmp/tini-static-amd64.asc
+
+# omitting gpg verification during development/demo
+# RUN for key in \
+#       595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \
+#     ; do \
+#       gpg --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" || \
+#       gpg --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \
+#       gpg --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" ; \
+#     done
+
+# RUN gpg --verify /tmp/tini-static-amd64.asc
+
+RUN install -m 0755 /tmp/tini-static-amd64 /bin/tini
+
+## END tini
+
+RUN useradd -u 1020 -ms /bin/bash developer
+RUN groupadd -g 2000 developers
+RUN usermod -g developers developer
+RUN chown -R developer:developers /bin/tini
+
+ADD hack/fuse-demo/mock_application_pg.sh .
+RUN chmod a+x mock_application_pg.sh
+
+USER developer
+
+RUN touch ~/.zshrc
+
+ENTRYPOINT [ "/tmp/coord/.scripts/wrap_application.sh"]
+CMD [ "./mock_application.sh"]

hack/fuse-demo/create_coord_pg_pod.sh

@@ -0,0 +1,46 @@
+#! /bin/zsh
+
+SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+
+proj_root_dir="$(dirname "$(dirname "$SCRIPT_DIR")")"
+
+pull_policy=Always
+
+while getopts o opt; do
+    case $opt in
+        (o)
+            # local deploy
+            pull_policy=IfNotPresent
+            ;;
+        (\?)
+            print Bad option, aborting.
+            exit 1
+            ;;
+    esac
+done
+(( OPTIND > 1 )) && shift $(( OPTIND - 1 ))
+
+if [[ -z $GOOGLE_APPLICATION_CREDENTIALS ]]; then
+	echo 'GOOGLE_APPLICATION_CREDENTIALS env variable not set' 1>&2
+	exit 1
+fi
+
+if kubectl get secret google-application-credentials &> /dev/null; then
+	kubectl delete secret google-application-credentials
+fi
+
+# https://cloud.google.com/kubernetes-engine/docs/tutorials/authenticating-to-cloud-platform#step_4_import_credentials_as_a_secret
+kubectl create secret generic \
+	google-application-credentials \
+	--from-file=google-application-credentials.json=$GOOGLE_APPLICATION_CREDENTIALS
+
+RES_DEF="$proj_root_dir"/hack/k8s/gen/example-coord-pg.yaml
+
+PULL_POLICY=$pull_policy \
+	"$proj_root_dir"/hack/envexpand "$proj_root_dir"/hack/k8s/example-coord-pg.template.yaml > "$RES_DEF"
+
+if kubectl get deployment datamon-coord-pg-demo &> /dev/null; then
+	kubectl delete -f "$RES_DEF"
+fi
+
+kubectl create -f "$RES_DEF"

hack/fuse-demo/mock_application_pg.sh

@@ -0,0 +1,49 @@
+#! /bin/zsh
+
+setopt ERR_EXIT
+setopt PIPE_FAIL
+
+### data-science application placeholder/mock
+# this program is a stand-in for a data-science application
+#
+# todo: sandbox poc python (sqla/pandas) use rather than psql
+
+MY_PG_PORT="$1"
+
+usage() {
+    print -- 'usage: ./mock_application_pg.sh pg-port' 1>&2
+    exit 1
+}
+
+if [ -z "$MY_PG_PORT" ]; then
+    usage
+fi
+
+# convention from sidecar is to create provide initial postgres su
+PG_SU=postgres
+
+# other pg users and setup are the responsibility of the application
+PG_U=testpguser
+PG_DB=testdb
+
+# aside: postgres defaults to UNIX (filesystem) socket at /var/run/postgresql/*,
+#   not IP (network) socket, so it's the client's responsibility to ensure that
+#   the conn is opened at the network location
+
+print -- "CREATE ROLE ${PG_U} WITH LOGIN CREATEDB;
+CREATE DATABASE ${PG_DB} WITH OWNER ${PG_U};" | \
+    psql -h localhost -p $MY_PG_PORT -U $PG_SU
+
+
+run_sql() {
+    print -- "$1" | psql -h localhost -p $MY_PG_PORT -U ${PG_U} ${PG_DB}
+}
+
+run_sql 'CREATE TABLE tabla_e (
+  id serial PRIMARY KEY,
+  an_idx integer
+);'
+
+for idx in $(seq 1 2 9); do
+    run_sql "INSERT INTO tabla_e (an_idx) VALUES (${idx}) RETURNING id;"
+done

hack/fuse-demo/run_coord_pg_shell.sh

@@ -0,0 +1,30 @@
+#! /bin/zsh
+
+container_name=demo-app
+
+while getopts s opt; do
+    case $opt in
+        (s)
+            container_name='datamon-sidecar'
+            ;;
+        (\?)
+            print Bad option, aborting.
+            exit 1
+            ;;
+    esac
+done
+(( OPTIND > 1 )) && shift $(( OPTIND - 1 ))
+
+STARTUP_POLL_INTERVAL=1
+typeset pod_name
+
+print -- "waiting on pod start"
+
+while [[ -z $pod_name ]]; do
+    sleep "$STARTUP_POLL_INTERVAL"
+    pod_name=$(kubectl get pods -l app=datamon-coord-pg-demo | grep Running | sed 's/ .*//')
+done
+
+kubectl exec -it "$pod_name" \
+        -c "$container_name" \
+        -- "/bin/zsh"

hack/fuse-demo/wrap_application.sh

@@ -15,17 +15,31 @@ POLL_INTERVAL=1 # sec
 
 SLEEP_INSTEAD_OF_EXIT=
 
-while getopts sc: opt; do
+SC_FUSE=
+SC_PG=
+
+while getopts sc:b: opt; do
     case $opt in
         (s)
             SLEEP_INSTEAD_OF_EXIT=true
             ;;
         (c)
             COORD_POINT="$OPTARG"
             ;;
+        (b)
+            battery_type="$OPTARG"
+            if [ "$battery_type" = "fuse" ]; then
+                SC_FUSE=true
+            elif [ "$battery_type" = "postgres" ]; then
+                SC_PG=true
+            else
+                echo "unkown battery type $battery_type" 1>&2
+                exit 1
+            fi
+            ;;
         (\?)
             echo "Bad option, aborting."
-            return 1
+            exit 1
             ;;
     esac
 done
@@ -71,9 +85,16 @@ emit_event() {
 ### application wrapper
 
 ## the following waits on datamon to make a FUSE mount available
-await_event \
-    'mountdone' \
-    'waiting on datamon mount (app wrap)'
+if [ -n "$SC_FUSE" ]; then
+    await_event \
+        'mountdone' \
+        'waiting on datamon mount (app wrap)'
+fi
+if [ -n "$SC_PG" ]; then
+    await_event \
+        'dbstarted' \
+        'waiting on db start (app wrap)'
+fi
 
 ## once data is available, the data-science application is started
 echo "mount done, executing mock application, '" "$@" "'"
@@ -88,15 +109,28 @@ fi
 echo "mock application done"
 
 ## after the application writes its output, notify the sidecar to start uploading it
-
-emit_event \
-    'initupload' \
-    'dispatching init upload event'
+if [ -n "$SC_FUSE" ]; then
+    emit_event \
+        'initupload' \
+        'dispatching init upload event'
+fi
+if [ -n "$SC_PG" ]; then
+    emit_event \
+        'initdbupload' \
+        'dispatching init db upload event'
+fi
 
 ## block until the sidecar finishes uploading
-await_event \
-    'uploaddone' \
-    'waiting on upload'
+if [ -n "$SC_FUSE" ]; then
+    await_event \
+        'uploaddone' \
+        'waiting on upload'
+fi
+if [ -n "$SC_PG" ]; then
+    await_event \
+        'dbuploaddone' \
+        'waiting on db upload'
+fi
 
 ## COORDINATION ENDS with this container exiting
 echo "recved upload done event, exiting"