[#55] Fix das-node pipeline and create pipeline for python and c++ tests

.github/workflows/publish-pypi.yml

@@ -1,4 +1,5 @@
-name: Publish to PyPI
+---
+name: Publish DAS Node to PyPI
 
 on:
   workflow_dispatch:
@@ -32,19 +33,12 @@ jobs:
 
       - name: Update version
         run: sed -i "s/version=.*/version='${{ github.event.inputs.version }}',/" pyproject.toml
-
 
-      - name: Build (Docker Image)
-        run: ./scripts/docker_image_build.sh
+      - name: Build bazel
+        run: make cpp-build
 
-      - name: Build (Bazel)
-        run: ./scripts/build.sh
-
-      - name: Build (Docker Image Wheel)
-        run: ./scripts/docker_image_wheel.sh
-
-      - name: Build (Wheel)
-        run: ./scripts/wheel.sh
+      - name: Build wheel package
+        run: make wheeler-build
 
       - name: Install twine
         run: pip install twine

.github/workflows/run-tests.yml

@@ -0,0 +1,148 @@
+---
+name: Run DAS Node tests
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - edited
+      - synchronize
+
+  workflow_dispatch:
+
+jobs:
+  build-bazel:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+
+      - name: Build docker image
+        run: make cpp-image
+
+      - name: Login to Github Registry
+        if: success()
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push image to repository
+        run: |-
+          docker tag das-node-builder:latest ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }}
+          docker push ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }}
+
+      - name: Build bazel
+        run: make cpp-build
+
+      - name: Cache build and bazel_assets directories
+        uses: actions/cache@v3
+        with:
+          path: |
+            build
+            bazel_assets
+            docker/volumes/bazel_cache
+          key: ${{ runner.os }}-cpp-build-${{ hashFiles('**/CMakeLists.txt') }}-${{ hashFiles('**/*.cpp') }}
+          restore-keys: |
+            ${{ runner.os }}-cpp-build-${{ hashFiles('**/CMakeLists.txt') }}
+            ${{ runner.os }}-cpp-build
+
+  cpp-tests:
+    runs-on: ubuntu-22.04
+    needs: build-bazel
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+
+      - name: Login to Github Registry
+        if: success()
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull docker image from registry
+        run: |-
+          docker pull ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }}
+          docker tag ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }} das-node-builder:latest
+
+      - name: Run cpp tests
+        run: make cpp-test
+
+  build-wheeler:
+    runs-on: ubuntu-22.04
+    needs: build-bazel
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+
+      - name: Build docker image
+        run: make wheeler-image
+
+      - name: Cache build and bazel_assets directories
+        uses: actions/cache@v3
+        with:
+          path: |
+            build
+            bazel_assets
+          key: ${{ runner.os }}-cpp-build-${{ hashFiles('**/CMakeLists.txt') }}-${{ hashFiles('**/*.cpp') }}
+          restore-keys: |
+            ${{ runner.os }}-cpp-build-${{ hashFiles('**/CMakeLists.txt') }}
+            ${{ runner.os }}-cpp-build
+
+      - name: Build wheeler
+        run: make wheeler-build
+
+      - name: Upload wheeler artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheelers
+          path: dist/*
+
+  wheel-test:
+    runs-on: ubuntu-22.04
+    needs:
+      - build-bazel
+      - build-wheeler
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+
+      - name: Login to Github Registry
+        if: success()
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull docker image from registry
+        run: |-
+          docker pull ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }} 
+          docker tag ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }} das-node-builder:latest
+
+      - name: Cache build and bazel_assets directories
+        uses: actions/cache@v3
+        with:
+          path: |
+            build
+            bazel_assets
+          key: ${{ runner.os }}-cpp-build-${{ hashFiles('**/CMakeLists.txt') }}-${{ hashFiles('**/*.cpp') }}
+          restore-keys: |
+            ${{ runner.os }}-cpp-build-${{ hashFiles('**/CMakeLists.txt') }}
+            ${{ runner.os }}-cpp-build
+
+      - name: Download wheeler artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: wheelers
+          path: dist
+
+      - name: Install Python dependencies
+        run: pip install dist/*.whl
+
+      - name: Run python tests
+        run: make wheeler-test

.gitignore

@@ -26,6 +26,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+docker/volumes/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

Makefile

@@ -13,7 +13,7 @@ cpp-test:
 wheeler-image:
 	@./scripts/wheeler_image.sh
 
-wheeler-build:
+wheeler-build: 
 	@./scripts/wheeler_build.sh
 
 wheeler-test:
@@ -27,5 +27,4 @@ test-all: test-cpp test-python
 
 # Clean docker volumes and build directories
 clean:
-	@docker volume rm bazel_cache
-	@rm -rf src/bazel-* bazel_assets build dist
+	@rm -rf src/bazel-* bazel_assets build dist docker/volmes/bazel_cache

scripts/cpp_build.sh

@@ -5,7 +5,7 @@ CONTAINER_NAME="das-node-build"
 docker run \
   --rm \
   --name=$CONTAINER_NAME \
-  --mount type=volume,source=bazel_cache,target=/root/.cache/bazel \
+  --volume ./docker/volumes/bazel_cache:/root/.cache/bazel \
   --volume .:/opt/hyperon_das_node \
   --workdir /opt/hyperon_das_node/src \
   das-node-builder \

scripts/cpp_test.sh

@@ -5,7 +5,7 @@ CONTAINER_NAME="das-node-cpp-test"
 docker run \
   --rm \
   --name=$CONTAINER_NAME \
-  --mount type=volume,source=bazel_cache,target=/root/.cache/bazel \
+  --volume ./docker/volumes/bazel_cache:/root/.cache/bazel \
   --volume .:/opt/hyperon_das_node \
   --workdir /opt/hyperon_das_node/src \
   das-node-builder \

scripts/wheeler_test.sh

@@ -9,7 +9,7 @@ CONTAINER_NAME="das-node-wheeler-test"
 docker run \
   --rm \
   --name=$CONTAINER_NAME \
-  --mount type=volume,source=bazel_cache,target=/root/.cache/bazel \
+  --volume ./docker/volumes/bazel_cache:/root/.cache/bazel \
   --volume .:/opt/hyperon_das_node \
   --workdir /opt/hyperon_das_node \
   das-node-builder \