[#55] Fix das-node pipeline and create pipeline for python and c++ te…

…sts (#56) * das-node-55: Fix publish steps * das-node-55: Add python and c++ tests * das-node-55: build bazel in run-tests pipeline * das-node-55: Update github registry * das-node-55: Fix github repository image * das-node-55: Update pipeline * das-node-55: Get cache wheeler * das-node-55: Fix wheel dist * das-node-55: use bind volume for bazel cache * das-node-55: cache * das-node-55: Update run-tests pipeline * das-node-55: Give permissions * das-node-55: Set permissions for cache * das-node-55: Update volume owner
singnet · Nov 1, 2024 · 896268b · 896268b
1 parent ada3421
commit 896268b
Show file tree

Hide file tree

Showing 7 changed files with 185 additions and 17 deletions.
diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml
@@ -1,4 +1,5 @@
-name: Publish to PyPI
+---
+name: Publish DAS Node to PyPI
 
 on:
   workflow_dispatch:
@@ -32,19 +33,12 @@ jobs:
 
       - name: Update version
         run: sed -i "s/version=.*/version='${{ github.event.inputs.version }}',/" pyproject.toml
-
 
-      - name: Build (Docker Image)
-        run: ./scripts/docker_image_build.sh
+      - name: Build bazel
+        run: make cpp-build
 
-      - name: Build (Bazel)
-        run: ./scripts/build.sh
-
-      - name: Build (Docker Image Wheel)
-        run: ./scripts/docker_image_wheel.sh
-
-      - name: Build (Wheel)
-        run: ./scripts/wheel.sh
+      - name: Build wheel package
+        run: make wheeler-build
 
       - name: Install twine
         run: pip install twine

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -0,0 +1,174 @@
+---
+name: Run DAS Node tests
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - edited
+      - synchronize
+
+  workflow_dispatch:
+
+jobs:
+  build-bazel:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+
+      - name: Build docker image
+        run: make cpp-image
+
+      - name: Login to Github Registry
+        if: success()
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push image to repository
+        run: |-
+          docker tag das-node-builder:latest ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }}
+          docker push ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }}
+
+      - name: Build bazel
+        run: make cpp-build
+
+      - name: Set permissions for cached directories
+        run: |
+          sudo chmod -R 777 ./bazel_assets
+          sudo chmod -R 777 ./docker/volumes/
+
+      - name: Cache build and bazel_assets directories
+        uses: actions/cache@v3
+        with:
+          path: |
+            ./bazel_assets
+            ./docker/volumes/bazel_cache
+          key: ${{ runner.os }}-cpp-build-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-cpp-build-${{ github.sha }}
+
+  cpp-tests:
+    runs-on: ubuntu-22.04
+    needs: build-bazel
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+
+      - name: Login to Github Registry
+        if: success()
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull docker image from registry
+        run: |-
+          docker pull ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }}
+          docker tag ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }} das-node-builder:latest
+
+      - name: Cache build and bazel_assets directories
+        uses: actions/cache@v3
+        with:
+          path: |
+            ./bazel_assets
+            ./docker/volumes/bazel_cache
+          key: ${{ runner.os }}-cpp-build-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-cpp-build-${{ github.sha }}
+
+      - name: Set permissions for cached directories
+        run: |
+          sudo chown root:root -R ./bazel_assets
+          sudo chown root:root -R ./docker/volumes/
+
+      - name: Run cpp tests
+        run: make cpp-test
+
+  build-wheeler:
+    runs-on: ubuntu-22.04
+    needs: build-bazel
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+
+      - name: Cache build and bazel_assets directories
+        uses: actions/cache@v3
+        with:
+          path: |
+            ./bazel_assets
+            ./docker/volumes/bazel_cache
+          key: ${{ runner.os }}-cpp-build-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-cpp-build-${{ github.sha }}
+
+      - name: Set permissions for cached directories
+        run: |
+          sudo chmod -R 777 ./bazel_assets
+          sudo chmod -R 777 ./docker/volumes/
+
+      - name: Build docker image
+        run: make wheeler-image
+
+      - name: Build wheeler
+        run: make wheeler-build
+
+      - name: Upload wheeler artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheelers
+          path: dist/*
+
+  wheel-test:
+    runs-on: ubuntu-22.04
+    needs:
+      - build-bazel
+      - build-wheeler
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+
+      - name: Login to Github Registry
+        if: success()
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull docker image from registry
+        run: |-
+          docker pull ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }} 
+          docker tag ghcr.io/${{ github.repository_owner }}/das-node-builder:${{ github.sha }} das-node-builder:latest
+
+      - name: Cache build and bazel_assets directories
+        uses: actions/cache@v3
+        with:
+          path: |
+            ./bazel_assets
+            ./docker/volumes/bazel_cache
+          key: ${{ runner.os }}-cpp-build-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-cpp-build-${{ github.sha }}
+
+      - name: Set permissions for cached directories
+        run: |
+          sudo chmod -R 777 ./bazel_assets
+          sudo chmod -R 777 ./docker/volumes/
+
+      - name: Download wheeler artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: wheelers
+          path: dist
+
+      - name: Install Python dependencies
+        run: pip install dist/*.whl
+
+      - name: Run python tests
+        run: make wheeler-test
diff --git a/.gitignore b/.gitignore
@@ -26,6 +26,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+docker/volumes/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

diff --git a/Makefile b/Makefile
@@ -27,5 +27,4 @@ test-all: test-cpp test-python
 
 # Clean docker volumes and build directories
 clean:
-	@docker volume rm bazel_cache
-	@rm -rf src/bazel-* bazel_assets build dist
+	@rm -rf src/bazel-* bazel_assets build dist docker/volmes/bazel_cache
diff --git a/scripts/cpp_build.sh b/scripts/cpp_build.sh
@@ -5,7 +5,7 @@ CONTAINER_NAME="das-node-build"
 docker run \
   --rm \
   --name=$CONTAINER_NAME \
-  --mount type=volume,source=bazel_cache,target=/root/.cache/bazel \
+  --volume ./docker/volumes/bazel_cache:/root/.cache/bazel \
   --volume .:/opt/hyperon_das_node \
   --workdir /opt/hyperon_das_node/src \
   das-node-builder \

diff --git a/scripts/cpp_test.sh b/scripts/cpp_test.sh
@@ -5,7 +5,7 @@ CONTAINER_NAME="das-node-cpp-test"
 docker run \
   --rm \
   --name=$CONTAINER_NAME \
-  --mount type=volume,source=bazel_cache,target=/root/.cache/bazel \
+  --volume ./docker/volumes/bazel_cache:/root/.cache/bazel \
   --volume .:/opt/hyperon_das_node \
   --workdir /opt/hyperon_das_node/src \
   das-node-builder \

diff --git a/scripts/wheeler_test.sh b/scripts/wheeler_test.sh
@@ -9,7 +9,7 @@ CONTAINER_NAME="das-node-wheeler-test"
 docker run \
   --rm \
   --name=$CONTAINER_NAME \
-  --mount type=volume,source=bazel_cache,target=/root/.cache/bazel \
+  --volume ./docker/volumes/bazel_cache:/root/.cache/bazel \
   --volume .:/opt/hyperon_das_node \
   --workdir /opt/hyperon_das_node \
   das-node-builder \