Merge pull request #16 from mskcc-omics-workflows/enhancement/gh_acti…

…on_test

Add automated testing of changes in .github/workflows

.actrc

@@ -0,0 +1 @@
+-e act_event.json

.github/workflows/dev-build.yml

@@ -24,6 +24,10 @@ jobs:
       # Expose matched filters as job 'docker-images' output variable
       docker-images: ${{ steps.filter.outputs.changes }}
     steps:
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v4
         with:
           fetch-depth: 2
@@ -32,20 +36,42 @@ jobs:
       - name: Create tags.yml to grab changes
         id: get_tags
         run: |
-          for i in $(find containers -name "Dockerfile")
+          for i in $(find containers -name "Dockerfile" -not -path "containers/testapp/0.0.2/*")
           do
           path=$(dirname $i)
           tag=$(echo $path | cut -f 2- -d/)
           echo -e "$tag:\n  $path/**"
           done > .github/tags.yml
+      - name: Add to tags.yml to grab changes in testapp/0.0.2 and github action workflows
+        id: add_testapp_002_tag
+        run: |
+          echo -e "testapp/0.0.2:\n  - testapp/0.0.2/**\n  - .github/workflows/**" >> .github/tags.yml
       - name: debug
         run: cat .github/tags.yml
       - uses: mirpedrol/paths-filter@main
         id: filter
         with:
+          base: "develop"
           filters: ".github/tags.yml"
           token: ""
-  dockerfile-validate-build:
+  dockerfile-lint:
+    runs-on: ubuntu-latest
+    name: dockerfile-lint
+    needs: [dockerfile-changes]
+    if: needs.dockerfile-changes.outputs.docker-images != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: ["${{ fromJson(needs.dockerfile-changes.outputs.docker-images) }}"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Hadolint
+        uses: hadolint/[email protected]
+        with:
+          dockerfile: containers/${{ matrix.tags }}/Dockerfile
+          verbose: true
+  docker-validate-build:
     runs-on: ubuntu-latest
     name: dockerfile-build
     needs: [dockerfile-changes]
@@ -55,10 +81,12 @@ jobs:
       matrix:
         tags: ["${{ fromJson(needs.dockerfile-changes.outputs.docker-images) }}"]
     steps:
-      # save hadolint-action docker image, which for whatever reason builds before other steps
+      # save docker files
       - name: Move /var/lib/docker/
+        if: ${{ !github.event.act }}
         run: sudo mv /var/lib/docker/ "${GITHUB_WORKSPACE}/docker"
       - name: Maximize build space
+        if: ${{ !github.event.act }}
         uses: easimon/maximize-build-space@master
         with:
           build-mount-path: /var/lib/docker/
@@ -67,14 +95,12 @@ jobs:
           remove-haskell: 'true'
           remove-codeql: 'true'
       - name: Restore /var/lib/docker/
-        run: sudo sh -c "mv ${GITHUB_WORKSPACE}/docker/* /var/lib/docker"
+        if: ${{ !github.event.act }}
+        run: |
+          sudo rsync -aPq "${GITHUB_WORKSPACE}/docker/" /var/lib/docker
+          sudo rm -rf "${GITHUB_WORKSPACE}/docker"
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Hadolint
-        uses: hadolint/[email protected]
-        with:
-          dockerfile: containers/${{ matrix.tags }}/Dockerfile
-          verbose: true
       - name: Set up Python
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
         with:
@@ -108,7 +134,7 @@ jobs:
         run : pip install docker jsonschema
       - name: Validate image build metadata
         run: |
-          python .github/validate_docker.py ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}} .github/jsonschema/docker_image.json
+          python .github/workflows/scripts/validate_docker.py ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}} .github/workflows/scripts/jsonschema/docker_image.json
       - name: Test JFrog image build
         run: |
           docker run --rm ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
@@ -121,7 +147,7 @@ jobs:
           push: true
   confirm-pass:
     runs-on: ubuntu-latest
-    needs: [ dockerfile-changes, dockerfile-validate-build ]
+    needs: [ dockerfile-changes, docker-validate-build, dockerfile-lint ]
     if: always()
     steps:
       - name: All tests ok

.github/workflows/prod-build.yml

@@ -15,13 +15,18 @@ env:
 jobs:
   dockerfile-changes:
     name: dockerfile-changes
+    if: ${{ !github.event.act }}
     runs-on: ubuntu-latest
     permissions:
       pull-requests: read
     outputs:
       # Expose matched filters as job 'docker-images' output variable
       docker-images: ${{ steps.filter.outputs.changes }}
     steps:
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v4
         with:
           fetch-depth: 2
@@ -30,20 +35,42 @@ jobs:
       - name: Create tags.yml to grab changes
         id: get_tags
         run: |
-          for i in $(find containers -name "Dockerfile")
+          for i in $(find containers -name "Dockerfile" -not -path "containers/testapp/0.0.2/*")
           do
           path=$(dirname $i)
           tag=$(echo $path | cut -f 2- -d/)
           echo -e "$tag:\n  $path/**"
           done > .github/tags.yml
+      - name: Add to tags.yml to grab changes in testapp/0.0.2 and github action workflows
+        id: add_testapp_002_tag
+        run: |
+          echo -e "testapp/0.0.2:\n  - testapp/0.0.2/**\n  - .github/workflows/**" >> .github/tags.yml
       - name: debug
         run: cat .github/tags.yml
       - uses: mirpedrol/paths-filter@main
         id: filter
         with:
+          base: "main"
           filters: ".github/tags.yml"
           token: ""
-  dockerfile-validate-build:
+  dockerfile-lint:
+    runs-on: ubuntu-latest
+    name: dockerfile-lint
+    needs: [dockerfile-changes]
+    if: needs.dockerfile-changes.outputs.docker-images != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: ["${{ fromJson(needs.dockerfile-changes.outputs.docker-images) }}"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Hadolint
+        uses: hadolint/[email protected]
+        with:
+          dockerfile: containers/${{ matrix.tags }}/Dockerfile
+          verbose: true
+  docker-validate-build:
     runs-on: ubuntu-latest
     name: dockerfile-build
     needs: [dockerfile-changes]
@@ -53,26 +80,23 @@ jobs:
       matrix:
         tags: ["${{ fromJson(needs.dockerfile-changes.outputs.docker-images) }}"]
     steps:
-      # save hadolint-action docker image, which for whatever reason builds before other steps
+      # save docker files
       - name: Move /var/lib/docker/
-        run: sudo mv /var/lib/docker "${GITHUB_WORKSPACE}/docker"
+        run: sudo mv /var/lib/docker/ "${GITHUB_WORKSPACE}/docker"
       - name: Maximize build space
         uses: easimon/maximize-build-space@master
         with:
+          build-mount-path: /var/lib/docker/
           remove-dotnet: 'true'
           remove-android: 'true'
           remove-haskell: 'true'
           remove-codeql: 'true'
-          build-mount-path: '/var/lib/docker/'
       - name: Restore /var/lib/docker/
-        run: sudo sh -c "mv ${GITHUB_WORKSPACE}/docker/* /var/lib/docker"
+        run: |
+          sudo rsync -aPq "${GITHUB_WORKSPACE}/docker/" /var/lib/docker
+          sudo rm -rf "${GITHUB_WORKSPACE}/docker"
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Hadolint
-        uses: hadolint/[email protected]
-        with:
-          dockerfile: containers/${{ matrix.tags }}/Dockerfile
-          verbose: true
       - name: Set up Python
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
         with:
@@ -106,7 +130,7 @@ jobs:
         run : pip install docker jsonschema
       - name: Validate image build metadata
         run: |
-          python .github/validate_docker.py ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}} .github/jsonschema/docker_image.json
+          python .github/workflows/scripts/validate_docker.py ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}} .github/workflows/scripts/jsonschema/docker_image.json
       - name: Test JFrog image build
         run: |
           docker run --rm ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
@@ -129,8 +153,8 @@ jobs:
           push: true
   confirm-pass:
     runs-on: ubuntu-latest
-    needs: [ dockerfile-changes, dockerfile-validate-build ]
-    if: always()
+    needs: [ dockerfile-changes, docker-validate-build, dockerfile-lint ]
+    if: always() & ${{ !github.event.act }}
     steps:
       - name: All tests ok
         if: ${{ success() || !contains(needs.*.result, 'failure') }}

act_event.json

@@ -0,0 +1,3 @@
+{
+    "act": true
+}