ci.yml

# name of the workflow. Link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#name
name: CI
# running on push to main and develop branches or on pull reuqests or on manual trigger
on: 
  # manual trigger
  workflow_dispatch:
    inputs:
      ssh_debug_enabled:
        type: boolean
        description: 'Run the build/test with ssh debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
        required: false
        default: false
      debug_deployment:
        type: boolean
        description: 'Run the pipeline with debug deployment enabled'
        required: false
        default: false

  # runnnig on push to main and develop branches
  push:
    branches:
      - main
      - develop
    paths-ignore:
      - '**/README.md'
      - '.devcontainer/**'
  # running on pull requests to main and develop branches
  pull_request:
    branches:
      - main
      - develop
    paths-ignore:
      - '**/README.md'

# defining global environment variables for all jobs
env:
  # define runner indexes for tests splitting and parallel execution
  total-runners: 5
  # defining GitHub registry for docker images
  REGISTRY: ghcr.io
  # github.repository as <account>/<repo>
  IMAGE_NAME: ${{ github.repository }}

# build job definition with 7 steps:
# 1. Checkout repository
# 2. Cache Maven packages
# 3. Running Liquibase Quality Checks link - https://docs.liquibase.com/tools-integrations/liquibase-quality-checks/overview.html
# 4. Initialize CodeQL
# 5. Build with Maven
# 6. Perform CodeQL Analysis link - https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning
# 7. Upload artifact
jobs:
  build:
    # build job will run on ubuntu-latest github-hosted runner
    runs-on: ubuntu-latest
    # defining permissions for the job - read contents, write packages, write id-token. Link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idpermissions
    # security-events permission is required for CodeQL analysis
    # enforcing policy for the job - only users with write access to the repository can trigger the job. Link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idenforce_admins
    permissions:
      actions: read
      contents: read
      packages: write
      id-token: write
      security-events: write
    # defining strategy for the job - matrix strategy for codeQL analysis
    strategy:
      fail-fast: false
      matrix:
        language: [ 'java', 'javascript' ]
      #   # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
      #   # Use only 'java' to analyze code written in Java, Kotlin or both
      #   # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
      #   # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
    
    # defining steps for the job as explained above 
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4.2.2 # cache maven packages step - caching maven packages to speed up the build process. Link to the documentation - https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
      - name: Cache Maven packages
        uses: actions/cache@v4.1.2 # defining cache key and restore keys for the cache step. Link to the documentation - https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows#matching-a-cache-key
        with:
          path: ~/.m2/repository # path to the directory where maven packages are stored - /root/.m2 in the container
          key: ${{ runner.os }}-build-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-build-
      # running liquibase quality checks step to ensure that the database changelogs are well-formed and follow best practices.
      # Link to the documentation - https://docs.liquibase.com/tools-integrations/liquibase-quality-checks/overview.html
      # Using the Liquibase Maven plugin, you can run Liquibase Quality Checks on your changelogs. 
      # The plugin is available in the Maven Central repository. Link to Maven Central - https://search.maven.org/artifact/org.liquibase/liquibase-maven-plugin
      - name: Running Liquibase Quality Checks to ensure that the database changelogs are well-formed and follow best practices
        run: |
          mvn process-resources liquibase:checks.run
      # runnning code scanning with CodeQL. Link to the documentation - https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning
      # first step is to initialize CodeQL
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v3
        with:
          languages: ${{ matrix.language }} # defining the language for the CodeQL analysis
          # debug: true # uncomment this line to enable debugging for CodeQL analysis step
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.

        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
        # queries: security-extended,security-and-quality

      # autobuild with codeql
      # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
      # If this step fails, then you should remove it and run the build manually (see below)
      - name: Autobuild
        uses: github/codeql-action/autobuild@v3
      
      # performing Code Quality Analysis with CodeQL. Link to the documentation - https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning
      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v3
        with:
          category: "/language:${{matrix.language}}" # defining the language for the CodeQL analysis
      
      - uses: actions/upload-artifact@v4.4.3 # uploading the artifact to the GitHub Artifacts. Link to the documentation - https://docs.github.com/en/actions/guides/storing-workflow-data-as-artifacts
        if: matrix.language == 'java' # upload the artifact only for the java language
        with:   
          name: jar-artifact # naming the artifact jar file/s path
          path: target/ # actual relative path to the artifact in the container - target/
  
  unit-parallel-tests:
    # unit-parallel-tests job will run on ubuntu-latest github-hosted runner
    name: UNIT-PARALLEL-TESTS
    runs-on: ubuntu-latest
    needs: # needs build job and runner-indexes job to be completed before running the unit-parallel-tests job
      - build
      - runner-indexes
    container:
      image: ghcr.io/tsviz/tsvi-spring-test:v2.0.0 # using the base maven:3.8.7-openjdk-18 image + jq + curl + nodejs 18
    services:
      # postgres service container
      postgres: # service name - postgres. This name is used to access the service container from the job container as the host name.
        image: postgres # running the job in a container - postgres link to the docker hub - https://hub.docker.com/_/postgres
        env:
          POSTGRES_PASSWORD: postgres # setting the password for the postgres database
        # exposing the port 5432 of the postgres service container to the host machine
        ports:
          - 5432:5432
      # redis service container for caching session data
      redis: # service name - redis. This name is used to access the service container from the job container as the host name.
        image: redis # running the job in a container - redis link to the docker hub - https://hub.docker.com/_/redis
        # exposing the port 6379 of the redis service container to the host machine
        ports:
          - 6379:6379
    #  defining the job permissions
    permissions:
      contents: read # read access to the repository contents
      packages: write # write access to the repository packages
      id-token: write # write access to the repository id token
    strategy: # defining the job to run in parallel with the matrix strategy and runner-indexes job output
      fail-fast: true # cancels all in-progress jobs if any matrix job fails link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast
      matrix: # defining the matrix strategy to run the job in parallel using x number of github-hosted runners defined in the env total_runners above
        runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} # using the runner-indexes job output to define the matrix strategy
    steps:
      - name: Checkout repository # checkout the repository
        uses: actions/checkout@v4.2.2
      # caching the maven packages to speed up the build process. 
      # Link to the documentation - https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
      - name: Cache Maven packages
        uses: actions/cache@v4.1.2 # defining the cache key and restore keys for the cache step. Link to the documentation - https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows#matching-a-cache-key
        with:
          path: /root/.m2 # path to cache
          key: ${{ runner.os }}-junit-${{ hashFiles('**/pom.xml') }} # key for restoring and saving the cache
          restore-keys: ${{ runner.os }}-junit- # key for restoring the cache if no exact match is found

      # In this step, we are downloading the latest artifact from the build job and storing it in the container
      - name: Download and unzip all test results artifacts
        run: |
            # Fetch the last successful workflow run ID
            LAST_SUCCESSFUL_RUN_ID=$(curl \
              -H "Accept: application/vnd.github+json" \
              -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
              -H "X-GitHub-Api-Version: 2022-11-28" \
              "https://api.github.com/repos/${{ github.repository }}/actions/runs?status=success&per_page=1" | \
              jq -r '.workflow_runs[0].id')

            if [ -z "$LAST_SUCCESSFUL_RUN_ID" ]; then
              echo "No successful runs found."
              exit 1
            fi
            
            ARTIFACT_IDS=$(curl \
              -H "Accept: application/vnd.github+json" \
              -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
              -H "X-GitHub-Api-Version: 2022-11-28" \
              "https://api.github.com/repos/${{ github.repository }}/actions/runs/${LAST_SUCCESSFUL_RUN_ID}/artifacts" | \
              jq -r '.artifacts | sort_by(.created_at) | .[] | select(.name | startswith("testresults-")) | .id')

            mkdir -p test_results

            echo "Downloading artifacts for run ID: $LAST_SUCCESSFUL_RUN_ID"

            for ARTIFACT_ID in $ARTIFACT_IDS; do
              curl \
                -H "Accept: application/vnd.github+json" \
                -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
                -H "X-GitHub-Api-Version: 2022-11-28" \
                -L -o my_artifact.zip \
                "https://api.github.com/repos/${{ github.repository }}/actions/artifacts/${ARTIFACT_ID}/zip"

              echo "Contents of my_artifact.zip for artifact ID: $ARTIFACT_ID"
              unzip -l my_artifact.zip

              unzip my_artifact.zip -d test_results 2> /dev/null || true
              echo "Contents of test_results directory for artifact ID: $ARTIFACT_ID"
              ls -l test_results
              rm my_artifact.zip
            done

      - name: Ensure junit-path exists and that all xml files are readable in path 'test_results/**/*xml' with this syntax
        run: |
          if [ ! -d "test_results" ]; then
            echo "test_results directory does not exist"
            exit 0
          fi
          if ! ls test_results/*xml 1> /dev/null 2>&1; then
            echo "No xml files found in test_results directory"
            exit 0
          fi
          if ! ls -l test_results/*xml 1> /dev/null 2>&1; then
            echo "Not all xml files in test_results directory are readable"
            exit 0
          fi

          echo "All xml files in test_results directory are readable"
          ls -l test_results/*xml

      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3.19
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.ssh_debug_enabled }}

      # split-tests action - splits the tests into x number of groups 
      # based on the total number of github-hosted runners and junit previous test results by time and line count. 
      # Link to the action - https://github.com/marketplace/actions/split-tests
      - uses: scruplelesswizard/split-tests@4f1ca766cb93923ca216e02f1aefed20944e313f
        id: split-tests
        name: Split tests
        with:
          glob: src/test/**/**/*.java # glob pattern to match the test files
          split-total: ${{ env.total-runners }} # total number of github-hosted runners
          split-index: ${{ matrix.runner-index }} # current runner index
          junit-path: test_results/**/*xml # path to the junit test results with wildcards to match all the files
          # line-count: true # split the tests based on the junit test results by line count
      # run the tests in parallel looping through the test-suite output from the split-tests action
      - run: 'echo "This runner will execute the following tests: ${{ steps.split-tests.outputs.test-suite }}"'
      - run: |
          LIST="${{ steps.split-tests.outputs.test-suite }}"
          for file in $LIST
          do
          # sleep for 10 seconds to avoid timeout errors
            sleep 10
            mvn -Dtest=$(basename $file | sed -e "s/.java/,/" | tr -d '\r\n') -e test -Dspring.datasource.url=${{ secrets.LIQUIBASE_COMMAND_URL }} -Dspring.datasource.username=${{ secrets.LIQUIBASE_COMMAND_USERNAME }} -Dspring.datasource.password=${{ secrets.LIQUIBASE_COMMAND_PASSWORD }} -Dspring.liquibase.change-log=classpath:db/changelog/changelog_version-3.3.xml -Dserver.port=8086 -Dspring.redis.host=redis -Dspring.redis.port=6379 -Dspring.redis.mode=standalone
          done
      
      - uses: actions/upload-artifact@v4.4.3 # uploading the artifact to the GitHub Artifacts. Link to the documentation - https://docs.github.com/en/actions/guides/storing-workflow-data-as-artifacts
        with:
          name: testresults-${{ github.run_id }}-split-${{ matrix.runner-index }} # naming the artifact with the test results
          path: ./target/surefire-reports # path to the test results
          retention-days: 90 # retention period for the artifact in days. Link to the documentation - https://docs.github.com/en/actions/guides/storing-workflow-data-as-artifacts#about-workflow-artifact-retention
  
  publish-test-results:
    needs: [build, unit-parallel-tests]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4.2.2

      - name: List Artifacts
        id: list-artifacts
        run: |
          curl -s -u ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} \
               -H 'Accept: application/vnd.github.v3+json' \
               https://api.github.com/repos/octodemo/java-springboot-demo/actions/runs/${{ github.run_id }}/artifacts > artifacts.json
          cat artifacts.json

      - name: Download Artifacts
        run: |
          mkdir -p test_results
          for url in $(jq -r '.artifacts[] | select(.name | startswith("testresults-")) | .archive_download_url' artifacts.json); do
              artifact_name=$(echo $url | awk -F/ '{print $NF}' | awk -F? '{print $1}')
              curl -s -u ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} -L -o test_results/testresults.zip $url
              unzip -o test_results/testresults.zip -d test_results
              rm test_results/testresults.zip
          done

      - name: Publish Test Results
        uses: dorny/test-reporter@v1.8.0
        if: success() || failure()
        with:
          reporter: java-junit
          name: JUnit Test Results
          path: test_results/*.xml

  build-and-publish-docker-image: # job to build the docker image and publish it to the GitHub Container Registry
    runs-on: ubuntu-latest # using the latest ubuntu runner
    outputs:
      image_tag: ghcr.io/${{ github.repository }}:${{ github.run_number }} # output the image tag to be used in the build-and-publish-docker-image job
    needs: [build, unit-parallel-tests] # this job needs build and unit-parallel-tests jobs as a requirement to run
    if: github.ref == 'refs/heads/main' # run this job only when the branch is main branch and not on pull requests or other branches - https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context
    # permissions for write acces to the packages and id-token and push access to the repository to create the container registry token
    permissions:
      packages: write
      id-token: write
      contents: write

    # steps to run the unit-parallel-tests job are as follows:
    # 1. checkout the repository
    # 2. download the jar artifact from the build job
    # 3. use the docker layer caching action to speed up the docker image build process
    # 4. build the docker image
    # 5. log in to the GitHub Container Registry
    # 6. push the docker image to the GitHub Container Registry
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4.2.2
      - uses: actions/download-artifact@v4.1.8
        with:
          name: jar-artifact
          path: target/
      
      # build the docker image using the Dockerfile in the root of the repository 
      # and tag it with the current run number from the github action workflow run
      - name: Log in to the GH Container Registry
        uses: docker/login-action@v3.3.0 # using the docker login action from the github marketplace - github.com/marketplace/actions/docker-login
        with:
          registry: ${{ env.REGISTRY }} # using the registry environment variable
          username: ${{ github.actor }} # using the github.actor context
          password: ${{ secrets.GITHUB_TOKEN }} # using the GITHUB_TOKEN secret
      
      - name: Build and push Docker image
        id: build_image
        uses: docker/build-push-action@v6.9.0 # using the docker build and push action from the github marketplace - github.com/marketplace/actions/build-and-push-docker-images
        with: 
          context: . # using the current directory as the context
          push: true # push the docker image to the registry
          tags: |
            ghcr.io/${{ github.repository }}:${{ github.run_number }}
            ghcr.io/${{ github.repository }}:latest
          cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:latest # use the docker layer caching to speed up the docker image build process
          cache-to: type=inline

  runner-indexes: # job to generate the runner indexes for the unit-parallel-tests job
    runs-on: ubuntu-latest
    name: Generate runner indexes
    outputs:
      json: ${{ steps.generate-index-list.outputs.json }} # output the json with the runner indexes
    steps:
      - id: generate-index-list # generate the runner indexes and save them to the json file
        run: |
          MAX_INDEX=$((${{ env.total-runners }}-1)) # calculate the max index
          INDEX_LIST=$(seq 0 ${MAX_INDEX}) # generate the list of indexes
          INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) # convert the list to the json
          echo "json=${INDEX_JSON}" >> $GITHUB_OUTPUT # save the json to the GITHUB_OUTPUT environment variable

  deploy:
    needs: [build-and-publish-docker-image] # this job needs build-and-publish-docker-image job as a requirement to run
    uses: ./.github/workflows/cd.yml
    with:
      # with tag from the build-and-publish-docker-image job in the output_tags step
      image_tag: "${{ needs.build-and-publish-docker-image.outputs.image_tag }}"
      debug: "${{ github.event.inputs.debug_deployment }}"
    secrets: inherit