tests_and_publish.yml

name: "Test and Publish"

on:
  push:
    branches: 
      - master
      - develop
    paths-ignore:
      - "docs/**"
      - CHANGELOG.md
      - README.md
      - CONTRIBUTING.md
  pull_request:
    branches:
      - master
      - develop
    paths-ignore:
      - "docs/**"
      - CHANGELOG.md
      - README.md
      - CONTRIBUTING.md
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:
  repository_dispatch:
    types: [ok-to-test-command]
jobs:
  unit_tests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ '3.8', '3.9', '3.10', '3.11']

    steps:
    - uses: actions/checkout@v4
    
    - name: Setup python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Setup poetry
      run: |
        python -V
        python -m pip install poetry
   
    - name: Check pre-commit status
      run: |
        poetry install -v
        poetry run pre-commit run --all-files

    - name: Test with tox
      run: |
        pip install tox-pip-version tox-gh-actions "tox<4.0.0"
        tox -v

    - name: Store coverage reports
      uses: actions/upload-artifact@v4
      with:
        name: coverage-${{ matrix.python-version }}
        path: coverage.xml
        if-no-files-found: error

  sonarcloud_trusted_branch:
    if: (github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request == null)
    runs-on: ubuntu-latest
    needs: unit_tests
    steps:

    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - uses: actions/download-artifact@v4
      with:
        name: coverage-3.9
        path: .

    - name: SonarCloud Scan
      uses: sonarsource/sonarcloud-github-action@master
      with:
        projectBaseDir: ${{ github.workspace }}
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}

  sonarcloud_forked_branch:
    if: |
      github.event_name == 'repository_dispatch' &&
      github.event.client_payload.slash_command.args.named.sha != '' &&
      contains(
        github.event.client_payload.pull_request.head.sha,
        github.event.client_payload.slash_command.args.named.sha
      )
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
      checks: write
    needs: unit_tests
    steps:

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'

      - uses: actions/download-artifact@v4
        with:
          name: coverage-3.9
          path: .

      - name: SonarCloud Scan
        uses: sonarsource/sonarcloud-github-action@master
        with:
          projectBaseDir: ${{ github.workspace }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}

      - uses: actions/github-script@v6
        id: update-check-run
        if: ${{ always() }}
        env:
          number: ${{ github.event.client_payload.pull_request.number }}
          job: ${{ github.job }}
          # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run
          conclusion: ${{ job.status }}
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const { data: pull } = await github.rest.pulls.get({
              ...context.repo,
              pull_number: process.env.number
            });
            const ref = pull.head.sha;

            const { data: checks } = await github.rest.checks.listForRef({
              ...context.repo,
              ref
            });

            const check = checks.check_runs.filter(c => c.name === process.env.job);

            const { data: result } = await github.rest.checks.update({
              ...context.repo,
              check_run_id: check[0].id,
              status: 'completed',
              conclusion: process.env.conclusion
            });

            return result;

  codeql:
    runs-on: ubuntu-latest
    needs: unit_tests
    permissions:
      actions: read
      contents: read
      security-events: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v3
        with:
          languages: python

      - name: Autobuild
        uses: github/codeql-action/autobuild@v3

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v3

  ### PROJECT SPECIFIC CONFIGURATION HERE

  e2e_tests_trusted_branch:
    if: (github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request == null)
    needs: [unit_tests, sonarcloud_trusted_branch]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        e2e_config: ["e2e", "e2e_pipeline_data_passing"]
    steps:
      - uses: actions/checkout@v4

      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Setup poetry
        run: |
          python -V
          python -m pip install poetry

      - name: Build the package
        run: |
          poetry build -f sdist

      - name: Initialize starter project
        run: |
          pip install $(find "./dist" -name "*.tar.gz")
          kedro new --starter spaceflights --config tests/conf/${{ matrix.e2e_config }}/starter-config.yml --verbose

      - name: Install starter requirements
        working-directory: ./spaceflights
        run: |
          find "../dist" -name "*.tar.gz" | xargs -I@ cp @ kedro-azureml.tar.gz
          echo -e "\n./kedro-azureml.tar.gz\n" >> src/requirements.txt
          echo -e "kedro-docker<0.5.0\n" >> src/requirements.txt
          echo -e "openpyxl\n" >> src/requirements.txt  # temp fix for kedro-datasets issues with optional packages
          sed -i '/kedro-telemetry/d' src/requirements.txt
          sed -i '/kedro-viz/d' src/requirements.txt # starter version requirements make tests fail
          echo $(cat src/requirements.txt)
          pip install -r src/requirements.txt

      - name: Update starter configs
        working-directory: ./spaceflights
        run: |
          kedro docker init
          sed -i 's/\(COPY src\/requirements.txt.*\)$/\1\nCOPY kedro-azureml.tar.gz ./g' Dockerfile
          cat Dockerfile
          echo "!data/01_raw" >> .dockerignore
          rm conf/base/catalog.yml
          cp ../tests/conf/${{ matrix.e2e_config }}/catalog.yml conf/base/catalog.yml
          cp ../tests/conf/${{ matrix.e2e_config }}/azureml.yml conf/base/azureml.yml
          sed -i 's/{container_registry}/${{ secrets.REGISTRY_LOGIN_SERVER }}/g' conf/base/azureml.yml
          sed -i 's/{image_tag}/${{ matrix.e2e_config }}/g' conf/base/azureml.yml
          
          cat conf/base/azureml.yml

      - name: Login via Azure CLI
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}

      - name: Login to acr.io
        uses: azure/docker-login@v1
        with:
          login-server: ${{ secrets.REGISTRY_LOGIN_SERVER }}
          username: ${{ secrets.AZURE_CLIENT_ID }}
          password: ${{ secrets.AZURE_CLIENT_SECRET }}

      - name: Build and push docker image
        working-directory: ./spaceflights
        run: |
          docker pull ${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }} || true
          docker build --build-arg BASE_IMAGE=python:3.10-buster -t ${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }} --cache-from=${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }} .
          docker push ${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }}

      - name: Run on Azure ML Pipelines
        working-directory: ./spaceflights
        env:
          AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
        run: |
          kedro azureml run --wait-for-completion --env-var 'GETINDATA=ROCKS!'

  e2e_tests_forked_branch:
    if: |
      github.event_name == 'repository_dispatch' &&
      github.event.client_payload.slash_command.args.named.sha != '' &&
      contains(
        github.event.client_payload.pull_request.head.sha,
        github.event.client_payload.slash_command.args.named.sha
      )
    needs: [unit_tests, sonarcloud_forked_branch]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        e2e_config: ["e2e", "e2e_pipeline_data_passing"]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'

      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Setup poetry
        run: |
          python -V
          python -m pip install poetry

      - name: Build the package
        run: |
          poetry build -f sdist

      - name: Initialize starter project
        run: |
          pip install $(find "./dist" -name "*.tar.gz")
          kedro new --starter spaceflights --config tests/conf/${{ matrix.e2e_config }}/starter-config.yml --verbose

      - name: Install starter requirements
        working-directory: ./spaceflights
        run: |
          find "../dist" -name "*.tar.gz" | xargs -I@ cp @ kedro-azureml.tar.gz
          echo -e "\n./kedro-azureml.tar.gz\n" >> src/requirements.txt
          echo -e "kedro-docker<0.5.0\n" >> src/requirements.txt
          echo -e "openpyxl\n" >> src/requirements.txt  # temp fix for kedro-datasets issues with optional packages
          sed -i '/kedro-telemetry/d' src/requirements.txt
          sed -i '/kedro-viz/d' src/requirements.txt # starter version requirements make tests fail
          echo $(cat src/requirements.txt)
          pip install -r src/requirements.txt

      - name: Update starter configs
        working-directory: ./spaceflights
        run: |
          kedro docker init
          sed -i 's/\(COPY src\/requirements.txt.*\)$/\1\nCOPY kedro-azureml.tar.gz ./g' Dockerfile
          cat Dockerfile
          echo "!data/01_raw" >> .dockerignore
          rm conf/base/catalog.yml
          cp ../tests/conf/${{ matrix.e2e_config }}/catalog.yml conf/base/catalog.yml
          cp ../tests/conf/${{ matrix.e2e_config }}/azureml.yml conf/base/azureml.yml
          sed -i 's/{container_registry}/${{ secrets.REGISTRY_LOGIN_SERVER }}/g' conf/base/azureml.yml
          sed -i 's/{image_tag}/${{ matrix.e2e_config }}/g' conf/base/azureml.yml
          
          cat conf/base/azureml.yml

      - name: Login via Azure CLI
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}

      - name: Login to acr.io
        uses: azure/docker-login@v1
        with:
          login-server: ${{ secrets.REGISTRY_LOGIN_SERVER }}
          username: ${{ secrets.AZURE_CLIENT_ID }}
          password: ${{ secrets.AZURE_CLIENT_SECRET }}

      - name: Build and push docker image
        working-directory: ./spaceflights
        run: |
          docker pull ${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }} || true
          docker build --build-arg BASE_IMAGE=python:3.10-buster -t ${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }} --cache-from=${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }} .
          docker push ${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }}

      - name: Run on Azure ML Pipelines
        working-directory: ./spaceflights
        env:
          AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
        run: |
          kedro azureml run --wait-for-completion --env-var 'GETINDATA=ROCKS!'
          
      - uses: actions/github-script@v6
        id: update-check-run
        if: ${{ always() }}
        env:
          number: ${{ github.event.client_payload.pull_request.number }}
          job: ${{ github.job }}
          # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run
          conclusion: ${{ job.status }}
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const { data: pull } = await github.rest.pulls.get({
              ...context.repo,
              pull_number: process.env.number
            });
            const ref = pull.head.sha;

            const { data: checks } = await github.rest.checks.listForRef({
              ...context.repo,
              ref
            });

            const check = checks.check_runs.filter(c => c.name === process.env.job);

            const { data: result } = await github.rest.checks.update({
              ...context.repo,
              check_run_id: check[0].id,
              status: 'completed',
              conclusion: process.env.conclusion
            });

            return result;
  publish:
    # only run on push to master
    if: github.event.pull_request == null && github.ref == 'refs/heads/master'
    needs: [e2e_tests_trusted_branch, codeql]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ 3.8 ]
    env:
      PYTHON_PACKAGE: kedro_azureml
    steps:
      - name: Checkout the repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0  # necessary to enable merging, all the history is needed
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Build package dist from source # A better way will be : https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ but pep 517 is still marked as experimental
        run: |
          pip install poetry
          poetry build
      - name: Merge back to develop  # we have to set the config first on a fresh machine
        run: |
          git config user.name github-actions
          git config user.email github-actions@github.com
          git checkout -B develop --track origin/develop
          git merge master
          git push
      - name: Set dynamically package version as output variable # see https://github.com/actions/create-release/issues/39
        # see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
        id: set_package_version
        run: |
          echo "::set-output name=PACKAGE_VERSION::$(cat $PYTHON_PACKAGE/__init__.py | grep -Po  '\d+\.\d+\.\d+')"
      - name: Create temporary file with the body content for the release
        run: |
          grep -Poz "## \[${{steps.set_package_version.outputs.PACKAGE_VERSION}}] - \d{4}-\d{2}-\d{2}[\S\s]+?(?=## \[\d+\.\d+\.\d+\]|\[.+\]:)" CHANGELOG.md > release_body.md
      - name: Create Release # https://github.com/actions/create-release
        id: create_release
        uses: actions/create-release@v1.1.4
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
        with:
          tag_name: ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
          release_name: Release ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
          body_path: ./release_body.md
          draft: false
          prerelease: false
      - name: Rollback Release in case of run failure
        if: failure() && steps.create_release.outputs.id != ''
        uses: author/action-rollback@stable
        with:
          # Using a known release ID
          release_id: ${{ steps.create_release.outputs.id }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Publish distribution to PyPI  # official action from python maintainers
        uses: pypa/gh-action-pypi-publish@master
        with:
          user: __token__
          password: ${{ secrets.PYPI_PASSWORD }}
          verbose: true  # trace if the upload fails