#295: Made notebook-tests mandatory for merge

.github/workflows/check_ci.yaml → .github/workflows/ci.yaml

@@ -1,24 +1,29 @@
-name: CI Build
+name: CI
 
 on:
-  push:
-    branches-ignore:
-      - "main"
+  pull_request:
 
 jobs:
-  run_unit_tests:
+  run-unit-tests:
+    name: Unit Tests
     runs-on: ubuntu-latest
 
     steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
 
+     - name: Run shellcheck
+       run: ./scripts/build/shellcheck.sh
+
      - name: Setup Python & Poetry Environment
        uses: exasol/python-toolbox/.github/actions/[email protected]
        with:
          python-version: "3.10"
-         poetry-version: '1.8.2'
+         poetry-version: "1.8.2"
+
+     - name: Check Version Number
+       run: poetry run python3 -u "./scripts/build/check_release.py"
 
      - name: Run Unit Tests
        run: >
@@ -28,14 +33,14 @@ jobs:
          --override-ini=log_cli_level=INFO
          test/unit
 
-  run_integration_tests:
+  run-integration-tests:
     name: Integration Tests
     environment: AWS_CI_TESTS
     runs-on: ubuntu-latest
-    needs: run_unit_tests
+    needs: run-unit-tests
 
     steps:
-     - uses: actions/checkout@v3
+     - uses: actions/checkout@v4
        with:
          fetch-depth: 0
 
@@ -61,8 +66,7 @@ jobs:
        uses: exasol/python-toolbox/.github/actions/[email protected]
        with:
          python-version: "3.10"
-         poetry-version: '1.8.2'
-
+         poetry-version: "1.8.2"
 
      - name: Run Integration Tests
        run: >
@@ -76,15 +80,23 @@ jobs:
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
 
-  run_notebook_tests:
+  approval-for-notebook-tests:
+    name: Run Jupyter Notebook Tests?
+    runs-on: ubuntu-latest
+    steps:
+      - name: Detect Running Notebook Tests
+        run: true
+    environment:
+      approve-test-execution
+
+  run-notebook-tests:
     name: Jupyter Notebook Tests
-    if: "contains(github.event.head_commit.message, '[run-notebook-tests]')"
     environment: AWS_SAGEMAKER
     runs-on: ubuntu-latest
-    needs: run_unit_tests
+    needs: approval-for-notebook-tests
 
     steps:
-     - uses: actions/checkout@v3
+     - uses: actions/checkout@v4
        with:
          fetch-depth: 0
 
@@ -110,8 +122,7 @@ jobs:
        uses: exasol/python-toolbox/.github/actions/[email protected]
        with:
          python-version: "3.10"
-         poetry-version: '1.8.2'
-
+         poetry-version: "1.8.2"
 
      - name: Run notebook tests
        run: >
@@ -127,3 +138,11 @@ jobs:
          SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }}
          SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }}
          SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }}
+
+  gate-2:
+    name: Gate 2 - Allow Merge
+    runs-on: ubuntu-latest
+    needs: [ run-unit-tests, run-integration-tests, run-notebook-tests ]
+    steps:
+      - name: Branch Protection
+        run: true

.github/workflows/check_notebook_links.yaml → .github/workflows/hyperlinks.yaml

@@ -1,25 +1,31 @@
-name: Check Notebook Links
+name: Check Hyperlinks
 
 on:
-  push:
-    branches-ignore:
-      - "main"
+  pull_request:
 
 jobs:
-  check_notebook_links:
+  check-links:
+    name: Find Broken Links in Documentation and Jupyter Notebooks
     runs-on: ubuntu-latest
-
     steps:
+
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
+
      - name: Setup Python & Poetry Environment
        uses: exasol/python-toolbox/.github/actions/[email protected]
        with:
          python-version: "3.10"
          poetry-version: '1.8.2'
 
-     - name: Run build ai-lab tests
+     - name: Check Hyperlinks in Documentation
+       id: lychee
+       uses: lycheeverse/[email protected]
+       with:
+         fail: true
+         args: --verbose --no-progress 'doc/**/*.md' 'README.md'
+
+     - name: Check Hyperlinks in Jupyter Notebooks
        run: >
          poetry run pytest --check-links exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/
-

doc/changes/changes_2.1.0.md

@@ -35,3 +35,4 @@ Version: 2.1.0
 * #267: Switched CodeBuildWaiter to use tenacity
 * #276: Started using the new ITDE Manager interface in the notebook-connector 0.2.9
 * #282: Updated python version to Python 3.10
+* #295: Made notebook-tests mandatory for merge 

doc/developer_guide/ci.md

@@ -3,20 +3,16 @@
 The project has two types of CI tests:
 * Unit tests and integration tests which run in a Github workflow
 * Special integration tests verifying the content of the Jupyter notebook files
-* A system test which runs on a AWS Codebuild
+* A system test which runs in a AWS Codebuild
 
 All these tests need to pass before the approval of a Github PR.
 The Github workflow will run on each push to a branch in the Github repository.
 
-However, the notebook tests and the AWS Codebuild will only run after you push a commit containing a special string in the commit message, see the following sections.
+However, the notebook tests and the AWS Codebuild will only run under specific conditions, e.g. manual approval or push a commit containing a special string in the commit message, see the following sections.
 
-### Executing Notebook Tests
+### Executing Jupyter Notebook Tests
 
-Use the following git commands to execute the notebook tests:
-
-```shell
-git commit -m "[run-notebook-tests]" --allow-empty && git push
-```
+The regular CI build will ask for confirmation (aka. "review") before executing these tests, see [ETAJ developer guide](https://github.com/exasol/exasol-test-setup-abstraction-java/blob/main/doc/developer_guide/developer_guide.md#ci-build) for details.
 
 ### Executing AWS CodeBuild
 

test/integration/test_create_dss_docker_image.py

@@ -299,6 +299,7 @@ def user_and_group(ls_line: str) -> str:
                 'bind': notebooks_folder,
                 'mode': 'rw', }, },
     ) as container:
+        wait_for(container, "entrypoint.py: Did chown -R")
         testees = [tmp_path, child, sub, grand_child]
         command = ls_command(str(tmp_path), notebooks_folder, testees)
         output = assert_exec_run(container, command, user=JUPYTER_USER)