Add itde support (#115)

exasol · Apr 25, 2024 · 7bd2d05 · 7bd2d05
1 parent bb23ca6
commit 7bd2d05
Show file tree

Hide file tree

Showing 12 changed files with 1,498 additions and 352 deletions.
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: [ "3.11" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
 
     steps:
     - name: SCM Checkout
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: [ "3.11" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
 
     steps:
     - name: SCM Checkout

diff --git a/.github/workflows/ci-master.yml b/.github/workflows/ci-master.yml
@@ -1,15 +1,20 @@
 name: Continues Integration (Master)
 
 on:
-  workflow_dispatch:
   push:
-    - master
-    - main
-    - "Test-Automation"
+    branches:
+      - "master"
+      - "main"
   schedule:
     # “At 00:00 on every 7th day-of-month from 1 through 31.” (https://crontab.guru)
     - cron: "0 0 1/7 * *"
 
 jobs:
   verify:
     uses: ./.github/workflows/checks.yml
+
+  examples:
+    uses: ./.github/workflows/examples.yml
+
+  ssl_cert:
+    uses: ./.github/workflows/ssl_cert.yml
diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml
@@ -2,8 +2,13 @@ name: Continues Integration (PR)
 
 on:
   pull_request:
-
+  
 jobs:
-
   verify:
     uses: ./.github/workflows/checks.yml
+
+  examples:
+    uses: ./.github/workflows/examples.yml
+
+  ssl_cert:
+    uses: ./.github/workflows/ssl_cert.yml
diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
@@ -1,88 +1,81 @@
 name: Examples
 
 on:
-  push:
-    paths:
-      - "**.py"
-      - "**.yml"
-  pull_request:
-
-  workflow_dispatch:
+  workflow_call:
 
 jobs:
   examples:
+    name: Run Examples
+    # For all containers with versions 7.1.x and older versions 8.y.z of docker-db,
+    # there will be a crash if UDFs (User Defined Functions) are utilized.
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - python-version: "3.11"
-            exasol-tag: latest-7.1
-            exasol-port: 8563
-
-    name: "Build with Python ${{ matrix.python-version }} and Exasol ${{ matrix.exasol-tag }}"
-
-    runs-on: ubuntu-20.04
-    timeout-minutes: 15
-
-    services:
-      exasol:
-        image: exasol/docker-db:${{ matrix.exasol-tag }}
-        ports:
-          - 8563:${{ matrix.exasol-port }}
-        options: --privileged
-
+        python-version: ["3.11"]
     steps:
-      - name: "Checkout repo (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
+      - name: "Checkout repo"
         uses: actions/checkout@v2
 
-      - name: "Setup Python (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        uses: actions/setup-python@v2
+      - name: "Setup Python & Poetry Environment"
+        uses: exasol/python-toolbox/.github/actions/python[email protected]
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: "Install dependencies (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        run: pip install -e .[pandas,ujson,rapidjson,orjson,examples]
+      - name: "Install all extras"
+        run: poetry install --all-extras
+
+      - name: "Start test db"
+        run: >
+          poetry run itde spawn-test-environment --environment-name test 
+          --database-port-forward 8563 --bucketfs-port-forward 2580 
+          --docker-db-image-version 7.1.17 --db-mem-size 4GB
 
-      - name: "Wait for Exasol container (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        run: python examples/_wait_container.py
+      - name: "Wait for Exasol container (UDF extraction)"
+        run: poetry run python examples/_wait_container.py
+
+      - name: "Run examples"
+        run: |
+          poetry run python examples/a00_prepare.py
+          poetry run python examples/a01_basic.py
+          poetry run python examples/a02_fetch_tuple.py
+          poetry run python examples/a03_fetch_dict.py
+          poetry run python examples/a04_fetch_mapper.py
+          poetry run python examples/a05_formatting.py
+          poetry run python examples/a06_transaction.py
+          poetry run python examples/a07_exceptions.py
+          poetry run python examples/a08_ext.py
+          poetry run python examples/a09_abort_query.py
+          poetry run python examples/a10_context_manager.py
+          poetry run python examples/a11_insert_multi.py
+          poetry run python examples/a12_meta.py
+          poetry run python examples/a13_meta_nosql.py
+          poetry run python examples/b01_pandas.py
+          poetry run python examples/b02_import_export.py
+          poetry run python examples/b03_parallel_export.py
+          poetry run python examples/b04_parallel_import.py
+          poetry run python examples/b05_parallel_export_import.py
+          poetry run python examples/b06_http_transport_errors.py
+          poetry run python examples/c01_redundancy.py
+          poetry run python examples/c02_edge_case.py
+          poetry run python examples/c03_db2_compat.py
+          poetry run python examples/c04_encryption.py
+          poetry run python examples/c05_session_params.py
+          poetry run python examples/c06_local_config.py
+          poetry run python examples/c07_profiling.py
+          poetry run python examples/c08_snapshot_transactions.py
+          poetry run python examples/c09_script_output.py
+          poetry run python examples/c10_overload.py
+          poetry run python examples/c11_quote_ident.py
+          poetry run python examples/c12_thread_safety.py
+          poetry run python examples/c13_dsn_parsing.py
+          poetry run python examples/c14_http_proxy.py
+          poetry run python examples/c15_garbage_collection.py
+          poetry run python examples/j01_rapidjson.py
+          poetry run python examples/j02_ujson.py
+          poetry run python examples/j03_orjson.py
 
-      - name: "Run examples (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
+      - name: "Stop test db"
         run: |
-          python examples/a00_prepare.py
-          python examples/a01_basic.py
-          python examples/a02_fetch_tuple.py
-          python examples/a03_fetch_dict.py
-          python examples/a04_fetch_mapper.py
-          python examples/a05_formatting.py
-          python examples/a06_transaction.py
-          python examples/a07_exceptions.py
-          python examples/a08_ext.py
-          python examples/a09_abort_query.py
-          python examples/a10_context_manager.py
-          python examples/a11_insert_multi.py
-          python examples/a12_meta.py
-          python examples/a13_meta_nosql.py
-          python examples/b01_pandas.py
-          python examples/b02_import_export.py
-          python examples/b03_parallel_export.py
-          python examples/b04_parallel_import.py
-          python examples/b05_parallel_export_import.py
-          python examples/b06_http_transport_errors.py
-          python examples/c01_redundancy.py
-          python examples/c02_edge_case.py
-          python examples/c03_db2_compat.py
-          python examples/c04_encryption.py
-          python examples/c05_session_params.py
-          python examples/c06_local_config.py
-          python examples/c07_profiling.py
-          python examples/c08_snapshot_transactions.py
-          python examples/c09_script_output.py
-          python examples/c10_overload.py
-          python examples/c11_quote_ident.py
-          python examples/c12_thread_safety.py
-          python examples/c13_dsn_parsing.py
-          python examples/c14_http_proxy.py
-          python examples/c15_garbage_collection.py
-          python examples/j01_rapidjson.py
-          python examples/j02_ujson.py
-          python examples/j03_orjson.py
+          docker kill db_container_test
diff --git a/.github/workflows/ssl_cert.yml b/.github/workflows/ssl_cert.yml
@@ -1,57 +1,43 @@
 name: SSL certificate test
 
 on:
- push:
-  paths:
-    - "**.py"
-    - "**.yml"
-  branches:
-    - master
-  pull_request:
-
-  workflow_dispatch:
+  workflow_call:
 
 jobs:
   ssl_cert:
+    name: Build SSL-Cert
+    # For all containers with versions 7.1.x and older versions 8.y.z of docker-db,
+    # there will be a crash if UDFs (User Defined Functions) are utilized.
+    runs-on: ubuntu-20.04
+    timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - python-version: "3.11"
-            exasol-tag: 7.1.6
-    name: "Build SSL-Cert with Python ${{ matrix.python-version }} and Exasol ${{ matrix.exasol-tag }}"
-
-    runs-on: ubuntu-20.04
-    timeout-minutes: 15
-
+        python-version: ["3.11"]
     steps:
-      - name: "Checkout repo (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
+      - name: "Checkout repo"
         uses: actions/checkout@v2
 
-      - name: "Setup Python (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        uses: actions/setup-python@v2
+      - name: "Setup Python & Poetry Environment"
+        uses: exasol/python-toolbox/.github/actions/python[email protected]
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: "Install dependencies (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        run: pip install -e .[pandas]
-
-      - name: "Checkout test environment (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        uses: actions/checkout@v2
-        with:
-          repository: exasol/integration-test-docker-environment
-          path: './test_env'
-          ref: '1.7.1'
+      - name: "Install all extras"
+        run: poetry install --all-extras
 
-      - name: "Spawn EXASOL environemnt (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        run: ./start-test-env spawn-test-environment --environment-name ssl --docker-db-image-version=${{ matrix.exasol-tag }} --database-port-forward 8888 --create-certificates
-        working-directory: ./test_env
+      - name: "Start test db"
+        run: >
+          poetry run itde spawn-test-environment --environment-name ssl
+          --database-port-forward 8888 --bucketfs-port-forward 2580 
+          --docker-db-image-version 7.1.17 --db-mem-size 4GB
+          --create-certificates
 
-      - name: "Pull certificate from container (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
+      - name: "Pull certificate from container"
         run: docker cp db_container_ssl:/certificates/rootCA.crt .
 
-      - name: "Setup custom host (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
+      - name: "Setup custom host"
         run: echo "127.0.0.1 exasol-test-database" | sudo tee -a /etc/hosts
 
-      - name: "Run test (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        run: python examples/_ssl_test.py
+      - name: "Run test"
+        run: poetry run python examples/_ssl_test.py
diff --git a/.gitignore b/.gitignore
@@ -16,3 +16,5 @@ build/
 .vscode
 
 .DS_Store
+
+.build_output/
diff --git a/examples/_config.py b/examples/_config.py
@@ -3,4 +3,4 @@
 dsn = os.environ.get('EXAHOST', 'localhost:8563')
 user = os.environ.get('EXAUID', 'SYS')
 password = os.environ.get('EXAPWD', 'exasol')
-schema = os.environ.get('EXASCHEMA', 'PYEXASOL_TEST')
+schema = os.environ.get('EXASCHEMA', 'TEST')
diff --git a/noxfile.py b/noxfile.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 from pathlib import Path
 from typing import Iterable
+from contextlib import contextmanager
 import nox
 from nox import Session
 
@@ -9,7 +10,7 @@
     "integration_tests",
 ]
 
-_ROOT : Path = Path(__file__).parent
+_ROOT: Path = Path(__file__).parent
 
 
 def _test_command(path: Path) -> Iterable[str]:
@@ -28,6 +29,41 @@ def _integration_tests(session: Session) -> None:
     session.run(*command)
 
 
+@contextmanager
+def test_db(session: Session, db_version: str, port: int):
+    with_db = db_version not in ["", None]
+
+    def nop():
+        pass
+
+    def start_db():
+        session.run(
+            "itde",
+            "spawn-test-environment",
+            "--environment-name",
+            "test",
+            "--database-port-forward",
+            f"{port}",
+            "--bucketfs-port-forward",
+            "2580",
+            "--docker-db-image-version",
+            db_version,
+            "--db-mem-size",
+            "4GB",
+            external=True,
+        )
+
+    def stop_db():
+        session.run("docker", "kill", "db_container_test", external=True)
+
+    start = start_db if with_db else nop
+    stop = stop_db if with_db else nop
+
+    start()
+    yield
+    stop()
+
+
 @nox.session(name="unit-tests", python=False)
 def unit_tests(session: Session) -> None:
     """Runs all unit tests"""
@@ -37,11 +73,12 @@ def unit_tests(session: Session) -> None:
 @nox.session(name="integration-tests", python=False)
 def integration_tests(session: Session) -> None:
     """Runs the all integration tests"""
-    _integration_tests(session)
+    with test_db(session, db_version="7.1.17", port=8563):
+        _integration_tests(session)
 
 
-@nox.session(name="all-tests", python=False)
+@ nox.session(name="all-tests", python=False)
 def all_tests(session: Session) -> None:
     """Runs all tests (Unit and Integration)"""
-    command = _test_command(_ROOT / "test")
+    command=_test_command(_ROOT / "test")
     session.run(*command)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -16,3 +16,5 @@ build/
		.vscode

		.DS_Store

		.build_output/