Merge pull request #111 from fpgmaas/chore/test-pyspark

chore: add tests for multiple versions of pyspark
MrPowers · Jul 29, 2024 · d876e24 · d876e24
2 parents 580631a + 81b8b9e
commit d876e24
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 2 deletions.
diff --git a/.github/actions/setup-poetry-env/action.yml b/.github/actions/setup-poetry-env/action.yml
@@ -6,12 +6,15 @@ inputs:
     required: false
     description: "The python version to use"
     default: "3.11"
+  pyspark-version:
+    required: false
+    description: "The pyspark version to use"
+    default: "3.5.1"
   with-docs:
     required: false
     description: "Install the docs dependency group"
     default: 'false'
 
-
 runs:
   using: "composite"
   steps:
@@ -40,7 +43,7 @@ runs:
       uses: actions/cache@v3
       with:
         path: .venv
-        key: venv-${{ runner.os }}-${{ inputs.python-version }}-${{ inputs.with-docs }}-${{ hashFiles('poetry.lock') }}
+        key: venv-${{ runner.os }}-${{ inputs.python-version }}-${{ inputs.pyspark-version }}-${{ inputs.with-docs }}-${{ hashFiles('poetry.lock') }}
 
     - name: Install dependencies
       run: |
@@ -49,5 +52,12 @@ runs:
         else
           poetry install --no-interaction
         fi
+        poetry run pip install pyspark==${{ inputs.pyspark-version }}
       shell: bash
       if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+
+    - name: Print python and pyspark version
+      run: |
+        poetry run python --version
+        poetry run pyspark --version
+      shell: bash
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,6 +5,7 @@ on:
     types: [opened, synchronize, reopened]
   push:
     branches: [main]
+  workflow_dispatch:
 
 jobs:
   quality:
@@ -27,6 +28,14 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        pyspark-version: ["3.4.3", "3.5.1"]
+        include:
+          - python-version: "3.8"
+            pyspark-version: "3.3.4"
+          - python-version: "3.9"
+            pyspark-version: "3.3.4"
+          - python-version: "3.10"
+            pyspark-version: "3.3.4"
       fail-fast: false
     defaults:
       run:
@@ -39,6 +48,7 @@ jobs:
         uses: ./.github/actions/setup-poetry-env
         with:
           python-version: ${{ matrix.python-version }}
+          pyspark-version: ${{ matrix.pyspark-version }}
 
       - name: Run tests
         run: poetry run pytest tests