diff --git a/.github/actions/setup-poetry-env/action.yml b/.github/actions/setup-poetry-env/action.yml index afd21bf..7969e6d 100644 --- a/.github/actions/setup-poetry-env/action.yml +++ b/.github/actions/setup-poetry-env/action.yml @@ -6,12 +6,15 @@ inputs: required: false description: "The python version to use" default: "3.11" + pyspark-version: + required: false + description: "The pyspark version to use" + default: "3.5.1" with-docs: required: false description: "Install the docs dependency group" default: 'false' - runs: using: "composite" steps: @@ -40,7 +43,7 @@ runs: uses: actions/cache@v3 with: path: .venv - key: venv-${{ runner.os }}-${{ inputs.python-version }}-${{ inputs.with-docs }}-${{ hashFiles('poetry.lock') }} + key: venv-${{ runner.os }}-${{ inputs.python-version }}-${{ inputs.pyspark-version }}-${{ inputs.with-docs }}-${{ hashFiles('poetry.lock') }} - name: Install dependencies run: | @@ -49,5 +52,12 @@ runs: else poetry install --no-interaction fi + poetry run pip install pyspark==${{ inputs.pyspark-version }} shell: bash if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + + - name: Print python and pyspark version + run: | + poetry run python --version + poetry run pyspark --version + shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 520bd14..2fc020f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: types: [opened, synchronize, reopened] push: branches: [main] + workflow_dispatch: jobs: quality: @@ -27,6 +28,14 @@ jobs: strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + pyspark-version: ["3.4.3", "3.5.1"] + include: + - python-version: "3.8" + pyspark-version: "3.3.4" + - python-version: "3.9" + pyspark-version: "3.3.4" + - python-version: "3.10" + pyspark-version: "3.3.4" fail-fast: false defaults: run: @@ -39,6 +48,7 @@ jobs: uses: ./.github/actions/setup-poetry-env with: python-version: ${{ matrix.python-version }} + pyspark-version: ${{ matrix.pyspark-version }} - name: Run tests run: poetry run pytest tests