feat: lab 3307 aau i use the kilillm functions to import llm data sta… #4016
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # several test modes: | |
| # - MAIN_STAGING: main branch against STAGING | |
| name: E2E tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| runNotebookTests: | |
| description: "Run notebook tests?" | |
| required: true | |
| type: boolean | |
| default: true | |
| push: | |
| branches: | |
| - "release/**" | |
| - main | |
| schedule: | |
| - cron: "0 7,15 * * 1-5" # MAIN_STAGING | |
| jobs: | |
| test_setup: | |
| name: E2E tests setup | |
| if: github.repository == 'kili-technology/kili-python-sdk' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test_mode_array: ${{ steps.setup_step.outputs.TEST_MODE_ARRAY }} | |
| steps: | |
| - name: Setup test mode | |
| id: setup_step | |
| shell: bash | |
| run: | | |
| if [[ "${{ github.event_name }}" == "push" ]]; then | |
| if [[ "${{ github.ref_name }}" == release/* ]]; then | |
| # if the push was on a release branch, it can be either | |
| # the lts branch -> LTS_CYCLE_FIRST_SDK_STAGING mode | |
| # or the latest release branch -> RELEASE_PREPROD mode | |
| if [[ "${{ github.ref_name }}" == release/2.146.0 ]]; then | |
| json_str="{\"array\": [\"LTS_CYCLE_FIRST_SDK_STAGING\"]}" | |
| else | |
| json_str="{\"array\": [\"RELEASE_PREPROD\"]}" | |
| fi | |
| else | |
| json_str="{\"array\": [\"MAIN_STAGING\"]}" | |
| fi | |
| elif [[ "${{ github.event_name }}" == "schedule" ]]; then | |
| if [[ "${{ github.event.schedule }}" == "0 7,15 * * 1-5" ]]; then | |
| json_str="{\"array\": [\"MAIN_STAGING\"]}" | |
| elif [[ "${{ github.event.schedule }}" == "0 9 * * 1-5" ]]; then | |
| json_str="{\"array\": [\"MAIN_LTS\"]}" | |
| elif [[ "${{ github.event.schedule }}" == "0 11 * * 1-5" ]]; then | |
| json_str="{\"array\": [\"PREV_RELEASE_TESTS_STAGING\"]}" | |
| elif [[ "${{ github.event.schedule }}" == "0 13 * * 1-5" ]]; then | |
| json_str="{\"array\": [\"LTS_CYCLE_FIRST_SDK_STAGING\"]}" | |
| fi | |
| fi | |
| echo "TEST_MODE_ARRAY=$json_str" >> $GITHUB_OUTPUT | |
| tests: | |
| name: ${{ matrix.test_mode }}, ${{ matrix.os }}, ${{ matrix.python-version }} | |
| if: github.repository == 'kili-technology/kili-python-sdk' | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| test_mode: ${{ fromJson(needs.test_setup.outputs.test_mode_array).array }} | |
| os: | |
| - ubuntu-latest | |
| - windows-latest | |
| include: | |
| - os: ubuntu-latest | |
| python-version: 3.8 | |
| - os: windows-latest | |
| python-version: 3.12 | |
| runs-on: ${{ matrix.os }} | |
| needs: [test_setup] | |
| steps: | |
| - name: Set test mode | |
| shell: bash | |
| run: echo "TEST_MODE=${{ matrix.test_mode }}" >> $GITHUB_ENV | |
| - name: Check that test mode is valid | |
| shell: bash | |
| run: | | |
| if [[ ${{ env.TEST_MODE }} != "MAIN_STAGING" ]] && \ | |
| [[ ${{ env.TEST_MODE }} != "CUSTOM" ]] && \ | |
| [[ ${{ env.TEST_MODE }} != "RELEASE_PREPROD" ]] && \ | |
| [[ ${{ env.TEST_MODE }} != "MAIN_LTS" ]] && \ | |
| [[ ${{ env.TEST_MODE }} != "PREV_RELEASE_TESTS_STAGING" ]] && \ | |
| [[ ${{ env.TEST_MODE }} != "LTS_CYCLE_FIRST_SDK_STAGING" ]]; then | |
| echo "TEST_MODE is not valid: ${{ env.TEST_MODE }}" | |
| exit 1 | |
| fi | |
| - name: Set first SDK version of LTS cycle | |
| if: env.TEST_MODE == 'LTS_CYCLE_FIRST_SDK_STAGING' | |
| shell: bash | |
| run: echo "LTS_CYCLE_FIRST_SDK_REF=release/2.146.0" >> $GITHUB_ENV # this should be updated when a new LTS cycle starts | |
| - name: Add latest release branch to github env | |
| shell: bash | |
| timeout-minutes: 3 | |
| run: | | |
| curl https://raw.githubusercontent.com/kili-technology/kili-python-sdk/main/.github/scripts/add_latest_release_branch_to_github_env.py --output add_latest_release_branch_to_github_env.py | |
| python add_latest_release_branch_to_github_env.py | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set checkout branch/tag and endpoint | |
| shell: bash | |
| run: | | |
| if [[ "${{ env.TEST_MODE }}" == "MAIN_STAGING" ]]; then | |
| echo "REF=main" >> $GITHUB_ENV | |
| echo "TEST_AGAINST=STAGING" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_MODE }}" == "RELEASE_PREPROD" ]]; then | |
| echo "REF=${{ env.LASTEST_RELEASE_BRANCH }}" >> $GITHUB_ENV | |
| echo "TEST_AGAINST=PREPROD" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_MODE }}" == "MAIN_LTS" ]]; then | |
| echo "REF=main" >> $GITHUB_ENV | |
| echo "TEST_AGAINST=LTS" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_MODE }}" == "PREV_RELEASE_TESTS_STAGING" ]]; then | |
| echo "REF=${{ env.LASTEST_RELEASE_BRANCH }}" >> $GITHUB_ENV | |
| echo "TEST_AGAINST=STAGING" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_MODE }}" == "LTS_CYCLE_FIRST_SDK_STAGING" ]]; then | |
| echo "REF=${{ env.LTS_CYCLE_FIRST_SDK_REF }}" >> $GITHUB_ENV | |
| echo "TEST_AGAINST=STAGING" >> $GITHUB_ENV | |
| fi | |
| - name: Set KILI_API_ENDPOINT | |
| shell: bash | |
| run: | | |
| if [[ "${{ env.TEST_AGAINST }}" == "STAGING" ]]; then | |
| echo "KILI_API_ENDPOINT=https://staging.cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_AGAINST }}" == "PREPROD" ]]; then | |
| echo "KILI_API_ENDPOINT=https://preproduction.cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_AGAINST }}" == "LTS" ]]; then | |
| echo "KILI_API_ENDPOINT=https://lts.cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_AGAINST }}" == "PROD" ]]; then | |
| echo "KILI_API_ENDPOINT=https://cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV | |
| else | |
| echo "TEST_AGAINST is not valid: ${{ env.TEST_AGAINST }}" | |
| exit 1 | |
| fi | |
| - name: Check REF is set | |
| shell: bash | |
| run: | | |
| if [[ -z "${{ env.REF }}" ]]; then | |
| echo "REF is not set" | |
| exit 1 | |
| fi | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.REF }} | |
| - name: Delete src for PREV_RELEASE_TESTS_STAGING mode | |
| if: env.TEST_MODE == 'PREV_RELEASE_TESTS_STAGING' | |
| shell: bash | |
| run: rm -rf src # we want the previous release tests but the kili from main | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "pip" | |
| - name: Upgrade pip | |
| run: python -m pip install --upgrade pip | |
| - name: Install kili | |
| shell: bash | |
| run: | | |
| if [[ "${{ env.TEST_MODE }}" == "PREV_RELEASE_TESTS_STAGING" ]]; then | |
| pip install "kili[dev] @ git+https://github.com/kili-technology/kili-python-sdk.git@main" | |
| else | |
| pip install -e ".[dev]" | |
| fi | |
| - name: Get kili SDK version | |
| shell: bash | |
| run: echo "KILI_SDK_VERSION=$(python -c "from kili import __version__; print(__version__)")" >> $GITHUB_ENV | |
| - name: Notebook tests | |
| # we don't run notebook tests on push to main | |
| # we run regardless of the outcome of the e2e tests | |
| if: ${{ !cancelled() && github.event.inputs.runNotebookTests != 'false' && (github.event_name != 'push' || github.ref_name != 'main') && (steps.e2e.outcome != 'cancelled') }} | |
| timeout-minutes: 60 | |
| run: pytest --timeout=600 -ra -sv --color yes --code-highlight yes --durations=0 -vv tests/e2e/test_notebooks.py | |
| env: | |
| KILI_API_CLOUD_VISION: ${{ secrets.KILI_API_CLOUD_VISION }} | |
| KILI_API_ENDPOINT: ${{ env.KILI_API_ENDPOINT }} | |
| KILI_API_KEY: ${{ secrets[format('KILI_USER_API_KEY_{0}', env.TEST_AGAINST)] }} | |
| PYDEVD_DISABLE_FILE_VALIDATION: true | |
| - name: Check that kili SDK version has not changed | |
| shell: bash | |
| run: | | |
| kili_sdk_version=$(python -c "from kili import __version__; print(__version__)") | |
| if [[ "${{ env.KILI_SDK_VERSION }}" != "$kili_sdk_version" ]]; then | |
| echo "Kili has been updated (maybe by notebooks)." | |
| echo "The version should not be updated during tests." | |
| echo "version before: ${{ env.KILI_SDK_VERSION }}" | |
| echo "version after: $kili_sdk_version" | |
| exit 1 | |
| fi | |
| - name: Slack notification if failure | |
| if: failure() && (github.event_name != 'workflow_dispatch') # doesn't notify for manual runs | |
| id: slack | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "text": "Notebook tests '${{ env.TEST_MODE }}' (branch '${{ env.REF }}' against endpoint '${{ env.TEST_AGAINST }}') failed with status '${{ job.status }}'\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Notebook tests '${{ env.TEST_MODE }}' (branch '${{ env.REF }}' against endpoint '${{ env.TEST_AGAINST }}') failed with status '${{ job.status }}'\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_PYTHON_SDK_ALARMING }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |