From 0ca64c7d458c2eea23586b291449458d0c479128 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 8 May 2024 10:10:32 +0100 Subject: [PATCH 01/14] adding pyflow in the dependency tree --- dependency_tree.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dependency_tree.yml b/dependency_tree.yml index 8ff565c..e315723 100644 --- a/dependency_tree.yml +++ b/dependency_tree.yml @@ -199,6 +199,14 @@ pyfdb: {"type":"local","engine":"toc","schema":"${{ github.workspace }}/tests/default_fdb_schema","spaces":[{"handler":"Default","roots":[{"path":"${{ github.workspace }}/data/fdb"}]}]} mkdir: - data/fdb + +pyflow: + type: python + master_branch: master + requirements_path: requirements.txt + deps: + - ecflow + pyodc: type: python deps: From 12a6dc27fea472f05a9bfc2f40053bbbd85531bb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 May 2024 09:11:34 +0000 Subject: [PATCH 02/14] Build workflows --- .github/workflows/downstream-ci-hpc.yml | 39 +++++++++++++++++++++ .github/workflows/downstream-ci.yml | 45 +++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/.github/workflows/downstream-ci-hpc.yml b/.github/workflows/downstream-ci-hpc.yml index b992461..2e3f3e7 100644 --- a/.github/workflows/downstream-ci-hpc.yml +++ b/.github/workflows/downstream-ci-hpc.yml @@ -78,6 +78,9 @@ name: downstream-ci-hpc pyfdb: required: false type: string + pyflow: + required: false + type: string pyodc: required: false type: string @@ -148,6 +151,8 @@ jobs: plume_matrix: ${{ steps.setup.outputs.plume }} pyfdb: ${{ steps.prepare-inputs.outputs.pyfdb }} pyfdb_matrix: ${{ steps.setup.outputs.pyfdb }} + pyflow: ${{ steps.prepare-inputs.outputs.pyflow }} + pyflow_matrix: ${{ steps.setup.outputs.pyflow }} pyodc: ${{ steps.prepare-inputs.outputs.pyodc }} pyodc_matrix: ${{ steps.setup.outputs.pyodc }} skinnywms: ${{ steps.prepare-inputs.outputs.skinnywms }} @@ -186,6 +191,7 @@ jobs: echo pdbufr="${{ inputs.pdbufr }}" >> $GITHUB_OUTPUT echo plume="${{ inputs.plume }}" >> $GITHUB_OUTPUT echo pyfdb="${{ inputs.pyfdb }}" >> $GITHUB_OUTPUT + echo pyflow="${{ inputs.pyflow }}" >> $GITHUB_OUTPUT echo pyodc="${{ inputs.pyodc }}" >> $GITHUB_OUTPUT echo skinnywms="${{ inputs.skinnywms }}" >> $GITHUB_OUTPUT echo thermofeel="${{ inputs.thermofeel }}" >> $GITHUB_OUTPUT @@ -362,6 +368,13 @@ jobs: develop_branch: develop input: ${{ steps.prepare-inputs.outputs.pyfdb }} optional_matrix: null + ecmwf/pyflow: + path: .github/ci-hpc-config.yml + python: true + master_branch: master + develop_branch: develop + input: ${{ steps.prepare-inputs.outputs.pyflow }} + optional_matrix: null ecmwf/pyodc: path: .github/ci-hpc-config.yml python: true @@ -1112,6 +1125,32 @@ jobs: ${{ needs.setup.outputs.eccodes }} ${{ needs.setup.outputs.eckit }} python_dependencies: '' + pyflow: + name: pyflow + needs: + - ecflow + - setup + if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow_matrix && (needs.setup.outputs.ecflow || needs.setup.outputs.pyflow) }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.pyflow_matrix) }} + env: + DEP_TREE: ${{ needs.setup.outputs.dep_tree }} + runs-on: + - self-hosted + - linux + - hpc + steps: + - uses: ecmwf-actions/reusable-workflows/ci-hpc@v2 + with: + github_user: ${{ secrets.BUILD_PACKAGE_HPC_GITHUB_USER }} + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + troika_user: ${{ secrets.HPC_CI_SSH_USER }} + repository: ${{ matrix.owner_repo_ref }} + build_config: ${{ matrix.config_path }} + dependencies: ${{ needs.setup.outputs.ecflow }} + python_dependencies: '' + python_requirements: requirements.txt pyodc: name: pyodc needs: diff --git a/.github/workflows/downstream-ci.yml b/.github/workflows/downstream-ci.yml index e58aeca..6302a55 100644 --- a/.github/workflows/downstream-ci.yml +++ b/.github/workflows/downstream-ci.yml @@ -82,6 +82,9 @@ name: downstream-ci pyfdb: required: false type: string + pyflow: + required: false + type: string pyodc: required: false type: string @@ -164,6 +167,8 @@ jobs: plume_matrix: ${{ steps.setup.outputs.plume }} pyfdb: ${{ steps.prepare-inputs.outputs.pyfdb }} pyfdb_matrix: ${{ steps.setup.outputs.pyfdb }} + pyflow: ${{ steps.prepare-inputs.outputs.pyflow }} + pyflow_matrix: ${{ steps.setup.outputs.pyflow }} pyodc: ${{ steps.prepare-inputs.outputs.pyodc }} pyodc_matrix: ${{ steps.setup.outputs.pyodc }} skinnywms: ${{ steps.prepare-inputs.outputs.skinnywms }} @@ -204,6 +209,7 @@ jobs: echo pdbufr="${{ inputs.pdbufr }}" >> $GITHUB_OUTPUT echo plume="${{ inputs.plume }}" >> $GITHUB_OUTPUT echo pyfdb="${{ inputs.pyfdb }}" >> $GITHUB_OUTPUT + echo pyflow="${{ inputs.pyflow }}" >> $GITHUB_OUTPUT echo pyodc="${{ inputs.pyodc }}" >> $GITHUB_OUTPUT echo skinnywms="${{ inputs.skinnywms }}" >> $GITHUB_OUTPUT echo thermofeel="${{ inputs.thermofeel }}" >> $GITHUB_OUTPUT @@ -380,6 +386,13 @@ jobs: develop_branch: develop input: ${{ steps.prepare-inputs.outputs.pyfdb }} optional_matrix: null + ecmwf/pyflow: + path: .github/ci-config.yml + python: true + master_branch: master + develop_branch: develop + input: ${{ steps.prepare-inputs.outputs.pyflow }} + optional_matrix: null ecmwf/pyodc: path: .github/ci-config.yml python: true @@ -1268,6 +1281,38 @@ jobs: github_token: ${{ secrets.GH_REPO_READ_TOKEN }} codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} codecov_token: ${{ secrets.CODECOV_UPLOAD_TOKEN }} + pyflow: + name: pyflow + needs: + - ecflow + - setup + - python-qa + if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow_matrix && (needs.setup.outputs.ecflow || needs.setup.outputs.pyflow) }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.pyflow_matrix) }} + env: + DEP_TREE: ${{ needs.setup.outputs.dep_tree }} + runs-on: ${{ matrix.labels }} + steps: + - name: Build dependencies + id: build-deps + uses: ecmwf-actions/reusable-workflows/build-package-with-config@v2 + with: + repository: ${{ matrix.owner_repo_ref }} + codecov_upload: false + build_package_inputs: 'repository: ${{ matrix.owner_repo_ref }}' + build_config: ${{ matrix.config_path }} + build_dependencies: ${{ needs.setup.outputs.ecflow }} + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + with: + lib_path: ${{ steps.build-deps.outputs.lib_path }} + python_dependencies: '' + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + requirements_path: requirements.txt + codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} + codecov_token: ${{ secrets.CODECOV_UPLOAD_TOKEN }} pyodc: name: pyodc needs: From 0236900aff9151232dccdb014d295ff1c072812a Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 8 May 2024 10:12:32 +0100 Subject: [PATCH 03/14] using default requirements for pyflow --- dependency_tree.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/dependency_tree.yml b/dependency_tree.yml index e315723..2a5c4d7 100644 --- a/dependency_tree.yml +++ b/dependency_tree.yml @@ -203,7 +203,6 @@ pyfdb: pyflow: type: python master_branch: master - requirements_path: requirements.txt deps: - ecflow From 98612790fb47e7ed80509f40c481cdcbafada3dd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 May 2024 09:13:59 +0000 Subject: [PATCH 04/14] Build workflows --- .github/workflows/downstream-ci-hpc.yml | 1 - .github/workflows/downstream-ci.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/downstream-ci-hpc.yml b/.github/workflows/downstream-ci-hpc.yml index 2e3f3e7..b8b884a 100644 --- a/.github/workflows/downstream-ci-hpc.yml +++ b/.github/workflows/downstream-ci-hpc.yml @@ -1150,7 +1150,6 @@ jobs: build_config: ${{ matrix.config_path }} dependencies: ${{ needs.setup.outputs.ecflow }} python_dependencies: '' - python_requirements: requirements.txt pyodc: name: pyodc needs: diff --git a/.github/workflows/downstream-ci.yml b/.github/workflows/downstream-ci.yml index 6302a55..fc73928 100644 --- a/.github/workflows/downstream-ci.yml +++ b/.github/workflows/downstream-ci.yml @@ -1310,7 +1310,6 @@ jobs: lib_path: ${{ steps.build-deps.outputs.lib_path }} python_dependencies: '' github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - requirements_path: requirements.txt codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} codecov_token: ${{ secrets.CODECOV_UPLOAD_TOKEN }} pyodc: From c544948e70790d232ac130a105e83f42e11dbdbc Mon Sep 17 00:00:00 2001 From: Dusan Figala Date: Wed, 8 May 2024 13:14:55 +0200 Subject: [PATCH 05/14] Test with ECFLOW_DIR variable --- dependency_tree.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dependency_tree.yml b/dependency_tree.yml index 2a5c4d7..7af7d26 100644 --- a/dependency_tree.yml +++ b/dependency_tree.yml @@ -205,6 +205,9 @@ pyflow: master_branch: master deps: - ecflow + downstream-ci: + env: + ECFLOW_DIR: /opt/actions-runner/work/_work/_temp/install/ecflow pyodc: type: python From 61f7803e06a7279779e11da4f9610e6b6b04030a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 May 2024 11:15:18 +0000 Subject: [PATCH 06/14] Build workflows --- .github/workflows/downstream-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/downstream-ci.yml b/.github/workflows/downstream-ci.yml index fc73928..c8e4361 100644 --- a/.github/workflows/downstream-ci.yml +++ b/.github/workflows/downstream-ci.yml @@ -1293,6 +1293,7 @@ jobs: matrix: ${{ fromJson(needs.setup.outputs.pyflow_matrix) }} env: DEP_TREE: ${{ needs.setup.outputs.dep_tree }} + ECFLOW_DIR: /opt/actions-runner/work/_work/_temp/install/ecflow runs-on: ${{ matrix.labels }} steps: - name: Build dependencies From fe308c003fc53b8b30b5aa576730dd010ef8b8e0 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 8 May 2024 13:25:46 +0100 Subject: [PATCH 07/14] adding wellies --- dependency_tree.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dependency_tree.yml b/dependency_tree.yml index 7af7d26..0c98817 100644 --- a/dependency_tree.yml +++ b/dependency_tree.yml @@ -205,9 +205,14 @@ pyflow: master_branch: master deps: - ecflow - downstream-ci: - env: - ECFLOW_DIR: /opt/actions-runner/work/_work/_temp/install/ecflow + # downstream-ci: + # env: + # ECFLOW_DIR: /opt/actions-runner/work/_work/_temp/install/ecflow + +pyflow-wellies: + type: python + deps: + - pyflow pyodc: type: python From 69d8c216536510abaad56bb3d2df659aee956fcb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 May 2024 12:29:23 +0000 Subject: [PATCH 08/14] Build workflows --- .github/workflows/downstream-ci-hpc.yml | 39 +++++++++++++++++++++ .github/workflows/downstream-ci.yml | 46 ++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/.github/workflows/downstream-ci-hpc.yml b/.github/workflows/downstream-ci-hpc.yml index b8b884a..71736d4 100644 --- a/.github/workflows/downstream-ci-hpc.yml +++ b/.github/workflows/downstream-ci-hpc.yml @@ -81,6 +81,9 @@ name: downstream-ci-hpc pyflow: required: false type: string + pyflow-wellies: + required: false + type: string pyodc: required: false type: string @@ -153,6 +156,8 @@ jobs: pyfdb_matrix: ${{ steps.setup.outputs.pyfdb }} pyflow: ${{ steps.prepare-inputs.outputs.pyflow }} pyflow_matrix: ${{ steps.setup.outputs.pyflow }} + pyflow-wellies: ${{ steps.prepare-inputs.outputs.pyflow-wellies }} + pyflow-wellies_matrix: ${{ steps.setup.outputs.pyflow-wellies }} pyodc: ${{ steps.prepare-inputs.outputs.pyodc }} pyodc_matrix: ${{ steps.setup.outputs.pyodc }} skinnywms: ${{ steps.prepare-inputs.outputs.skinnywms }} @@ -192,6 +197,7 @@ jobs: echo plume="${{ inputs.plume }}" >> $GITHUB_OUTPUT echo pyfdb="${{ inputs.pyfdb }}" >> $GITHUB_OUTPUT echo pyflow="${{ inputs.pyflow }}" >> $GITHUB_OUTPUT + echo pyflow-wellies="${{ inputs.pyflow-wellies }}" >> $GITHUB_OUTPUT echo pyodc="${{ inputs.pyodc }}" >> $GITHUB_OUTPUT echo skinnywms="${{ inputs.skinnywms }}" >> $GITHUB_OUTPUT echo thermofeel="${{ inputs.thermofeel }}" >> $GITHUB_OUTPUT @@ -375,6 +381,13 @@ jobs: develop_branch: develop input: ${{ steps.prepare-inputs.outputs.pyflow }} optional_matrix: null + ecmwf/pyflow-wellies: + path: .github/ci-hpc-config.yml + python: true + master_branch: master + develop_branch: develop + input: ${{ steps.prepare-inputs.outputs.pyflow-wellies }} + optional_matrix: null ecmwf/pyodc: path: .github/ci-hpc-config.yml python: true @@ -1150,6 +1163,32 @@ jobs: build_config: ${{ matrix.config_path }} dependencies: ${{ needs.setup.outputs.ecflow }} python_dependencies: '' + pyflow-wellies: + name: pyflow-wellies + needs: + - pyflow + - ecflow + - setup + if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow-wellies_matrix && (needs.setup.outputs.pyflow || needs.setup.outputs.ecflow || needs.setup.outputs.pyflow-wellies) }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.pyflow-wellies_matrix) }} + env: + DEP_TREE: ${{ needs.setup.outputs.dep_tree }} + runs-on: + - self-hosted + - linux + - hpc + steps: + - uses: ecmwf-actions/reusable-workflows/ci-hpc@v2 + with: + github_user: ${{ secrets.BUILD_PACKAGE_HPC_GITHUB_USER }} + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + troika_user: ${{ secrets.HPC_CI_SSH_USER }} + repository: ${{ matrix.owner_repo_ref }} + build_config: ${{ matrix.config_path }} + dependencies: ${{ needs.setup.outputs.ecflow }} + python_dependencies: ${{ needs.setup.outputs.pyflow }} pyodc: name: pyodc needs: diff --git a/.github/workflows/downstream-ci.yml b/.github/workflows/downstream-ci.yml index c8e4361..f4e578f 100644 --- a/.github/workflows/downstream-ci.yml +++ b/.github/workflows/downstream-ci.yml @@ -85,6 +85,9 @@ name: downstream-ci pyflow: required: false type: string + pyflow-wellies: + required: false + type: string pyodc: required: false type: string @@ -169,6 +172,8 @@ jobs: pyfdb_matrix: ${{ steps.setup.outputs.pyfdb }} pyflow: ${{ steps.prepare-inputs.outputs.pyflow }} pyflow_matrix: ${{ steps.setup.outputs.pyflow }} + pyflow-wellies: ${{ steps.prepare-inputs.outputs.pyflow-wellies }} + pyflow-wellies_matrix: ${{ steps.setup.outputs.pyflow-wellies }} pyodc: ${{ steps.prepare-inputs.outputs.pyodc }} pyodc_matrix: ${{ steps.setup.outputs.pyodc }} skinnywms: ${{ steps.prepare-inputs.outputs.skinnywms }} @@ -210,6 +215,7 @@ jobs: echo plume="${{ inputs.plume }}" >> $GITHUB_OUTPUT echo pyfdb="${{ inputs.pyfdb }}" >> $GITHUB_OUTPUT echo pyflow="${{ inputs.pyflow }}" >> $GITHUB_OUTPUT + echo pyflow-wellies="${{ inputs.pyflow-wellies }}" >> $GITHUB_OUTPUT echo pyodc="${{ inputs.pyodc }}" >> $GITHUB_OUTPUT echo skinnywms="${{ inputs.skinnywms }}" >> $GITHUB_OUTPUT echo thermofeel="${{ inputs.thermofeel }}" >> $GITHUB_OUTPUT @@ -393,6 +399,13 @@ jobs: develop_branch: develop input: ${{ steps.prepare-inputs.outputs.pyflow }} optional_matrix: null + ecmwf/pyflow-wellies: + path: .github/ci-config.yml + python: true + master_branch: master + develop_branch: develop + input: ${{ steps.prepare-inputs.outputs.pyflow-wellies }} + optional_matrix: null ecmwf/pyodc: path: .github/ci-config.yml python: true @@ -1293,7 +1306,6 @@ jobs: matrix: ${{ fromJson(needs.setup.outputs.pyflow_matrix) }} env: DEP_TREE: ${{ needs.setup.outputs.dep_tree }} - ECFLOW_DIR: /opt/actions-runner/work/_work/_temp/install/ecflow runs-on: ${{ matrix.labels }} steps: - name: Build dependencies @@ -1313,6 +1325,38 @@ jobs: github_token: ${{ secrets.GH_REPO_READ_TOKEN }} codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} codecov_token: ${{ secrets.CODECOV_UPLOAD_TOKEN }} + pyflow-wellies: + name: pyflow-wellies + needs: + - pyflow + - ecflow + - setup + - python-qa + if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow-wellies_matrix && (needs.setup.outputs.pyflow || needs.setup.outputs.ecflow || needs.setup.outputs.pyflow-wellies) }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.pyflow-wellies_matrix) }} + env: + DEP_TREE: ${{ needs.setup.outputs.dep_tree }} + runs-on: ${{ matrix.labels }} + steps: + - name: Build dependencies + id: build-deps + uses: ecmwf-actions/reusable-workflows/build-package-with-config@v2 + with: + repository: ${{ matrix.owner_repo_ref }} + codecov_upload: false + build_package_inputs: 'repository: ${{ matrix.owner_repo_ref }}' + build_config: ${{ matrix.config_path }} + build_dependencies: ${{ needs.setup.outputs.ecflow }} + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + with: + lib_path: ${{ steps.build-deps.outputs.lib_path }} + python_dependencies: ${{ needs.setup.outputs.pyflow }} + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} + codecov_token: ${{ secrets.CODECOV_UPLOAD_TOKEN }} pyodc: name: pyodc needs: From aac9d6143d0bc20b19853bc2439ca29526355ad3 Mon Sep 17 00:00:00 2001 From: Iain Russell <40060766+iainrussell@users.noreply.github.com> Date: Wed, 8 May 2024 15:45:41 +0100 Subject: [PATCH 09/14] Pass bin_paths to ci-python in generate-workflows.py --- generate-workflows.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/generate-workflows.py b/generate-workflows.py index f3ea7e2..2a92346 100644 --- a/generate-workflows.py +++ b/generate-workflows.py @@ -377,6 +377,9 @@ def generate_package_jobs(self, dep_tree: dict): "lib_path": ( "${{ steps.build-deps.outputs.lib_path }}" ), + "bin_paths": ( + "${{ steps.build-deps.outputs.bin_paths }}" + ), "python_dependencies": "\n".join(python_deps), "github_token": "${{ secrets.GH_REPO_READ_TOKEN }}", }, From 42727a2c5479bea2c797550a23261d1e85e7a19c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 May 2024 14:45:56 +0000 Subject: [PATCH 10/14] Build workflows --- .github/workflows/downstream-ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/downstream-ci.yml b/.github/workflows/downstream-ci.yml index f4e578f..47af0db 100644 --- a/.github/workflows/downstream-ci.yml +++ b/.github/workflows/downstream-ci.yml @@ -674,6 +674,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: ${{ needs.setup.outputs.eccodes-python }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} requirements_path: ci/requirements-tests.in @@ -720,6 +721,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: |- ${{ needs.setup.outputs.earthkit-data }} ${{ needs.setup.outputs.cfgrib }} @@ -774,6 +776,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: |- ${{ needs.setup.outputs.cfgrib }} ${{ needs.setup.outputs.multiurl }} @@ -904,6 +907,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: '' github_token: ${{ secrets.GH_REPO_READ_TOKEN }} codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} @@ -1216,6 +1220,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: ${{ needs.setup.outputs.eccodes-python }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} requirements_path: tests/downstream-ci-requirements.txt @@ -1290,6 +1295,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: '' github_token: ${{ secrets.GH_REPO_READ_TOKEN }} codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} @@ -1321,6 +1327,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: '' github_token: ${{ secrets.GH_REPO_READ_TOKEN }} codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} @@ -1353,6 +1360,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: ${{ needs.setup.outputs.pyflow }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} @@ -1387,6 +1395,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: '' github_token: ${{ secrets.GH_REPO_READ_TOKEN }} codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} From afd2c6d097556c3a17ef32af527c62c94b9bdf84 Mon Sep 17 00:00:00 2001 From: Iain Russell <40060766+iainrussell@users.noreply.github.com> Date: Wed, 8 May 2024 15:53:09 +0100 Subject: [PATCH 11/14] Test ci-python v2.21 --- generate-workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate-workflows.py b/generate-workflows.py index 2a92346..d0bdfeb 100644 --- a/generate-workflows.py +++ b/generate-workflows.py @@ -372,7 +372,7 @@ def generate_package_jobs(self, dep_tree: dict): for path in mkdir: steps.append({"run": f"mkdir -p {path}"}) ci_python_step = { - "uses": "ecmwf-actions/reusable-workflows/ci-python@v2", + "uses": "ecmwf-actions/reusable-workflows/ci-python@v2.21", "with": { "lib_path": ( "${{ steps.build-deps.outputs.lib_path }}" From 0b70c102a5eb1a1a2996fb7198fe4597223f07e1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 May 2024 14:53:21 +0000 Subject: [PATCH 12/14] Build workflows --- .github/workflows/downstream-ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/downstream-ci.yml b/.github/workflows/downstream-ci.yml index 47af0db..e95753e 100644 --- a/.github/workflows/downstream-ci.yml +++ b/.github/workflows/downstream-ci.yml @@ -671,7 +671,7 @@ jobs: build_config: ${{ matrix.config_path }} build_dependencies: ${{ needs.setup.outputs.eccodes }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -718,7 +718,7 @@ jobs: ${{ needs.setup.outputs.odc }} ${{ needs.setup.outputs.eckit }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -773,7 +773,7 @@ jobs: ${{ needs.setup.outputs.odc }} ${{ needs.setup.outputs.eckit }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -904,7 +904,7 @@ jobs: build_config: ${{ matrix.config_path }} build_dependencies: ${{ needs.setup.outputs.eccodes }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -1217,7 +1217,7 @@ jobs: build_config: ${{ matrix.config_path }} build_dependencies: ${{ needs.setup.outputs.eccodes }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -1292,7 +1292,7 @@ jobs: ${{ needs.setup.outputs.eckit }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - run: mkdir -p data/fdb - - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -1324,7 +1324,7 @@ jobs: build_config: ${{ matrix.config_path }} build_dependencies: ${{ needs.setup.outputs.ecflow }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -1357,7 +1357,7 @@ jobs: build_config: ${{ matrix.config_path }} build_dependencies: ${{ needs.setup.outputs.ecflow }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -1392,7 +1392,7 @@ jobs: ${{ needs.setup.outputs.odc }} ${{ needs.setup.outputs.eckit }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} From 1cf4ec84ff3b212251554eab2137f0735211bad2 Mon Sep 17 00:00:00 2001 From: Iain Russell <40060766+iainrussell@users.noreply.github.com> Date: Wed, 8 May 2024 16:12:17 +0100 Subject: [PATCH 13/14] Back to ci-python v2 --- generate-workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate-workflows.py b/generate-workflows.py index d0bdfeb..2a92346 100644 --- a/generate-workflows.py +++ b/generate-workflows.py @@ -372,7 +372,7 @@ def generate_package_jobs(self, dep_tree: dict): for path in mkdir: steps.append({"run": f"mkdir -p {path}"}) ci_python_step = { - "uses": "ecmwf-actions/reusable-workflows/ci-python@v2.21", + "uses": "ecmwf-actions/reusable-workflows/ci-python@v2", "with": { "lib_path": ( "${{ steps.build-deps.outputs.lib_path }}" From d0182156a1978daab0f619a069cb6a21092fca73 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 May 2024 15:12:30 +0000 Subject: [PATCH 14/14] Build workflows --- .github/workflows/downstream-ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/downstream-ci.yml b/.github/workflows/downstream-ci.yml index e95753e..47af0db 100644 --- a/.github/workflows/downstream-ci.yml +++ b/.github/workflows/downstream-ci.yml @@ -671,7 +671,7 @@ jobs: build_config: ${{ matrix.config_path }} build_dependencies: ${{ needs.setup.outputs.eccodes }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -718,7 +718,7 @@ jobs: ${{ needs.setup.outputs.odc }} ${{ needs.setup.outputs.eckit }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -773,7 +773,7 @@ jobs: ${{ needs.setup.outputs.odc }} ${{ needs.setup.outputs.eckit }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -904,7 +904,7 @@ jobs: build_config: ${{ matrix.config_path }} build_dependencies: ${{ needs.setup.outputs.eccodes }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -1217,7 +1217,7 @@ jobs: build_config: ${{ matrix.config_path }} build_dependencies: ${{ needs.setup.outputs.eccodes }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -1292,7 +1292,7 @@ jobs: ${{ needs.setup.outputs.eckit }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - run: mkdir -p data/fdb - - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -1324,7 +1324,7 @@ jobs: build_config: ${{ matrix.config_path }} build_dependencies: ${{ needs.setup.outputs.ecflow }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -1357,7 +1357,7 @@ jobs: build_config: ${{ matrix.config_path }} build_dependencies: ${{ needs.setup.outputs.ecflow }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }} @@ -1392,7 +1392,7 @@ jobs: ${{ needs.setup.outputs.odc }} ${{ needs.setup.outputs.eckit }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - - uses: ecmwf-actions/reusable-workflows/ci-python@v2.21 + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} bin_paths: ${{ steps.build-deps.outputs.bin_paths }}