diff --git a/.github/workflows/downstream-ci-hpc.yml b/.github/workflows/downstream-ci-hpc.yml index b992461..71736d4 100644 --- a/.github/workflows/downstream-ci-hpc.yml +++ b/.github/workflows/downstream-ci-hpc.yml @@ -78,6 +78,12 @@ name: downstream-ci-hpc pyfdb: required: false type: string + pyflow: + required: false + type: string + pyflow-wellies: + required: false + type: string pyodc: required: false type: string @@ -148,6 +154,10 @@ jobs: plume_matrix: ${{ steps.setup.outputs.plume }} pyfdb: ${{ steps.prepare-inputs.outputs.pyfdb }} pyfdb_matrix: ${{ steps.setup.outputs.pyfdb }} + pyflow: ${{ steps.prepare-inputs.outputs.pyflow }} + pyflow_matrix: ${{ steps.setup.outputs.pyflow }} + pyflow-wellies: ${{ steps.prepare-inputs.outputs.pyflow-wellies }} + pyflow-wellies_matrix: ${{ steps.setup.outputs.pyflow-wellies }} pyodc: ${{ steps.prepare-inputs.outputs.pyodc }} pyodc_matrix: ${{ steps.setup.outputs.pyodc }} skinnywms: ${{ steps.prepare-inputs.outputs.skinnywms }} @@ -186,6 +196,8 @@ jobs: echo pdbufr="${{ inputs.pdbufr }}" >> $GITHUB_OUTPUT echo plume="${{ inputs.plume }}" >> $GITHUB_OUTPUT echo pyfdb="${{ inputs.pyfdb }}" >> $GITHUB_OUTPUT + echo pyflow="${{ inputs.pyflow }}" >> $GITHUB_OUTPUT + echo pyflow-wellies="${{ inputs.pyflow-wellies }}" >> $GITHUB_OUTPUT echo pyodc="${{ inputs.pyodc }}" >> $GITHUB_OUTPUT echo skinnywms="${{ inputs.skinnywms }}" >> $GITHUB_OUTPUT echo thermofeel="${{ inputs.thermofeel }}" >> $GITHUB_OUTPUT @@ -362,6 +374,20 @@ jobs: develop_branch: develop input: ${{ steps.prepare-inputs.outputs.pyfdb }} optional_matrix: null + ecmwf/pyflow: + path: .github/ci-hpc-config.yml + python: true + master_branch: master + develop_branch: develop + input: ${{ steps.prepare-inputs.outputs.pyflow }} + optional_matrix: null + ecmwf/pyflow-wellies: + path: .github/ci-hpc-config.yml + python: true + master_branch: master + develop_branch: develop + input: ${{ steps.prepare-inputs.outputs.pyflow-wellies }} + optional_matrix: null ecmwf/pyodc: path: .github/ci-hpc-config.yml python: true @@ -1112,6 +1138,57 @@ jobs: ${{ needs.setup.outputs.eccodes }} ${{ needs.setup.outputs.eckit }} python_dependencies: '' + pyflow: + name: pyflow + needs: + - ecflow + - setup + if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow_matrix && (needs.setup.outputs.ecflow || needs.setup.outputs.pyflow) }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.pyflow_matrix) }} + env: + DEP_TREE: ${{ needs.setup.outputs.dep_tree }} + runs-on: + - self-hosted + - linux + - hpc + steps: + - uses: ecmwf-actions/reusable-workflows/ci-hpc@v2 + with: + github_user: ${{ secrets.BUILD_PACKAGE_HPC_GITHUB_USER }} + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + troika_user: ${{ secrets.HPC_CI_SSH_USER }} + repository: ${{ matrix.owner_repo_ref }} + build_config: ${{ matrix.config_path }} + dependencies: ${{ needs.setup.outputs.ecflow }} + python_dependencies: '' + pyflow-wellies: + name: pyflow-wellies + needs: + - pyflow + - ecflow + - setup + if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow-wellies_matrix && (needs.setup.outputs.pyflow || needs.setup.outputs.ecflow || needs.setup.outputs.pyflow-wellies) }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.pyflow-wellies_matrix) }} + env: + DEP_TREE: ${{ needs.setup.outputs.dep_tree }} + runs-on: + - self-hosted + - linux + - hpc + steps: + - uses: ecmwf-actions/reusable-workflows/ci-hpc@v2 + with: + github_user: ${{ secrets.BUILD_PACKAGE_HPC_GITHUB_USER }} + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + troika_user: ${{ secrets.HPC_CI_SSH_USER }} + repository: ${{ matrix.owner_repo_ref }} + build_config: ${{ matrix.config_path }} + dependencies: ${{ needs.setup.outputs.ecflow }} + python_dependencies: ${{ needs.setup.outputs.pyflow }} pyodc: name: pyodc needs: diff --git a/.github/workflows/downstream-ci.yml b/.github/workflows/downstream-ci.yml index e58aeca..47af0db 100644 --- a/.github/workflows/downstream-ci.yml +++ b/.github/workflows/downstream-ci.yml @@ -82,6 +82,12 @@ name: downstream-ci pyfdb: required: false type: string + pyflow: + required: false + type: string + pyflow-wellies: + required: false + type: string pyodc: required: false type: string @@ -164,6 +170,10 @@ jobs: plume_matrix: ${{ steps.setup.outputs.plume }} pyfdb: ${{ steps.prepare-inputs.outputs.pyfdb }} pyfdb_matrix: ${{ steps.setup.outputs.pyfdb }} + pyflow: ${{ steps.prepare-inputs.outputs.pyflow }} + pyflow_matrix: ${{ steps.setup.outputs.pyflow }} + pyflow-wellies: ${{ steps.prepare-inputs.outputs.pyflow-wellies }} + pyflow-wellies_matrix: ${{ steps.setup.outputs.pyflow-wellies }} pyodc: ${{ steps.prepare-inputs.outputs.pyodc }} pyodc_matrix: ${{ steps.setup.outputs.pyodc }} skinnywms: ${{ steps.prepare-inputs.outputs.skinnywms }} @@ -204,6 +214,8 @@ jobs: echo pdbufr="${{ inputs.pdbufr }}" >> $GITHUB_OUTPUT echo plume="${{ inputs.plume }}" >> $GITHUB_OUTPUT echo pyfdb="${{ inputs.pyfdb }}" >> $GITHUB_OUTPUT + echo pyflow="${{ inputs.pyflow }}" >> $GITHUB_OUTPUT + echo pyflow-wellies="${{ inputs.pyflow-wellies }}" >> $GITHUB_OUTPUT echo pyodc="${{ inputs.pyodc }}" >> $GITHUB_OUTPUT echo skinnywms="${{ inputs.skinnywms }}" >> $GITHUB_OUTPUT echo thermofeel="${{ inputs.thermofeel }}" >> $GITHUB_OUTPUT @@ -380,6 +392,20 @@ jobs: develop_branch: develop input: ${{ steps.prepare-inputs.outputs.pyfdb }} optional_matrix: null + ecmwf/pyflow: + path: .github/ci-config.yml + python: true + master_branch: master + develop_branch: develop + input: ${{ steps.prepare-inputs.outputs.pyflow }} + optional_matrix: null + ecmwf/pyflow-wellies: + path: .github/ci-config.yml + python: true + master_branch: master + develop_branch: develop + input: ${{ steps.prepare-inputs.outputs.pyflow-wellies }} + optional_matrix: null ecmwf/pyodc: path: .github/ci-config.yml python: true @@ -648,6 +674,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: ${{ needs.setup.outputs.eccodes-python }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} requirements_path: ci/requirements-tests.in @@ -694,6 +721,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: |- ${{ needs.setup.outputs.earthkit-data }} ${{ needs.setup.outputs.cfgrib }} @@ -748,6 +776,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: |- ${{ needs.setup.outputs.cfgrib }} ${{ needs.setup.outputs.multiurl }} @@ -878,6 +907,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: '' github_token: ${{ secrets.GH_REPO_READ_TOKEN }} codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} @@ -1190,6 +1220,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: ${{ needs.setup.outputs.eccodes-python }} github_token: ${{ secrets.GH_REPO_READ_TOKEN }} requirements_path: tests/downstream-ci-requirements.txt @@ -1264,10 +1295,76 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: '' github_token: ${{ secrets.GH_REPO_READ_TOKEN }} codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} codecov_token: ${{ secrets.CODECOV_UPLOAD_TOKEN }} + pyflow: + name: pyflow + needs: + - ecflow + - setup + - python-qa + if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow_matrix && (needs.setup.outputs.ecflow || needs.setup.outputs.pyflow) }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.pyflow_matrix) }} + env: + DEP_TREE: ${{ needs.setup.outputs.dep_tree }} + runs-on: ${{ matrix.labels }} + steps: + - name: Build dependencies + id: build-deps + uses: ecmwf-actions/reusable-workflows/build-package-with-config@v2 + with: + repository: ${{ matrix.owner_repo_ref }} + codecov_upload: false + build_package_inputs: 'repository: ${{ matrix.owner_repo_ref }}' + build_config: ${{ matrix.config_path }} + build_dependencies: ${{ needs.setup.outputs.ecflow }} + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + with: + lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} + python_dependencies: '' + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} + codecov_token: ${{ secrets.CODECOV_UPLOAD_TOKEN }} + pyflow-wellies: + name: pyflow-wellies + needs: + - pyflow + - ecflow + - setup + - python-qa + if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow-wellies_matrix && (needs.setup.outputs.pyflow || needs.setup.outputs.ecflow || needs.setup.outputs.pyflow-wellies) }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.pyflow-wellies_matrix) }} + env: + DEP_TREE: ${{ needs.setup.outputs.dep_tree }} + runs-on: ${{ matrix.labels }} + steps: + - name: Build dependencies + id: build-deps + uses: ecmwf-actions/reusable-workflows/build-package-with-config@v2 + with: + repository: ${{ matrix.owner_repo_ref }} + codecov_upload: false + build_package_inputs: 'repository: ${{ matrix.owner_repo_ref }}' + build_config: ${{ matrix.config_path }} + build_dependencies: ${{ needs.setup.outputs.ecflow }} + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + - uses: ecmwf-actions/reusable-workflows/ci-python@v2 + with: + lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} + python_dependencies: ${{ needs.setup.outputs.pyflow }} + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} + codecov_token: ${{ secrets.CODECOV_UPLOAD_TOKEN }} pyodc: name: pyodc needs: @@ -1298,6 +1395,7 @@ jobs: - uses: ecmwf-actions/reusable-workflows/ci-python@v2 with: lib_path: ${{ steps.build-deps.outputs.lib_path }} + bin_paths: ${{ steps.build-deps.outputs.bin_paths }} python_dependencies: '' github_token: ${{ secrets.GH_REPO_READ_TOKEN }} codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }} diff --git a/dependency_tree.yml b/dependency_tree.yml index 8ff565c..0c98817 100644 --- a/dependency_tree.yml +++ b/dependency_tree.yml @@ -199,6 +199,21 @@ pyfdb: {"type":"local","engine":"toc","schema":"${{ github.workspace }}/tests/default_fdb_schema","spaces":[{"handler":"Default","roots":[{"path":"${{ github.workspace }}/data/fdb"}]}]} mkdir: - data/fdb + +pyflow: + type: python + master_branch: master + deps: + - ecflow + # downstream-ci: + # env: + # ECFLOW_DIR: /opt/actions-runner/work/_work/_temp/install/ecflow + +pyflow-wellies: + type: python + deps: + - pyflow + pyodc: type: python deps: diff --git a/generate-workflows.py b/generate-workflows.py index f3ea7e2..2a92346 100644 --- a/generate-workflows.py +++ b/generate-workflows.py @@ -377,6 +377,9 @@ def generate_package_jobs(self, dep_tree: dict): "lib_path": ( "${{ steps.build-deps.outputs.lib_path }}" ), + "bin_paths": ( + "${{ steps.build-deps.outputs.bin_paths }}" + ), "python_dependencies": "\n".join(python_deps), "github_token": "${{ secrets.GH_REPO_READ_TOKEN }}", },