ecmwf-actions · corentincarton · May 8, 2024 · May 8, 2024 · May 8, 2024 · May 8, 2024
diff --git a/.github/workflows/downstream-ci-hpc.yml b/.github/workflows/downstream-ci-hpc.yml
@@ -78,6 +78,12 @@ name: downstream-ci-hpc
       pyfdb:
         required: false
         type: string
+      pyflow:
+        required: false
+        type: string
+      pyflow-wellies:
+        required: false
+        type: string
       pyodc:
         required: false
         type: string
@@ -148,6 +154,10 @@ jobs:
       plume_matrix: ${{ steps.setup.outputs.plume }}
       pyfdb: ${{ steps.prepare-inputs.outputs.pyfdb }}
       pyfdb_matrix: ${{ steps.setup.outputs.pyfdb }}
+      pyflow: ${{ steps.prepare-inputs.outputs.pyflow }}
+      pyflow_matrix: ${{ steps.setup.outputs.pyflow }}
+      pyflow-wellies: ${{ steps.prepare-inputs.outputs.pyflow-wellies }}
+      pyflow-wellies_matrix: ${{ steps.setup.outputs.pyflow-wellies }}
       pyodc: ${{ steps.prepare-inputs.outputs.pyodc }}
       pyodc_matrix: ${{ steps.setup.outputs.pyodc }}
       skinnywms: ${{ steps.prepare-inputs.outputs.skinnywms }}
@@ -186,6 +196,8 @@ jobs:
         echo pdbufr="${{ inputs.pdbufr }}" >> $GITHUB_OUTPUT
         echo plume="${{ inputs.plume }}" >> $GITHUB_OUTPUT
         echo pyfdb="${{ inputs.pyfdb }}" >> $GITHUB_OUTPUT
+        echo pyflow="${{ inputs.pyflow }}" >> $GITHUB_OUTPUT
+        echo pyflow-wellies="${{ inputs.pyflow-wellies }}" >> $GITHUB_OUTPUT
         echo pyodc="${{ inputs.pyodc }}" >> $GITHUB_OUTPUT
         echo skinnywms="${{ inputs.skinnywms }}" >> $GITHUB_OUTPUT
         echo thermofeel="${{ inputs.thermofeel }}" >> $GITHUB_OUTPUT
@@ -362,6 +374,20 @@ jobs:
             develop_branch: develop
             input: ${{ steps.prepare-inputs.outputs.pyfdb }}
             optional_matrix: null
+          ecmwf/pyflow:
+            path: .github/ci-hpc-config.yml
+            python: true
+            master_branch: master
+            develop_branch: develop
+            input: ${{ steps.prepare-inputs.outputs.pyflow }}
+            optional_matrix: null
+          ecmwf/pyflow-wellies:
+            path: .github/ci-hpc-config.yml
+            python: true
+            master_branch: master
+            develop_branch: develop
+            input: ${{ steps.prepare-inputs.outputs.pyflow-wellies }}
+            optional_matrix: null
           ecmwf/pyodc:
             path: .github/ci-hpc-config.yml
             python: true
@@ -1112,6 +1138,57 @@ jobs:
           ${{ needs.setup.outputs.eccodes }}
           ${{ needs.setup.outputs.eckit }}
         python_dependencies: ''
+  pyflow:
+    name: pyflow
+    needs:
+    - ecflow
+    - setup
+    if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow_matrix && (needs.setup.outputs.ecflow || needs.setup.outputs.pyflow) }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.setup.outputs.pyflow_matrix) }}
+    env:
+      DEP_TREE: ${{ needs.setup.outputs.dep_tree }}
+    runs-on:
+    - self-hosted
+    - linux
+    - hpc
+    steps:
+    - uses: ecmwf-actions/reusable-workflows/ci-hpc@v2
+      with:
+        github_user: ${{ secrets.BUILD_PACKAGE_HPC_GITHUB_USER }}
+        github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
+        troika_user: ${{ secrets.HPC_CI_SSH_USER }}
+        repository: ${{ matrix.owner_repo_ref }}
+        build_config: ${{ matrix.config_path }}
+        dependencies: ${{ needs.setup.outputs.ecflow }}
+        python_dependencies: ''
+  pyflow-wellies:
+    name: pyflow-wellies
+    needs:
+    - pyflow
+    - ecflow
+    - setup
+    if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow-wellies_matrix && (needs.setup.outputs.pyflow || needs.setup.outputs.ecflow || needs.setup.outputs.pyflow-wellies) }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.setup.outputs.pyflow-wellies_matrix) }}
+    env:
+      DEP_TREE: ${{ needs.setup.outputs.dep_tree }}
+    runs-on:
+    - self-hosted
+    - linux
+    - hpc
+    steps:
+    - uses: ecmwf-actions/reusable-workflows/ci-hpc@v2
+      with:
+        github_user: ${{ secrets.BUILD_PACKAGE_HPC_GITHUB_USER }}
+        github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
+        troika_user: ${{ secrets.HPC_CI_SSH_USER }}
+        repository: ${{ matrix.owner_repo_ref }}
+        build_config: ${{ matrix.config_path }}
+        dependencies: ${{ needs.setup.outputs.ecflow }}
+        python_dependencies: ${{ needs.setup.outputs.pyflow }}
   pyodc:
     name: pyodc
     needs:

diff --git a/.github/workflows/downstream-ci.yml b/.github/workflows/downstream-ci.yml
@@ -82,6 +82,12 @@ name: downstream-ci
       pyfdb:
         required: false
         type: string
+      pyflow:
+        required: false
+        type: string
+      pyflow-wellies:
+        required: false
+        type: string
       pyodc:
         required: false
         type: string
@@ -164,6 +170,10 @@ jobs:
       plume_matrix: ${{ steps.setup.outputs.plume }}
       pyfdb: ${{ steps.prepare-inputs.outputs.pyfdb }}
       pyfdb_matrix: ${{ steps.setup.outputs.pyfdb }}
+      pyflow: ${{ steps.prepare-inputs.outputs.pyflow }}
+      pyflow_matrix: ${{ steps.setup.outputs.pyflow }}
+      pyflow-wellies: ${{ steps.prepare-inputs.outputs.pyflow-wellies }}
+      pyflow-wellies_matrix: ${{ steps.setup.outputs.pyflow-wellies }}
       pyodc: ${{ steps.prepare-inputs.outputs.pyodc }}
       pyodc_matrix: ${{ steps.setup.outputs.pyodc }}
       skinnywms: ${{ steps.prepare-inputs.outputs.skinnywms }}
@@ -204,6 +214,8 @@ jobs:
         echo pdbufr="${{ inputs.pdbufr }}" >> $GITHUB_OUTPUT
         echo plume="${{ inputs.plume }}" >> $GITHUB_OUTPUT
         echo pyfdb="${{ inputs.pyfdb }}" >> $GITHUB_OUTPUT
+        echo pyflow="${{ inputs.pyflow }}" >> $GITHUB_OUTPUT
+        echo pyflow-wellies="${{ inputs.pyflow-wellies }}" >> $GITHUB_OUTPUT
         echo pyodc="${{ inputs.pyodc }}" >> $GITHUB_OUTPUT
         echo skinnywms="${{ inputs.skinnywms }}" >> $GITHUB_OUTPUT
         echo thermofeel="${{ inputs.thermofeel }}" >> $GITHUB_OUTPUT
@@ -380,6 +392,20 @@ jobs:
             develop_branch: develop
             input: ${{ steps.prepare-inputs.outputs.pyfdb }}
             optional_matrix: null
+          ecmwf/pyflow:
+            path: .github/ci-config.yml
+            python: true
+            master_branch: master
+            develop_branch: develop
+            input: ${{ steps.prepare-inputs.outputs.pyflow }}
+            optional_matrix: null
+          ecmwf/pyflow-wellies:
+            path: .github/ci-config.yml
+            python: true
+            master_branch: master
+            develop_branch: develop
+            input: ${{ steps.prepare-inputs.outputs.pyflow-wellies }}
+            optional_matrix: null
           ecmwf/pyodc:
             path: .github/ci-config.yml
             python: true
@@ -648,6 +674,7 @@ jobs:
     - uses: ecmwf-actions/reusable-workflows/ci-python@v2
       with:
         lib_path: ${{ steps.build-deps.outputs.lib_path }}
+        bin_paths: ${{ steps.build-deps.outputs.bin_paths }}
         python_dependencies: ${{ needs.setup.outputs.eccodes-python }}
         github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
         requirements_path: ci/requirements-tests.in
@@ -694,6 +721,7 @@ jobs:
     - uses: ecmwf-actions/reusable-workflows/ci-python@v2
       with:
         lib_path: ${{ steps.build-deps.outputs.lib_path }}
+        bin_paths: ${{ steps.build-deps.outputs.bin_paths }}
         python_dependencies: |-
           ${{ needs.setup.outputs.earthkit-data }}
           ${{ needs.setup.outputs.cfgrib }}
@@ -748,6 +776,7 @@ jobs:
     - uses: ecmwf-actions/reusable-workflows/ci-python@v2
       with:
         lib_path: ${{ steps.build-deps.outputs.lib_path }}
+        bin_paths: ${{ steps.build-deps.outputs.bin_paths }}
         python_dependencies: |-
           ${{ needs.setup.outputs.cfgrib }}
           ${{ needs.setup.outputs.multiurl }}
@@ -878,6 +907,7 @@ jobs:
     - uses: ecmwf-actions/reusable-workflows/ci-python@v2
       with:
         lib_path: ${{ steps.build-deps.outputs.lib_path }}
+        bin_paths: ${{ steps.build-deps.outputs.bin_paths }}
         python_dependencies: ''
         github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
         codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }}
@@ -1190,6 +1220,7 @@ jobs:
     - uses: ecmwf-actions/reusable-workflows/ci-python@v2
       with:
         lib_path: ${{ steps.build-deps.outputs.lib_path }}
+        bin_paths: ${{ steps.build-deps.outputs.bin_paths }}
         python_dependencies: ${{ needs.setup.outputs.eccodes-python }}
         github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
         requirements_path: tests/downstream-ci-requirements.txt
@@ -1264,10 +1295,76 @@ jobs:
     - uses: ecmwf-actions/reusable-workflows/ci-python@v2
       with:
         lib_path: ${{ steps.build-deps.outputs.lib_path }}
+        bin_paths: ${{ steps.build-deps.outputs.bin_paths }}
         python_dependencies: ''
         github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
         codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }}
         codecov_token: ${{ secrets.CODECOV_UPLOAD_TOKEN }}
+  pyflow:
+    name: pyflow
+    needs:
+    - ecflow
+    - setup
+    - python-qa
+    if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow_matrix && (needs.setup.outputs.ecflow || needs.setup.outputs.pyflow) }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.setup.outputs.pyflow_matrix) }}
+    env:
+      DEP_TREE: ${{ needs.setup.outputs.dep_tree }}
+    runs-on: ${{ matrix.labels }}
+    steps:
+    - name: Build dependencies
+      id: build-deps
+      uses: ecmwf-actions/reusable-workflows/build-package-with-config@v2
+      with:
+        repository: ${{ matrix.owner_repo_ref }}
+        codecov_upload: false
+        build_package_inputs: 'repository: ${{ matrix.owner_repo_ref }}'
+        build_config: ${{ matrix.config_path }}
+        build_dependencies: ${{ needs.setup.outputs.ecflow }}
+        github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
+    - uses: ecmwf-actions/reusable-workflows/ci-python@v2
+      with:
+        lib_path: ${{ steps.build-deps.outputs.lib_path }}
+        bin_paths: ${{ steps.build-deps.outputs.bin_paths }}
+        python_dependencies: ''
+        github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
+        codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }}
+        codecov_token: ${{ secrets.CODECOV_UPLOAD_TOKEN }}
+  pyflow-wellies:
+    name: pyflow-wellies
+    needs:
+    - pyflow
+    - ecflow
+    - setup
+    - python-qa
+    if: ${{ (always() && !cancelled()) && contains(join(needs.*.result, ','), 'success') && needs.setup.outputs.pyflow-wellies_matrix && (needs.setup.outputs.pyflow || needs.setup.outputs.ecflow || needs.setup.outputs.pyflow-wellies) }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.setup.outputs.pyflow-wellies_matrix) }}
+    env:
+      DEP_TREE: ${{ needs.setup.outputs.dep_tree }}
+    runs-on: ${{ matrix.labels }}
+    steps:
+    - name: Build dependencies
+      id: build-deps
+      uses: ecmwf-actions/reusable-workflows/build-package-with-config@v2
+      with:
+        repository: ${{ matrix.owner_repo_ref }}
+        codecov_upload: false
+        build_package_inputs: 'repository: ${{ matrix.owner_repo_ref }}'
+        build_config: ${{ matrix.config_path }}
+        build_dependencies: ${{ needs.setup.outputs.ecflow }}
+        github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
+    - uses: ecmwf-actions/reusable-workflows/ci-python@v2
+      with:
+        lib_path: ${{ steps.build-deps.outputs.lib_path }}
+        bin_paths: ${{ steps.build-deps.outputs.bin_paths }}
+        python_dependencies: ${{ needs.setup.outputs.pyflow }}
+        github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
+        codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }}
+        codecov_token: ${{ secrets.CODECOV_UPLOAD_TOKEN }}
   pyodc:
     name: pyodc
     needs:
@@ -1298,6 +1395,7 @@ jobs:
     - uses: ecmwf-actions/reusable-workflows/ci-python@v2
       with:
         lib_path: ${{ steps.build-deps.outputs.lib_path }}
+        bin_paths: ${{ steps.build-deps.outputs.bin_paths }}
         python_dependencies: ''
         github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
         codecov_upload: ${{ needs.setup.outputs.trigger_repo == github.job && inputs.codecov_upload && needs.setup.outputs.py_codecov_platform == matrix.name }}

diff --git a/dependency_tree.yml b/dependency_tree.yml
@@ -199,6 +199,21 @@ pyfdb:
         {"type":"local","engine":"toc","schema":"${{ github.workspace }}/tests/default_fdb_schema","spaces":[{"handler":"Default","roots":[{"path":"${{ github.workspace }}/data/fdb"}]}]}
     mkdir:
       - data/fdb
+
+pyflow:
+  type: python
+  master_branch: master
+  deps:
+    - ecflow
+  # downstream-ci:
+  #   env:
+  #     ECFLOW_DIR: /opt/actions-runner/work/_work/_temp/install/ecflow
+
+pyflow-wellies:
+  type: python
+  deps:
+    - pyflow
+
 pyodc:
   type: python
   deps:

diff --git a/generate-workflows.py b/generate-workflows.py
@@ -377,6 +377,9 @@ def generate_package_jobs(self, dep_tree: dict):
                                 "lib_path": (
                                     "${{ steps.build-deps.outputs.lib_path }}"
                                 ),
+                                "bin_paths": (
+                                    "${{ steps.build-deps.outputs.bin_paths }}"
+                                ),
                                 "python_dependencies": "\n".join(python_deps),
                                 "github_token": "${{ secrets.GH_REPO_READ_TOKEN }}",
                             },