From 392aa2d70bab68db5d51a02849437b07b97616c8 Mon Sep 17 00:00:00 2001 From: MiKyung Lee <58964324+mlee03@users.noreply.github.com> Date: Wed, 26 Jun 2024 10:23:11 -0400 Subject: [PATCH 1/2] Updated container for Github Action and reusable workflow (#87) * updated generate_eta_files * remove notebook, add script * lint * correct path * update constraints * python3 execute * Update main_unit_tests_mpich.yaml * Update main_unit_tests_mpich.yaml * update input directory * get ..s correct * lint --------- Co-authored-by: mlee03 --- .github/workflows/main_unit_tests_mpich.yaml | 24 ++++++++++------- .../workflows/main_unit_tests_openmpi.yaml | 26 ------------------- constraints.txt | 2 +- 3 files changed, 15 insertions(+), 37 deletions(-) delete mode 100644 .github/workflows/main_unit_tests_openmpi.yaml diff --git a/.github/workflows/main_unit_tests_mpich.yaml b/.github/workflows/main_unit_tests_mpich.yaml index 9804bc15..7fa3af70 100644 --- a/.github/workflows/main_unit_tests_mpich.yaml +++ b/.github/workflows/main_unit_tests_mpich.yaml @@ -1,5 +1,6 @@ name: "Main unit tests with mpich" on: + push: pull_request: types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] @@ -12,15 +13,18 @@ jobs: main_unit_tests: runs-on: ubuntu-latest container: - image: ghcr.io/noaa-gfdl/pace_mpich:3.11.9 + image: ghcr.io/noaa-gfdl/miniforge:mpich steps: - - name: Checkout Pace repository - uses: actions/checkout@v4 - with: + - name: Checkout Pace repository + uses: actions/checkout@v4 + with: submodules: 'recursive' - - name: setup env and run tests (test) - run: | - cp /home/scripts/setup_env.sh . && chmod +x setup_env.sh - cp /home/scripts/run_tests.sh . && chmod +x run_tests.sh - ./setup_env.sh - ./run_tests.sh + - name: setup env and generate input files + run: | + pip3 install --upgrade pip setuptools wheel + pip3 install -r requirements_dev.txt -c constraints.txt + cd tests/main && mkdir -p input && cd input + python3 ../../../examples/generate_eta_files.py + cd ../../../ + - name: + run: pytest -x tests/main diff --git a/.github/workflows/main_unit_tests_openmpi.yaml b/.github/workflows/main_unit_tests_openmpi.yaml deleted file mode 100644 index fcb19de2..00000000 --- a/.github/workflows/main_unit_tests_openmpi.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: "Main unit tests with openmpi" -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] - -# cancel running jobs if theres a newer push -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - main_unit_tests: - runs-on: ubuntu-latest - container: - image: ghcr.io/noaa-gfdl/pace_openmpi:3.11.9 - steps: - - name: Checkout Pace repository - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - name: setup env and run tests - run: | - cp /home/scripts/setup_env.sh . && chmod +x setup_env.sh - cp /home/scripts/run_tests.sh . && chmod +x run_tests.sh - ./setup_env.sh - ./run_tests.sh diff --git a/constraints.txt b/constraints.txt index 0d520450..19e15772 100644 --- a/constraints.txt +++ b/constraints.txt @@ -297,7 +297,7 @@ pathspec==0.12.1 # via black pexpect==4.9.0 # via ipython -platformdirs==2.6.2 +platformdirs==3.10.0 # via # black # jupyter-core From 777b332989faf33a946c7bb080621a4829eeb96f Mon Sep 17 00:00:00 2001 From: MiKyung Lee <58964324+mlee03@users.noreply.github.com> Date: Wed, 26 Jun 2024 12:05:48 -0400 Subject: [PATCH 2/2] Updated container for Github Action and reusable workflow - Redo (#89) * test * tet * fix test dir * test * checkout pace * rename * test * rm * remove dir * fix name * test * pyshield * test * fix trigger mistake * test * change lint version * remove push * lint --------- Co-authored-by: mlee03 --- .github/workflows/lint.yml | 4 +- .github/workflows/main_unit_tests.yaml | 82 ++++++++++++++++++++ .github/workflows/main_unit_tests_mpich.yaml | 30 ------- 3 files changed, 84 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/main_unit_tests.yaml delete mode 100644 .github/workflows/main_unit_tests_mpich.yaml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 23a63e6d..a768fdb9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,10 +11,10 @@ jobs: uses: actions/checkout@v3.5.2 with: submodules: 'recursive' - - name: Step Python 3.11.9 + - name: Step Python 3.11.7 uses: actions/setup-python@v4.6.0 with: - python-version: '3.11.9' + python-version: '3.11.7' - name: Install OpenMPI for gt4py run: | sudo apt-get install libopenmpi-dev diff --git a/.github/workflows/main_unit_tests.yaml b/.github/workflows/main_unit_tests.yaml new file mode 100644 index 00000000..9165727e --- /dev/null +++ b/.github/workflows/main_unit_tests.yaml @@ -0,0 +1,82 @@ +name: "Pace main unit tests" +on: + workflow_call: + inputs: + ndsl_trigger: + type: boolean + default: false + required: false + fv3_trigger: + type: boolean + default: false + required: false + shield_trigger: + type: boolean + default: false + required: false + pull_request: + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] + + +# cancel running jobs if theres a newer push +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + github_repository: ${{github.event.repository.name}} + +jobs: + main_unit_tests: + runs-on: ubuntu-latest + container: + image: ghcr.io/noaa-gfdl/miniforge:mpich + steps: + - name: Checkout pace if externally triggered + if: ${{ inputs.fv3_trigger || inputs.ndsl_trigger || inputs.shield_trigger }} + uses: actions/checkout@v4 + with: + submodules: 'recursive' + repository: NOAA-GFDL/pace + path: pace + + - name: Checkout repo that triggered the workflow + uses: actions/checkout@v4 + with: + submodules: 'recursive' + path: ${{ env.github_repository }} + + - name: mv PyFV3 to pace + if: ${{inputs.fv3_trigger}} + run: | + rm -r ${GITHUB_WORKSPACE}/pace/pyFV3 + mv ${GITHUB_WORKSPACE}/${{env.github_repository}} ${GITHUB_WORKSPACE}/pace/pyFV3 + + - name: mv NDSL to pace + if: ${{inputs.ndsl_trigger}} + run: | + rm -r ${GITHUB_WORKSPACE}/pace/NDSL + mv ${GITHUB_WORKSPACE}/${{env.github_repository}} ${GITHUB_WORKSPACE}/pace/NDSL + + - name: mv pyShield to pace + if: ${{inputs.shield_trigger}} + run: | + rm -r ${GITHUB_WORKSPACE}/pace/pySHiELD + mv ${GITHUB_WORKSPACE}/${{env.github_repository}} ${GITHUB_WORKSPACE}/pace/pySHiELD + + - name: install packages + run: | + cd ${GITHUB_WORKSPACE}/pace + pip3 install --upgrade pip setuptools wheel + pip3 install -r requirements_dev.txt -c constraints.txt + + - name: prepare input files + run: | + cd ${GITHUB_WORKSPACE}/pace + python3 examples/generate_eta_files.py + mkdir tests/main/input && mv eta*.nc tests/main/input/. + + - name: run tests + run: | + cd ${GITHUB_WORKSPACE}/pace + pytest -x tests/main diff --git a/.github/workflows/main_unit_tests_mpich.yaml b/.github/workflows/main_unit_tests_mpich.yaml deleted file mode 100644 index 7fa3af70..00000000 --- a/.github/workflows/main_unit_tests_mpich.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: "Main unit tests with mpich" -on: - push: - pull_request: - types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] - -# cancel running jobs if theres a newer push -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - main_unit_tests: - runs-on: ubuntu-latest - container: - image: ghcr.io/noaa-gfdl/miniforge:mpich - steps: - - name: Checkout Pace repository - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - name: setup env and generate input files - run: | - pip3 install --upgrade pip setuptools wheel - pip3 install -r requirements_dev.txt -c constraints.txt - cd tests/main && mkdir -p input && cd input - python3 ../../../examples/generate_eta_files.py - cd ../../../ - - name: - run: pytest -x tests/main