Merge pull request #546 from jtwhite79/feat_ppu3

Feat ppu3

.gitattributes

@@ -1,15 +1,26 @@
 pyemu/_version.py export-subst
-*.rei linguist-vendored # ignore because recognized as Reason
-*.ins linguist-vendored # ignore because recognized as LaTex
-*.aux linguist-vendored # ignore because recognized as LaTex
-*.tex linguist-vendored # ignore because recognized as LaTex
-*.go linguist-vendored # ignore because recognized as Go
-*.tpl linguist-vendored # ignore because recognized as Smarty
-*.bas linguist-vendored # ignore because recognized as VBA
-*.res linguist-vendored # ignore because recognized as ReScript
-*.dsp linguist-vendored # ignore because recognized as Faust
-*.spc linguist-vendored # ignore because recognized as PLSQL
-*.bat linguist-vendored # ignore because recognized as Batchfile
-*.arr linguist-vendored # ignore because recognized as Pyret
-*.mps linguist-vendored # ignore because recognized as JetBrains MPS
+# ignore .rei because recognized as Reason
+*.rei linguist-vendored
+# ignore .ins, .aux, .tex because recognized as LaTex
+*.ins linguist-vendored
+*.aux linguist-vendored
+*.tex linguist-vendored
+# ignore .go because recognized as Go
+*.go linguist-vendored
+# ignore .tpl because recognized as Smarty
+*.tpl linguist-vendored
+# ignore .bas because recognized as VBA
+*.bas linguist-vendored
+# ignore .res because recognized as ReScript
+*.res linguist-vendored
+# ignore .dsp because recognized as Faust
+*.dsp linguist-vendored
+# ignore .spc because recognized as PLSQL
+*.spc linguist-vendored
+# ignore .bat because recognized as Batchfile
+*.bat linguist-vendored
+# ignore .arr because recognized as Pyret
+*.arr linguist-vendored
+# ignore .mps because recognized as JetBrains MPS
+*.mps linguist-vendored
 

.github/workflows/ci.yml

@@ -5,6 +5,7 @@ on:
     - cron: '0 8 * * *' # run at 8 AM UTC (12 AM PST, 8 PM NZST)
   push:
   pull_request:
+  workflow_dispatch:
 
 jobs:
   pyemuCI:
@@ -14,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest] # , macos-latest] 
+        os: [windows-latest, ubuntu-latest]
         python-version: [3.9, "3.10", "3.11", "3.12", "3.13"]
         run-type: [std]
         test-path: ["."]
@@ -43,6 +44,52 @@ jobs:
         cache-environment-key: environment-${{ steps.date.outputs.date }}
         cache-downloads-key: downloads-${{ steps.date.outputs.date }}
 
+    - name: Checkout pypestutils
+      uses:  actions/checkout@v4
+      with:
+        repository: pypest/pypestutils
+        ref: develop
+        path: pypestutils
+
+#    - name: Install MinGW-w64 tools (Windows)
+#      if: runner.os == 'Windows'
+#      uses: msys2/setup-msys2@v2
+#      with:
+#        msystem: MINGW64
+#        path-type: inherit
+##        install: >-
+##          mingw-w64-x86_64-gcc-fortran
+##          mingw-w64-x86_64-lapack
+##          mingw-w64-x86_64-meson
+##          mingw-w64-x86_64-ninja
+
+    - name: Install meson and gfortran (for ppu)
+      shell: bash -l {0}
+      run: |
+        micromamba install meson gfortran
+
+    - name: Build pypestutils (Windows)
+      if: runner.os == 'Windows'
+      shell: bash -l {0}
+      working-directory: pypestutils
+      env:
+        LDFLAGS: -static-libgcc -static-libgfortran -static-libquadmath -Wl,-Bstatic,--whole-archive -lwinpthread -Wl,--no-whole-archive
+      run: |
+        scripts/build_lib.sh
+
+    - name: Build pypestutils (non-Windows)
+      if: runner.os != 'Windows'
+      shell: bash -l {0}
+      working-directory: pypestutils
+      run: |
+        scripts/build_lib.sh
+
+    - name: Install pypestutils
+      shell: bash -l {0}
+      working-directory: pypestutils
+      run: |
+        pip install -e .
+
     - name: Install pyemu
       shell: bash -l {0}
       run: |

autotest/conftest.py

@@ -1,10 +1,9 @@
 from pathlib import Path
 import pytest
-from pst_from_tests import setup_freyberg_mf6
+# from pst_from_tests import setup_freyberg_mf6
 
 pytest_plugins = ["modflow_devtools.fixtures"]
 
-
 collect_ignore = [
     # "utils_tests.py",
     # "pst_tests.py",
@@ -17,5 +16,11 @@
     # "metrics_tests.py",
     # "moouu_tests.py",
     # "mat_tests.py",
-    # "da_tests.py"
+    # "da_tests.py",
+    # "get_pestpp_tests.py"
 ]
+
+@pytest.fixture(autouse=True)
+def _ch2testdir(monkeypatch):
+    testdir = Path(__file__).parent
+    monkeypatch.chdir(testdir)

autotest/la_tests.py

@@ -2,6 +2,7 @@
 import copy
 from pathlib import Path
 import shutil
+from pst_from_tests import setup_tmp, ies_exe_path, _get_port
 
 import pytest
 
@@ -57,20 +58,6 @@ def schur_test_nonpest():
     sc = Schur(jco=jco, forecasts=ffile, parcov=parcov, obscov=obscov)
 
 
-def setup_tmp(od, tmp_path, sub=None):
-    basename = Path(od).name
-    if sub is not None:
-        new_d = Path(tmp_path, basename, sub)
-    else:
-        new_d = Path(tmp_path, basename)
-    if new_d.exists():
-        shutil.rmtree(new_d)
-    Path(tmp_path).mkdir(exist_ok=True)
-    # creation functionality
-    shutil.copytree(od, new_d)
-    return new_d
-
-
 def schur_test(tmp_path):
     import os
     import numpy as np
@@ -599,7 +586,7 @@ def ends_freyberg_test(tmp_path):
     ends = pyemu.EnDS(pst=pst_name, sim_ensemble=oe_name,predictions=predictions)
     cov = pyemu.Cov.from_observation_data(pst)
     cov.to_uncfile(os.path.join(test_d, "obs.unc"), covmat_file=None)
-    cov.to_binary(os.path.join(test_d, "cov.jcb"))
+    cov.to_coo(os.path.join(test_d, "cov.jcb"))
     cov.to_ascii(os.path.join(test_d, "cov.mat"))
 
     ends = pyemu.EnDS(pst=pst, sim_ensemble=oe, obscov=cov,predictions=predictions)
@@ -608,11 +595,11 @@ def ends_freyberg_test(tmp_path):
 
 
 
-def ends_run_freyberg_dsi(tmp_path,nst=False,nst_extrap=None,ztz=False,energy=1.0):
+def ends_run_freyberg_dsi(tmp_d, nst=False, nst_extrap=None, ztz=False, energy=1.0):
     import pyemu
-    import os 
+    import os
     test_d = "ends_master"
-    test_d = setup_tmp(test_d, tmp_path)
+    test_d = setup_tmp(test_d, tmp_d)
     case = "freyberg6_run_ies"
     pst_name = os.path.join(test_d, case + ".pst")
     pst = pyemu.Pst(pst_name)
@@ -623,7 +610,7 @@ def ends_run_freyberg_dsi(tmp_path,nst=False,nst_extrap=None,ztz=False,energy=1.
     oe = pyemu.ObservationEnsemble.from_csv(pst=pst, filename=oe_name).iloc[:100, :]
 
     ends = pyemu.EnDS(pst=pst, sim_ensemble=oe,verbose=True)
-    t_d = os.path.join(tmp_path,"dsi_template")
+    t_d = os.path.join(tmp_d, "dsi_template")
 
     ends.prep_for_dsi(t_d=t_d,
                       use_ztz=ztz,
@@ -637,12 +624,17 @@ def ends_run_freyberg_dsi(tmp_path,nst=False,nst_extrap=None,ztz=False,energy=1.
         os.remove(filename)
     pst = pyemu.Pst(os.path.join(t_d,"dsi.pst"))
     pst.control_data.noptmax = -1
+    pst.pestpp_options["overdue_giveup_fac"] = 1000
     pst.write(os.path.join(t_d,"dsi.pst"),version=2)
-    pyemu.os_utils.run("pestpp-ies dsi.pst",cwd=t_d)
-
+    #pyemu.os_utils.run("pestpp-ies dsi.pst",cwd=t_d)
+    m_d = t_d.replace("template","master")
+    port = _get_port()
+    pyemu.os_utils.start_workers(t_d, ies_exe_path,"dsi.pst",
+                                 worker_root=tmp_d,
+                                 master_dir=m_d, num_workers=10, port=port)
     #read in the results
-    oe = pyemu.ObservationEnsemble.from_csv(pst=pst, filename=os.path.join(t_d,"dsi.0.obs.csv"))
-    assert oe.shape[0]==50, f"{50-oe.shape} failed runs"
+    oe = pyemu.ObservationEnsemble.from_csv(pst=pst, filename=os.path.join(m_d,"dsi.0.obs.csv"))
+    assert oe.shape[0]==50, f"{50-oe.shape[0]} failed runs"
     phi_vector = oe.phi_vector.sort_values().values
     assert phi_vector[0] != phi_vector[1],phi_vector
 
@@ -750,7 +742,8 @@ def dsi_normscoretransform_test():
 if __name__ == "__main__":
     #dsi_normscoretransform_test()
     #ends_freyberg_dev()
-    ends_freyberg_dsi_test("temp")
+    #ends_freyberg_dsi_test("temp")
+    ends_freyberg_dsi_ztz_test('temp')
     #plot_freyberg_dsi()
     #obscomp_test()
     #alternative_dw()