diff --git a/.gitattributes b/.gitattributes index fe7b3ff5..1c158c8a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,15 +1,26 @@ pyemu/_version.py export-subst -*.rei linguist-vendored # ignore because recognized as Reason -*.ins linguist-vendored # ignore because recognized as LaTex -*.aux linguist-vendored # ignore because recognized as LaTex -*.tex linguist-vendored # ignore because recognized as LaTex -*.go linguist-vendored # ignore because recognized as Go -*.tpl linguist-vendored # ignore because recognized as Smarty -*.bas linguist-vendored # ignore because recognized as VBA -*.res linguist-vendored # ignore because recognized as ReScript -*.dsp linguist-vendored # ignore because recognized as Faust -*.spc linguist-vendored # ignore because recognized as PLSQL -*.bat linguist-vendored # ignore because recognized as Batchfile -*.arr linguist-vendored # ignore because recognized as Pyret -*.mps linguist-vendored # ignore because recognized as JetBrains MPS +# ignore .rei because recognized as Reason +*.rei linguist-vendored +# ignore .ins, .aux, .tex because recognized as LaTex +*.ins linguist-vendored +*.aux linguist-vendored +*.tex linguist-vendored +# ignore .go because recognized as Go +*.go linguist-vendored +# ignore .tpl because recognized as Smarty +*.tpl linguist-vendored +# ignore .bas because recognized as VBA +*.bas linguist-vendored +# ignore .res because recognized as ReScript +*.res linguist-vendored +# ignore .dsp because recognized as Faust +*.dsp linguist-vendored +# ignore .spc because recognized as PLSQL +*.spc linguist-vendored +# ignore .bat because recognized as Batchfile +*.bat linguist-vendored +# ignore .arr because recognized as Pyret +*.arr linguist-vendored +# ignore .mps because recognized as JetBrains MPS +*.mps linguist-vendored diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2c64f82..03ab3294 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: - cron: '0 8 * * *' # run at 8 AM UTC (12 AM PST, 8 PM NZST) push: pull_request: + workflow_dispatch: jobs: pyemuCI: @@ -14,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] # , macos-latest] + os: [windows-latest, ubuntu-latest] python-version: [3.9, "3.10", "3.11", "3.12", "3.13"] run-type: [std] test-path: ["."] @@ -43,6 +44,52 @@ jobs: cache-environment-key: environment-${{ steps.date.outputs.date }} cache-downloads-key: downloads-${{ steps.date.outputs.date }} + - name: Checkout pypestutils + uses: actions/checkout@v4 + with: + repository: pypest/pypestutils + ref: develop + path: pypestutils + +# - name: Install MinGW-w64 tools (Windows) +# if: runner.os == 'Windows' +# uses: msys2/setup-msys2@v2 +# with: +# msystem: MINGW64 +# path-type: inherit +## install: >- +## mingw-w64-x86_64-gcc-fortran +## mingw-w64-x86_64-lapack +## mingw-w64-x86_64-meson +## mingw-w64-x86_64-ninja + + - name: Install meson and gfortran (for ppu) + shell: bash -l {0} + run: | + micromamba install meson gfortran + + - name: Build pypestutils (Windows) + if: runner.os == 'Windows' + shell: bash -l {0} + working-directory: pypestutils + env: + LDFLAGS: -static-libgcc -static-libgfortran -static-libquadmath -Wl,-Bstatic,--whole-archive -lwinpthread -Wl,--no-whole-archive + run: | + scripts/build_lib.sh + + - name: Build pypestutils (non-Windows) + if: runner.os != 'Windows' + shell: bash -l {0} + working-directory: pypestutils + run: | + scripts/build_lib.sh + + - name: Install pypestutils + shell: bash -l {0} + working-directory: pypestutils + run: | + pip install -e . + - name: Install pyemu shell: bash -l {0} run: | diff --git a/autotest/conftest.py b/autotest/conftest.py index 9d5453d0..8a6cca34 100644 --- a/autotest/conftest.py +++ b/autotest/conftest.py @@ -1,10 +1,9 @@ from pathlib import Path import pytest -from pst_from_tests import setup_freyberg_mf6 +# from pst_from_tests import setup_freyberg_mf6 pytest_plugins = ["modflow_devtools.fixtures"] - collect_ignore = [ # "utils_tests.py", # "pst_tests.py", @@ -17,5 +16,11 @@ # "metrics_tests.py", # "moouu_tests.py", # "mat_tests.py", - # "da_tests.py" + # "da_tests.py", + # "get_pestpp_tests.py" ] + +@pytest.fixture(autouse=True) +def _ch2testdir(monkeypatch): + testdir = Path(__file__).parent + monkeypatch.chdir(testdir) \ No newline at end of file diff --git a/autotest/la_tests.py b/autotest/la_tests.py index 42d7cea2..0fdfa3ea 100644 --- a/autotest/la_tests.py +++ b/autotest/la_tests.py @@ -2,6 +2,7 @@ import copy from pathlib import Path import shutil +from pst_from_tests import setup_tmp, ies_exe_path, _get_port import pytest @@ -57,20 +58,6 @@ def schur_test_nonpest(): sc = Schur(jco=jco, forecasts=ffile, parcov=parcov, obscov=obscov) -def setup_tmp(od, tmp_path, sub=None): - basename = Path(od).name - if sub is not None: - new_d = Path(tmp_path, basename, sub) - else: - new_d = Path(tmp_path, basename) - if new_d.exists(): - shutil.rmtree(new_d) - Path(tmp_path).mkdir(exist_ok=True) - # creation functionality - shutil.copytree(od, new_d) - return new_d - - def schur_test(tmp_path): import os import numpy as np @@ -599,7 +586,7 @@ def ends_freyberg_test(tmp_path): ends = pyemu.EnDS(pst=pst_name, sim_ensemble=oe_name,predictions=predictions) cov = pyemu.Cov.from_observation_data(pst) cov.to_uncfile(os.path.join(test_d, "obs.unc"), covmat_file=None) - cov.to_binary(os.path.join(test_d, "cov.jcb")) + cov.to_coo(os.path.join(test_d, "cov.jcb")) cov.to_ascii(os.path.join(test_d, "cov.mat")) ends = pyemu.EnDS(pst=pst, sim_ensemble=oe, obscov=cov,predictions=predictions) @@ -608,11 +595,11 @@ def ends_freyberg_test(tmp_path): -def ends_run_freyberg_dsi(tmp_path,nst=False,nst_extrap=None,ztz=False,energy=1.0): +def ends_run_freyberg_dsi(tmp_d, nst=False, nst_extrap=None, ztz=False, energy=1.0): import pyemu - import os + import os test_d = "ends_master" - test_d = setup_tmp(test_d, tmp_path) + test_d = setup_tmp(test_d, tmp_d) case = "freyberg6_run_ies" pst_name = os.path.join(test_d, case + ".pst") pst = pyemu.Pst(pst_name) @@ -623,7 +610,7 @@ def ends_run_freyberg_dsi(tmp_path,nst=False,nst_extrap=None,ztz=False,energy=1. oe = pyemu.ObservationEnsemble.from_csv(pst=pst, filename=oe_name).iloc[:100, :] ends = pyemu.EnDS(pst=pst, sim_ensemble=oe,verbose=True) - t_d = os.path.join(tmp_path,"dsi_template") + t_d = os.path.join(tmp_d, "dsi_template") ends.prep_for_dsi(t_d=t_d, use_ztz=ztz, @@ -637,12 +624,17 @@ def ends_run_freyberg_dsi(tmp_path,nst=False,nst_extrap=None,ztz=False,energy=1. os.remove(filename) pst = pyemu.Pst(os.path.join(t_d,"dsi.pst")) pst.control_data.noptmax = -1 + pst.pestpp_options["overdue_giveup_fac"] = 1000 pst.write(os.path.join(t_d,"dsi.pst"),version=2) - pyemu.os_utils.run("pestpp-ies dsi.pst",cwd=t_d) - + #pyemu.os_utils.run("pestpp-ies dsi.pst",cwd=t_d) + m_d = t_d.replace("template","master") + port = _get_port() + pyemu.os_utils.start_workers(t_d, ies_exe_path,"dsi.pst", + worker_root=tmp_d, + master_dir=m_d, num_workers=10, port=port) #read in the results - oe = pyemu.ObservationEnsemble.from_csv(pst=pst, filename=os.path.join(t_d,"dsi.0.obs.csv")) - assert oe.shape[0]==50, f"{50-oe.shape} failed runs" + oe = pyemu.ObservationEnsemble.from_csv(pst=pst, filename=os.path.join(m_d,"dsi.0.obs.csv")) + assert oe.shape[0]==50, f"{50-oe.shape[0]} failed runs" phi_vector = oe.phi_vector.sort_values().values assert phi_vector[0] != phi_vector[1],phi_vector @@ -750,7 +742,8 @@ def dsi_normscoretransform_test(): if __name__ == "__main__": #dsi_normscoretransform_test() #ends_freyberg_dev() - ends_freyberg_dsi_test("temp") + #ends_freyberg_dsi_test("temp") + ends_freyberg_dsi_ztz_test('temp') #plot_freyberg_dsi() #obscomp_test() #alternative_dw() diff --git a/autotest/pst_from_tests.py b/autotest/pst_from_tests.py index 0c1b80f7..1a280a9a 100644 --- a/autotest/pst_from_tests.py +++ b/autotest/pst_from_tests.py @@ -14,10 +14,10 @@ local_bins = False # change if wanting to test with local binary exes if local_bins: bin_path = os.path.join("..", "..", "bin") - if "linux" in platform.platform().lower(): + if "linux" in platform.system().lower(): pass bin_path = os.path.join(bin_path, "linux") - elif "darwin" in platform.platform().lower() or 'macos' in platform.platform().lower(): + elif "darwin" in platform.system().lower(): pass bin_path = os.path.join(bin_path, "mac") else: @@ -25,7 +25,7 @@ ext = '.exe' else: bin_path = '' - if "windows" in platform.platform().lower(): + if "windows" in platform.system().lower(): ext = '.exe' mf_exe_path = os.path.join(bin_path, "mfnwt") @@ -78,16 +78,17 @@ def _gen_dummy_obs_file(ws='.', sep=',', ext=None): return fnme, df -def setup_tmp(od, tmp_path, sub=None): +def setup_tmp(od, tmp_d, sub=None): basename = Path(od).name if sub is not None: - new_d = Path(tmp_path, basename, sub) + new_d = Path(tmp_d, basename, sub) else: - new_d = Path(tmp_path, basename) + new_d = Path(tmp_d, basename) if new_d.exists(): shutil.rmtree(new_d) - Path(tmp_path).mkdir(exist_ok=True) + Path(tmp_d).mkdir(exist_ok=True) # creation functionality + assert Path(od).exists(), f"can't find {Path(od).absolute()}" shutil.copytree(od, new_d) return new_d @@ -195,6 +196,7 @@ def freyberg_test(tmp_path): pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) + try: import flopy except: @@ -203,214 +205,211 @@ def freyberg_test(tmp_path): org_model_ws = os.path.join("..", "examples", "freyberg_sfr_update") tmp_model_ws = setup_tmp(org_model_ws, tmp_path) - bd = Path.cwd() os.chdir(tmp_path) nam_file = "freyberg.nam" - try: - org_model_ws = tmp_model_ws.relative_to(tmp_path) - m = flopy.modflow.Modflow.load(nam_file, model_ws=org_model_ws, - check=False, forgive=False, - exe_name=mf_exe_path) - flopy.modflow.ModflowRiv(m, stress_period_data={ - 0: [[0, 0, 0, m.dis.top.array[0, 0], 1.0, m.dis.botm.array[0, 0, 0]], - [0, 0, 1, m.dis.top.array[0, 1], 1.0, m.dis.botm.array[0, 0, 1]], - [0, 0, 1, m.dis.top.array[0, 1], 1.0, m.dis.botm.array[0, 0, 1]]]}) - - m.external_path = "." - m.write_input() - print("{0} {1}".format(mf_exe_path, m.name + ".nam"), org_model_ws) - os_utils.run("{0} {1}".format(mf_exe_path, m.name + ".nam"), - cwd=org_model_ws) - hds_kperk = [] - for k in range(m.nlay): - for kper in range(m.nper): - hds_kperk.append([kper, k]) - hds_runline, df = pyemu.gw_utils.setup_hds_obs( - os.path.join(org_model_ws, f"{m.name}.hds"), kperk_pairs=None, skip=None, - prefix="hds", include_path=False) - pyemu.gw_utils.apply_hds_obs(os.path.join(org_model_ws, f"{m.name}.hds")) - - sfo = flopy.utils.SfrFile(os.path.join(m.model_ws, 'freyberg.sfr.out')) - sfodf = sfo.get_dataframe() - sfodf[['kstp', 'kper']] = pd.DataFrame(sfodf.kstpkper.to_list(), - index=sfodf.index) - sfodf = sfodf.drop('kstpkper', axis=1) - # just adding a bit of header in for test purposes - sfo_pp_file = os.path.join(m.model_ws, 'freyberg.sfo.dat') - with open(sfo_pp_file, 'w') as fp: - fp.writelines(["This is a post processed sfr output file\n", - "Processed into tabular form using the lines:\n", - "sfo = flopy.utils.SfrFile('freyberg.sfr.out')\n", - "sfo.get_dataframe().to_csv('freyberg.sfo.dat')\n"]) - sfodf.sort_index(axis=1).to_csv(fp, sep=' ', index_label='idx', lineterminator='\n') - sfodf.sort_index(axis=1).to_csv(os.path.join(m.model_ws, 'freyberg.sfo.csv'), - index_label='idx',lineterminator='\n') - template_ws = "new_temp" - if os.path.exists(template_ws): - shutil.rmtree(template_ws) + org_model_ws = tmp_model_ws.relative_to(tmp_path) - # sr0 = m.sr - sr = pyemu.helpers.SpatialReference.from_namfile( - os.path.join(m.model_ws, m.namefile), - delr=m.dis.delr, delc=m.dis.delc) - # set up PstFrom object - pf = PstFrom(original_d=org_model_ws, new_d=template_ws, - remove_existing=True, - longnames=True, spatial_reference=sr, - zero_based=False) - # obs - # using tabular style model output - # (generated by pyemu.gw_utils.setup_hds_obs()) - f, fdf = _gen_dummy_obs_file(pf.new_d) - pf.add_observations(f, index_cols='idx', use_cols='yes') - pf.add_py_function(_gen_dummy_obs_file, '_gen_dummy_obs_file()', + m = flopy.modflow.Modflow.load(nam_file, model_ws=org_model_ws, + check=False, forgive=False, + exe_name=mf_exe_path) + flopy.modflow.ModflowRiv(m, stress_period_data={ + 0: [[0, 0, 0, m.dis.top.array[0, 0], 1.0, m.dis.botm.array[0, 0, 0]], + [0, 0, 1, m.dis.top.array[0, 1], 1.0, m.dis.botm.array[0, 0, 1]], + [0, 0, 1, m.dis.top.array[0, 1], 1.0, m.dis.botm.array[0, 0, 1]]]}) + + m.external_path = "." + m.write_input() + print("{0} {1}".format(mf_exe_path, m.name + ".nam"), org_model_ws) + os_utils.run("{0} {1}".format(mf_exe_path, m.name + ".nam"), + cwd=org_model_ws) + # hds_kperk = [] + # for k in range(m.nlay): + # for kper in range(m.nper): + # hds_kperk.append([kper, k]) + + #hds_runline, df = pyemu.gw_utils.setup_hds_obs( + # os.path.join(org_model_ws, f"{m.name}.hds"), kperk_pairs=None, skip=None, + # prefix="hds", include_path=False, precision="double") + #pyemu.gw_utils.apply_hds_obs(os.path.join(org_model_ws, f"{m.name}.hds"),precision="double") + + + sfo = flopy.utils.SfrFile(os.path.join(m.model_ws, 'freyberg.sfr.out')) + sfodf = sfo.get_dataframe() + sfodf[['kstp', 'kper']] = pd.DataFrame(sfodf.kstpkper.to_list(), + index=sfodf.index) + sfodf = sfodf.drop('kstpkper', axis=1) + # just adding a bit of header in for test purposes + sfo_pp_file = os.path.join(m.model_ws, 'freyberg.sfo.dat') + with open(sfo_pp_file, 'w') as fp: + fp.writelines(["This is a post processed sfr output file\n", + "Processed into tabular form using the lines:\n", + "sfo = flopy.utils.SfrFile('freyberg.sfr.out')\n", + "sfo.get_dataframe().to_csv('freyberg.sfo.dat')\n"]) + sfodf.sort_index(axis=1).to_csv(fp, sep=' ', index_label='idx', lineterminator='\n') + sfodf.sort_index(axis=1).to_csv(os.path.join(m.model_ws, 'freyberg.sfo.csv'), + index_label='idx',lineterminator='\n') + template_ws = "new_temp" + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + + # sr0 = m.sr + sr = pyemu.helpers.SpatialReference.from_namfile( + os.path.join(m.model_ws, m.namefile), + delr=m.dis.delr, delc=m.dis.delc) + # set up PstFrom object + pf = PstFrom(original_d=org_model_ws, new_d=template_ws, + remove_existing=True, + longnames=True, spatial_reference=sr, + zero_based=False) + # obs + # using tabular style model output + # (generated by pyemu.gw_utils.setup_hds_obs()) + f, fdf = _gen_dummy_obs_file(pf.new_d) + pf.add_observations(f, index_cols='idx', use_cols='yes') + pf.add_py_function(_gen_dummy_obs_file, '_gen_dummy_obs_file()', + is_pre_cmd=False) + #pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', + # index_cols='obsnme', use_cols='obsval', prefix='hds') + # using the ins file generated by pyemu.gw_utils.setup_hds_obs() + #pf.add_observations_from_ins(ins_file='freyberg.hds.dat.ins') + #pf.post_py_cmds.append(hds_runline) + pf.tmp_files.append(f"{m.name}.hds") + # sfr outputs to obs + sfr_idx = ['segment', 'reach', 'kstp', 'kper'] + sfr_use = ["Qaquifer", "Qout", 'width'] + with pytest.warns(PyemuWarning): + pf.add_py_function(__file__, '_gen_dummy_obs_file()', is_pre_cmd=False) - pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', - index_cols='obsnme', use_cols='obsval', prefix='hds') - # using the ins file generated by pyemu.gw_utils.setup_hds_obs() - pf.add_observations_from_ins(ins_file='freyberg.hds.dat.ins') - pf.post_py_cmds.append(hds_runline) - pf.tmp_files.append(f"{m.name}.hds") - # sfr outputs to obs - sfr_idx = ['segment', 'reach', 'kstp', 'kper'] - sfr_use = ["Qaquifer", "Qout", 'width'] - with pytest.warns(PyemuWarning): - pf.add_py_function(__file__, '_gen_dummy_obs_file()', - is_pre_cmd=False) - pf.add_observations('freyberg.sfo.dat', insfile=None, - index_cols=sfr_idx, - use_cols=sfr_use, prefix='sfr', - ofile_skip=4, ofile_sep=' ', use_rows=np.arange(0, 50)) - # check obs set up - sfrobs = pf.obs_dfs[-1].copy() - sfrobs[["oname","otype",'usecol'] + sfr_idx] = sfrobs.obsnme.apply( - lambda x: pd.Series( - dict([s.split(':') for s in x.split('_') if ':' in s]))) - sfrobs.pop("oname") - sfrobs.pop("otype") - sfrobs.loc[:, sfr_idx] = sfrobs.loc[:, sfr_idx].astype(int) - sfrobs_p = sfrobs.pivot_table(index=sfr_idx, - columns=['usecol'], values='obsval') - sfodf_c = sfodf.set_index(sfr_idx).sort_index() - sfodf_c.columns = sfodf_c.columns.str.lower() - assert (sfrobs_p == sfodf_c.loc[sfrobs_p.index, - sfrobs_p.columns]).all().all(), ( - "Mis-match between expected and processed obs values\n", - sfrobs_p.head(), - sfodf_c.loc[sfrobs_p.index, sfrobs_p.columns].head()) - - pf.tmp_files.append(f"{m.name}.sfr.out") - pf.extra_py_imports.append('flopy') - pf.post_py_cmds.extend( - ["sfo_pp_file = 'freyberg.sfo.dat'", - "sfo = flopy.utils.SfrFile('freyberg.sfr.out')", - "sfodf = sfo.get_dataframe()", - "sfodf[['kstp', 'kper']] = pd.DataFrame(sfodf.kstpkper.to_list(), index=sfodf.index)", - "sfodf = sfodf.drop('kstpkper', axis=1)", - "with open(sfo_pp_file, 'w') as fp:", - " fp.writelines(['This is a post processed sfr output file\\n', " - "'Processed into tabular form using the lines:\\n', " - "'sfo = flopy.utils.SfrFile(`freyberg.sfr.out`)\\n', " - "'sfo.get_dataframe().to_csv(`freyberg.sfo.dat`)\\n'])", - " sfodf.sort_index(axis=1).to_csv(fp, sep=' ', index_label='idx',lineterminator='\\n')"]) - # csv version of sfr obs - # sfr outputs to obs - pf.add_observations('freyberg.sfo.csv', insfile=None, - index_cols=['segment', 'reach', 'kstp', 'kper'], - use_cols=["Qaquifer", "Qout", "width"], prefix='sfr2', - ofile_sep=',', obsgp=['qaquifer', 'qout', "width"], - use_rows=np.arange(50, 101)) - # check obs set up - sfrobs = pf.obs_dfs[-1].copy() - sfrobs[['oname','otype','usecol'] + sfr_idx] = sfrobs.obsnme.apply( - lambda x: pd.Series( - dict([s.split(':') for s in x.split('_') if ':' in s]))) - sfrobs.pop("oname") - sfrobs.pop("otype") - sfrobs.loc[:, sfr_idx] = sfrobs.loc[:, sfr_idx].astype(int) - sfrobs_p = sfrobs.pivot_table(index=sfr_idx, - columns=['usecol'], values='obsval') - sfodf_c = sfodf.set_index(sfr_idx).sort_index() - sfodf_c.columns = sfodf_c.columns.str.lower() - assert (sfrobs_p == sfodf_c.loc[sfrobs_p.index, - sfrobs_p.columns]).all().all(), ( - "Mis-match between expected and processed obs values") - obsnmes = pd.concat([df.obgnme for df in pf.obs_dfs]).unique() - assert all([gp in obsnmes for gp in ['qaquifer', 'qout']]) - pf.post_py_cmds.append( - "sfodf.sort_index(axis=1).to_csv('freyberg.sfo.csv', sep=',', index_label='idx')") - zone_array = np.arange(m.nlay*m.nrow*m.ncol) - s = lambda x: "zval_"+str(x) - zone_array = np.array([s(x) for x in zone_array]).reshape(m.nlay,m.nrow,m.ncol) - # pars - pf.add_parameters(filenames="RIV_0000.dat", par_type="grid", - index_cols=[0, 1, 2], use_cols=[3, 5], - par_name_base=["rivstage_grid", "rivbot_grid"], - mfile_fmt='%10d%10d%10d %15.8F %15.8F %15.8F', - pargp='rivbot') - pf.add_parameters(filenames="RIV_0000.dat", par_type="grid", - index_cols=[0, 1, 2], use_cols=4) - pf.add_parameters(filenames=["WEL_0000.dat", "WEL_0001.dat"], - par_type="grid", index_cols=[0, 1, 2], use_cols=3, - par_name_base="welflux_grid", - zone_array=zone_array) - pf.add_parameters(filenames="WEL_0000.dat", - par_type="grid", index_cols=[0, 1, 2], use_cols=3, - par_name_base="welflux_grid_direct", - zone_array=zone_array,par_style="direct",transform="none") - pf.add_parameters(filenames=["WEL_0000.dat"], par_type="constant", - index_cols=[0, 1, 2], use_cols=3, - par_name_base=["flux_const"]) - pf.add_parameters(filenames="rech_1.ref", par_type="grid", - zone_array=m.bas6.ibound[0].array, - par_name_base="rch_datetime:1-1-1970") - pf.add_parameters(filenames=["rech_1.ref", "rech_2.ref"], - par_type="zone", zone_array=m.bas6.ibound[0].array) - pf.add_parameters(filenames="rech_1.ref", par_type="pilot_point", - zone_array=m.bas6.ibound[0].array, - par_name_base="rch_datetime:1-1-1970", pp_space=4) - pf.add_parameters(filenames="rech_1.ref", par_type="pilot_point", - zone_array=m.bas6.ibound[0].array, - par_name_base="rch_datetime:1-1-1970", pp_space=1, - ult_ubound=100, ult_lbound=0.0) - pf.add_parameters(filenames="rech_1.ref", par_type="pilot_point", - par_name_base="rch_datetime:1-1-1970", pp_space=1, - ult_ubound=100, ult_lbound=0.0) - - - # add model run command - pf.mod_sys_cmds.append("{0} {1}".format(mf_exe_name, m.name + ".nam")) - print(pf.mult_files) - print(pf.org_files) - - - # build pest - pst = pf.build_pst('freyberg.pst') - - # check mult files are in pst input files - csv = os.path.join(template_ws, "mult2model_info.csv") - df = pd.read_csv(csv, index_col=0) - df = df.loc[pd.notna(df.mlt_file),:] - pst_input_files = {str(f) for f in pst.input_files} - mults_not_linked_to_pst = ((set(df.mlt_file.unique()) - - pst_input_files) - - set(df.loc[df.pp_file.notna()].mlt_file)) - assert len(mults_not_linked_to_pst) == 0, print(mults_not_linked_to_pst) - check_apply(pf) - pst.control_data.noptmax = 0 - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - - res_file = os.path.join(pf.new_d, "freyberg.base.rei") - assert os.path.exists(res_file), res_file - pst.set_res(res_file) - print(pst.phi) - assert np.isclose(pst.phi, 0.), pst.phi - except Exception as e: - os.chdir(bd) - raise e - os.chdir(bd) + pf.add_observations('freyberg.sfo.dat', insfile=None, + index_cols=sfr_idx, + use_cols=sfr_use, prefix='sfr', + ofile_skip=4, ofile_sep=' ', use_rows=np.arange(0, 50)) + # check obs set up + sfrobs = pf.obs_dfs[-1].copy() + sfrobs[["oname","otype",'usecol'] + sfr_idx] = sfrobs.obsnme.apply( + lambda x: pd.Series( + dict([s.split(':') for s in x.split('_') if ':' in s]))) + sfrobs.pop("oname") + sfrobs.pop("otype") + sfrobs.loc[:, sfr_idx] = sfrobs.loc[:, sfr_idx].astype(int) + sfrobs_p = sfrobs.pivot_table(index=sfr_idx, + columns=['usecol'], values='obsval') + sfodf_c = sfodf.set_index(sfr_idx).sort_index() + sfodf_c.columns = sfodf_c.columns.str.lower() + assert (sfrobs_p == sfodf_c.loc[sfrobs_p.index, + sfrobs_p.columns]).all().all(), ( + "Mis-match between expected and processed obs values\n", + sfrobs_p.head(), + sfodf_c.loc[sfrobs_p.index, sfrobs_p.columns].head()) + + pf.tmp_files.append(f"{m.name}.sfr.out") + pf.extra_py_imports.append('flopy') + pf.post_py_cmds.extend( + ["sfo_pp_file = 'freyberg.sfo.dat'", + "sfo = flopy.utils.SfrFile('freyberg.sfr.out')", + "sfodf = sfo.get_dataframe()", + "sfodf[['kstp', 'kper']] = pd.DataFrame(sfodf.kstpkper.to_list(), index=sfodf.index)", + "sfodf = sfodf.drop('kstpkper', axis=1)", + "with open(sfo_pp_file, 'w') as fp:", + " fp.writelines(['This is a post processed sfr output file\\n', " + "'Processed into tabular form using the lines:\\n', " + "'sfo = flopy.utils.SfrFile(`freyberg.sfr.out`)\\n', " + "'sfo.get_dataframe().to_csv(`freyberg.sfo.dat`)\\n'])", + " sfodf.sort_index(axis=1).to_csv(fp, sep=' ', index_label='idx',lineterminator='\\n')"]) + # csv version of sfr obs + # sfr outputs to obs + pf.add_observations('freyberg.sfo.csv', insfile=None, + index_cols=['segment', 'reach', 'kstp', 'kper'], + use_cols=["Qaquifer", "Qout", "width"], prefix='sfr2', + ofile_sep=',', obsgp=['qaquifer', 'qout', "width"], + use_rows=np.arange(50, 101)) + # check obs set up + sfrobs = pf.obs_dfs[-1].copy() + sfrobs[['oname','otype','usecol'] + sfr_idx] = sfrobs.obsnme.apply( + lambda x: pd.Series( + dict([s.split(':') for s in x.split('_') if ':' in s]))) + sfrobs.pop("oname") + sfrobs.pop("otype") + sfrobs.loc[:, sfr_idx] = sfrobs.loc[:, sfr_idx].astype(int) + sfrobs_p = sfrobs.pivot_table(index=sfr_idx, + columns=['usecol'], values='obsval') + sfodf_c = sfodf.set_index(sfr_idx).sort_index() + sfodf_c.columns = sfodf_c.columns.str.lower() + assert (sfrobs_p == sfodf_c.loc[sfrobs_p.index, + sfrobs_p.columns]).all().all(), ( + "Mis-match between expected and processed obs values") + obsnmes = pd.concat([df.obgnme for df in pf.obs_dfs]).unique() + assert all([gp in obsnmes for gp in ['qaquifer', 'qout']]) + pf.post_py_cmds.append( + "sfodf.sort_index(axis=1).to_csv('freyberg.sfo.csv', sep=',', index_label='idx')") + zone_array = np.arange(m.nlay*m.nrow*m.ncol) + s = lambda x: "zval_"+str(x) + zone_array = np.array([s(x) for x in zone_array]).reshape(m.nlay,m.nrow,m.ncol) + # pars + pf.add_parameters(filenames="RIV_0000.dat", par_type="grid", + index_cols=[0, 1, 2], use_cols=[3, 5], + par_name_base=["rivstage_grid", "rivbot_grid"], + mfile_fmt='%10d%10d%10d %15.8F %15.8F %15.8F', + pargp='rivbot') + pf.add_parameters(filenames="RIV_0000.dat", par_type="grid", + index_cols=[0, 1, 2], use_cols=4) + pf.add_parameters(filenames=["WEL_0000.dat", "WEL_0001.dat"], + par_type="grid", index_cols=[0, 1, 2], use_cols=3, + par_name_base="welflux_grid", + zone_array=zone_array) + pf.add_parameters(filenames="WEL_0000.dat", + par_type="grid", index_cols=[0, 1, 2], use_cols=3, + par_name_base="welflux_grid_direct", + zone_array=zone_array,par_style="direct",transform="none") + pf.add_parameters(filenames=["WEL_0000.dat"], par_type="constant", + index_cols=[0, 1, 2], use_cols=3, + par_name_base=["flux_const"]) + pf.add_parameters(filenames="rech_1.ref", par_type="grid", + zone_array=m.bas6.ibound[0].array, + par_name_base="rch_datetime:1-1-1970") + pf.add_parameters(filenames=["rech_1.ref", "rech_2.ref"], + par_type="zone", zone_array=m.bas6.ibound[0].array) + pf.add_parameters(filenames="rech_1.ref", par_type="pilot_point", + zone_array=m.bas6.ibound[0].array, + par_name_base="rch_datetime:1-1-1970", pp_space=4) + pf.add_parameters(filenames="rech_1.ref", par_type="pilot_point", + zone_array=m.bas6.ibound[0].array, + par_name_base="rch_datetime:1-1-1970", pp_space=1, + ult_ubound=100, ult_lbound=0.0) + pf.add_parameters(filenames="rech_1.ref", par_type="pilot_point", + par_name_base="rch_datetime:1-1-1970", pp_space=1, + ult_ubound=100, ult_lbound=0.0) + + + # add model run command + pf.mod_sys_cmds.append("{0} {1}".format(mf_exe_name, m.name + ".nam")) + print(pf.mult_files) + print(pf.org_files) + + + # build pest + pst = pf.build_pst('freyberg.pst') + + # check mult files are in pst input files + csv = os.path.join(template_ws, "mult2model_info.csv") + df = pd.read_csv(csv, index_col=0) + df = df.loc[pd.notna(df.mlt_file),:] + pst_input_files = {str(f) for f in pst.input_files} + mults_not_linked_to_pst = ((set(df.mlt_file.unique()) - + pst_input_files) - + set(df.loc[df.pp_file.notna()].mlt_file)) + assert len(mults_not_linked_to_pst) == 0, print(mults_not_linked_to_pst) + check_apply(pf) + pst.control_data.noptmax = 0 + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + + res_file = os.path.join(pf.new_d, "freyberg.base.rei") + assert os.path.exists(res_file), res_file + pst.set_res(res_file) + print(pst.phi) + assert np.isclose(pst.phi, 0.), pst.phi def freyberg_prior_build_test(tmp_path): @@ -426,245 +425,234 @@ def freyberg_prior_build_test(tmp_path): org_model_ws = os.path.join("..", "examples", "freyberg_sfr_update") tmp_model_ws = setup_tmp(org_model_ws, tmp_path) - bd = Path.cwd() os.chdir(tmp_path) nam_file = "freyberg.nam" - try: - org_model_ws = tmp_model_ws.relative_to(tmp_path) - m = flopy.modflow.Modflow.load(nam_file, model_ws=org_model_ws, - check=False, forgive=False, - exe_name=mf_exe_path) - flopy.modflow.ModflowRiv(m, stress_period_data={ - 0: [[0, 0, 0, m.dis.top.array[0, 0], 1.0, m.dis.botm.array[0, 0, 0]], - [0, 0, 1, m.dis.top.array[0, 1], 1.0, m.dis.botm.array[0, 0, 1]], - [0, 0, 1, m.dis.top.array[0, 1], 1.0, m.dis.botm.array[0, 0, 1]]]}) - - welsp = m.wel.stress_period_data.data.copy() - addwell = welsp[0].copy() - addwell['k'] = 1 - welsp[0] = np.rec.array(np.concatenate([welsp[0], addwell])) - samewell = welsp[1].copy() - samewell['flux'] *= 10 - welsp[1] = np.rec.array(np.concatenate([welsp[1], samewell])) - m.wel.stress_period_data = welsp - - m.external_path = "." - m.write_input() - - # for exe in [mf_exe_path, mt_exe_path, ies_exe_path]: - # shutil.copy(os.path.relpath(exe, '..'), org_model_ws) - - print("{0} {1}".format(mf_exe_path, m.name + ".nam"), org_model_ws) - os_utils.run("{0} {1}".format(mf_exe_path, m.name + ".nam"), - cwd=org_model_ws) - hds_kperk = [] - for k in range(m.nlay): - for kper in range(m.nper): - hds_kperk.append([kper, k]) - hds_runline, df = pyemu.gw_utils.setup_hds_obs( - os.path.join(m.model_ws, f"{m.name}.hds"), kperk_pairs=None, skip=None, - prefix="hds", include_path=False) - pyemu.gw_utils.apply_hds_obs(os.path.join(m.model_ws, f"{m.name}.hds")) - - template_ws = Path(tmp_path, "new_temp") - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - # sr0 = m.sr - sr = pyemu.helpers.SpatialReference.from_namfile( - os.path.join(m.model_ws, m.namefile), - delr=m.dis.delr, delc=m.dis.delc) - # set up PstFrom object - pf = PstFrom(original_d=org_model_ws, new_d=template_ws, - remove_existing=True, - longnames=True, spatial_reference=sr, - zero_based=False) - pf.extra_py_imports.append('flopy') - if "linux" in platform.platform().lower(): - pf.mod_sys_cmds.append("which python") - # obs - # using tabular style model output - # (generated by pyemu.gw_utils.setup_hds_obs()) - pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', - index_cols='obsnme', use_cols='obsval', prefix='hds') - pf.post_py_cmds.append(hds_runline) - pf.tmp_files.append(f"{m.name}.hds") - - # pars - v = pyemu.geostats.ExpVario(contribution=1.0, a=2500) - geostruct = pyemu.geostats.GeoStruct(variograms=v, transform='log') - # Pars for river list style model file, every entry in columns 3 and 4 - # specifying formatted model file and passing a geostruct # TODO method for appending specific ult bounds - # pf.add_parameters(filenames="RIV_0000.dat", par_type="grid", - # index_cols=[0, 1, 2], use_cols=[3, 4], - # par_name_base=["rivstage_grid", "rivcond_grid"], - # mfile_fmt='%10d%10d%10d %15.8F %15.8F %15.8F', - # geostruct=geostruct, lower_bound=[0.9, 0.01], - # upper_bound=[1.1, 100.], ult_lbound=[0.3, None]) - # # 2 constant pars applied to columns 3 and 4 - # # this time specifying free formatted model file - # pf.add_parameters(filenames="RIV_0000.dat", par_type="constant", - # index_cols=[0, 1, 2], use_cols=[3, 4], - # par_name_base=["rivstage", "rivcond"], - # mfile_fmt='free', lower_bound=[0.9, 0.01], - # upper_bound=[1.1, 100.], ult_lbound=[None, 0.01]) - # Pars for river list style model file, every entry in column 4 - pf.add_parameters(filenames="RIV_0000.dat", par_type="grid", - index_cols=[0, 1, 2], use_cols=[4], - par_name_base=["rivcond_grid"], - mfile_fmt='%10d%10d%10d %15.8F %15.8F %15.8F', - geostruct=geostruct, lower_bound=[0.01], - upper_bound=[100.], ult_lbound=[None]) - # constant par applied to column 4 - # this time specifying free formatted model file - pf.add_parameters(filenames="RIV_0000.dat", par_type="constant", - index_cols=[0, 1, 2], use_cols=[4], - par_name_base=["rivcond"], - mfile_fmt='free', lower_bound=[0.01], - upper_bound=[100.], ult_lbound=[0.01]) - # pf.add_parameters(filenames="RIV_0000.dat", par_type="constant", - # index_cols=[0, 1, 2], use_cols=5, - # par_name_base="rivbot", - # mfile_fmt='free', lower_bound=0.9, - # upper_bound=1.1, ult_ubound=100., - # ult_lbound=0.001) - # setting up temporal variogram for correlating temporal pars - date = m.dis.start_datetime - v = pyemu.geostats.ExpVario(contribution=1.0, a=180.0) # 180 correlation length - t_geostruct = pyemu.geostats.GeoStruct(variograms=v, transform='log') - # looping over temporal list style input files - # setting up constant parameters for col 3 for each temporal file - # making sure all are set up with same pargp and geostruct (to ensure correlation) - # Parameters for wel list style - well_mfiles = ["WEL_0000.dat", "WEL_0001.dat", "WEL_0002.dat"] - for t, well_file in enumerate(well_mfiles): - # passing same temporal geostruct and pargp, - # date is incremented and will be used for correlation with - pf.add_parameters(filenames=well_file, par_type="constant", - index_cols=[0, 1, 2], use_cols=3, - par_name_base="flux", alt_inst_str='kper', - datetime=date, geostruct=t_geostruct, - pargp='wellflux_t', lower_bound=0.25, - upper_bound=1.75) - date = (pd.to_datetime(date) + - pd.DateOffset(m.dis.perlen.array[t], 'day')) - # par for each well (same par through time) - pf.add_parameters(filenames=well_mfiles, - par_type="grid", index_cols=[0, 1, 2], use_cols=3, - par_name_base="welflux_grid", - zone_array=m.bas6.ibound.array, - geostruct=geostruct, lower_bound=0.25, upper_bound=1.75) - pf.add_parameters(filenames=well_mfiles, - par_type="grid", index_cols=[0, 1, 2], use_cols=3, - par_name_base="welflux_grid", - zone_array=m.bas6.ibound.array, - use_rows=(1, 3, 4), - geostruct=geostruct, lower_bound=0.25, upper_bound=1.75) - # global constant across all files - pf.add_parameters(filenames=well_mfiles, - par_type="constant", + org_model_ws = tmp_model_ws.relative_to(tmp_path) + m = flopy.modflow.Modflow.load(nam_file, model_ws=org_model_ws, + check=False, forgive=False, + exe_name=mf_exe_path) + flopy.modflow.ModflowRiv(m, stress_period_data={ + 0: [[0, 0, 0, m.dis.top.array[0, 0], 1.0, m.dis.botm.array[0, 0, 0]], + [0, 0, 1, m.dis.top.array[0, 1], 1.0, m.dis.botm.array[0, 0, 1]], + [0, 0, 1, m.dis.top.array[0, 1], 1.0, m.dis.botm.array[0, 0, 1]]]}) + + welsp = m.wel.stress_period_data.data.copy() + addwell = welsp[0].copy() + addwell['k'] = 1 + welsp[0] = np.rec.array(np.concatenate([welsp[0], addwell])) + samewell = welsp[1].copy() + samewell['flux'] *= 10 + welsp[1] = np.rec.array(np.concatenate([welsp[1], samewell])) + m.wel.stress_period_data = welsp + + m.external_path = "." + m.write_input() + + # for exe in [mf_exe_path, mt_exe_path, ies_exe_path]: + # shutil.copy(os.path.relpath(exe, '..'), org_model_ws) + + print("{0} {1}".format(mf_exe_path, m.name + ".nam"), org_model_ws) + os_utils.run("{0} {1}".format(mf_exe_path, m.name + ".nam"), + cwd=org_model_ws) + hds_kperk = [] + for k in range(m.nlay): + for kper in range(m.nper): + hds_kperk.append([kper, k]) + hds_runline, df = pyemu.gw_utils.setup_hds_obs( + os.path.join(m.model_ws, f"{m.name}.hds"), kperk_pairs=None, skip=None, + prefix="hds", include_path=False) + pyemu.gw_utils.apply_hds_obs(os.path.join(m.model_ws, f"{m.name}.hds")) + + template_ws = Path(tmp_path, "new_temp") + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + # sr0 = m.sr + sr = pyemu.helpers.SpatialReference.from_namfile( + os.path.join(m.model_ws, m.namefile), + delr=m.dis.delr, delc=m.dis.delc) + # set up PstFrom object + pf = PstFrom(original_d=org_model_ws, new_d=template_ws, + remove_existing=True, + longnames=True, spatial_reference=sr, + zero_based=False) + pf.extra_py_imports.append('flopy') + if "linux" in platform.platform().lower(): + pf.mod_sys_cmds.append("which python") + # obs + # using tabular style model output + # (generated by pyemu.gw_utils.setup_hds_obs()) + pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', + index_cols='obsnme', use_cols='obsval', prefix='hds') + pf.post_py_cmds.append(hds_runline) + pf.tmp_files.append(f"{m.name}.hds") + + # pars + v = pyemu.geostats.ExpVario(contribution=1.0, a=2500) + geostruct = pyemu.geostats.GeoStruct(variograms=v, transform='log') + # Pars for river list style model file, every entry in columns 3 and 4 + # specifying formatted model file and passing a geostruct # TODO method for appending specific ult bounds + # pf.add_parameters(filenames="RIV_0000.dat", par_type="grid", + # index_cols=[0, 1, 2], use_cols=[3, 4], + # par_name_base=["rivstage_grid", "rivcond_grid"], + # mfile_fmt='%10d%10d%10d %15.8F %15.8F %15.8F', + # geostruct=geostruct, lower_bound=[0.9, 0.01], + # upper_bound=[1.1, 100.], ult_lbound=[0.3, None]) + # # 2 constant pars applied to columns 3 and 4 + # # this time specifying free formatted model file + # pf.add_parameters(filenames="RIV_0000.dat", par_type="constant", + # index_cols=[0, 1, 2], use_cols=[3, 4], + # par_name_base=["rivstage", "rivcond"], + # mfile_fmt='free', lower_bound=[0.9, 0.01], + # upper_bound=[1.1, 100.], ult_lbound=[None, 0.01]) + # Pars for river list style model file, every entry in column 4 + pf.add_parameters(filenames="RIV_0000.dat", par_type="grid", + index_cols=[0, 1, 2], use_cols=[4], + par_name_base=["rivcond_grid"], + mfile_fmt='%10d%10d%10d %15.8F %15.8F %15.8F', + geostruct=geostruct, lower_bound=[0.01], + upper_bound=[100.], ult_lbound=[None]) + # constant par applied to column 4 + # this time specifying free formatted model file + pf.add_parameters(filenames="RIV_0000.dat", par_type="constant", + index_cols=[0, 1, 2], use_cols=[4], + par_name_base=["rivcond"], + mfile_fmt='free', lower_bound=[0.01], + upper_bound=[100.], ult_lbound=[0.01]) + # pf.add_parameters(filenames="RIV_0000.dat", par_type="constant", + # index_cols=[0, 1, 2], use_cols=5, + # par_name_base="rivbot", + # mfile_fmt='free', lower_bound=0.9, + # upper_bound=1.1, ult_ubound=100., + # ult_lbound=0.001) + # setting up temporal variogram for correlating temporal pars + date = m.dis.start_datetime + v = pyemu.geostats.ExpVario(contribution=1.0, a=180.0) # 180 correlation length + t_geostruct = pyemu.geostats.GeoStruct(variograms=v, transform='log') + # looping over temporal list style input files + # setting up constant parameters for col 3 for each temporal file + # making sure all are set up with same pargp and geostruct (to ensure correlation) + # Parameters for wel list style + well_mfiles = ["WEL_0000.dat", "WEL_0001.dat", "WEL_0002.dat"] + for t, well_file in enumerate(well_mfiles): + # passing same temporal geostruct and pargp, + # date is incremented and will be used for correlation with + pf.add_parameters(filenames=well_file, par_type="constant", index_cols=[0, 1, 2], use_cols=3, - par_name_base=["flux_global"], - lower_bound=0.25, upper_bound=1.75) - - # Spatial array style pars - cell-by-cell - hk_files = ["hk_Layer_{0:d}.ref".format(i) for i in range(1, 4)] - for hk in hk_files: - pf.add_parameters(filenames=hk, par_type="grid", - zone_array=m.bas6.ibound[0].array, - par_name_base="hk", alt_inst_str='lay', - geostruct=geostruct, - lower_bound=0.01, upper_bound=100.) - - # Pars for temporal array style model files - date = m.dis.start_datetime # reset date - rch_mfiles = ["rech_0.ref", "rech_1.ref", "rech_2.ref"] - for t, rch_file in enumerate(rch_mfiles): - # constant par for each file but linked by geostruct and pargp - pf.add_parameters(filenames=rch_file, par_type="constant", - zone_array=m.bas6.ibound[0].array, - par_name_base="rch", alt_inst_str='kper', - datetime=date, geostruct=t_geostruct, - pargp='rch_t', lower_bound=0.9, upper_bound=1.1) - date = (pd.to_datetime(date) + - pd.DateOffset(m.dis.perlen.array[t], 'day')) - # spatially distributed array style pars - cell-by-cell - # pf.add_parameters(filenames=rch_mfiles, par_type="grid", - # zone_array=m.bas6.ibound[0].array, - # par_name_base="rch", - # geostruct=geostruct) - pf.add_parameters(filenames=rch_mfiles, par_type="pilot_point", + par_name_base="flux", alt_inst_str='kper', + datetime=date, geostruct=t_geostruct, + pargp='wellflux_t', lower_bound=0.25, + upper_bound=1.75) + date = (pd.to_datetime(date) + + pd.DateOffset(m.dis.perlen.array[t], 'day')) + # par for each well (same par through time) + pf.add_parameters(filenames=well_mfiles, + par_type="grid", index_cols=[0, 1, 2], use_cols=3, + par_name_base="welflux_grid", + zone_array=m.bas6.ibound.array, + geostruct=geostruct, lower_bound=0.25, upper_bound=1.75) + pf.add_parameters(filenames=well_mfiles, + par_type="grid", index_cols=[0, 1, 2], use_cols=3, + par_name_base="welflux_grid", + zone_array=m.bas6.ibound.array, + use_rows=(1, 3, 4), + geostruct=geostruct, lower_bound=0.25, upper_bound=1.75) + # global constant across all files + pf.add_parameters(filenames=well_mfiles, + par_type="constant", + index_cols=[0, 1, 2], use_cols=3, + par_name_base=["flux_global"], + lower_bound=0.25, upper_bound=1.75) + + # Spatial array style pars - cell-by-cell + hk_files = ["hk_Layer_{0:d}.ref".format(i) for i in range(1, 4)] + for hk in hk_files: + pf.add_parameters(filenames=hk, par_type="grid", zone_array=m.bas6.ibound[0].array, - par_name_base="rch", pp_space=1, - ult_ubound=None, ult_lbound=None, - geostruct=geostruct, lower_bound=0.9, upper_bound=1.1) - # global constant recharge par - pf.add_parameters(filenames=rch_mfiles, par_type="constant", + par_name_base="hk", alt_inst_str='lay', + geostruct=geostruct, + lower_bound=0.01, upper_bound=100.) + + # Pars for temporal array style model files + date = m.dis.start_datetime # reset date + rch_mfiles = ["rech_0.ref", "rech_1.ref", "rech_2.ref"] + for t, rch_file in enumerate(rch_mfiles): + # constant par for each file but linked by geostruct and pargp + pf.add_parameters(filenames=rch_file, par_type="constant", zone_array=m.bas6.ibound[0].array, - par_name_base="rch_global", lower_bound=0.9, - upper_bound=1.1) - # zonal recharge pars - pf.add_parameters(filenames=rch_mfiles, - par_type="zone", par_name_base='rch_zone', - lower_bound=0.9, upper_bound=1.1, ult_lbound=1.e-6, - ult_ubound=100.) - - - # add model run command - pf.mod_sys_cmds.append("{0} {1}".format(mf_exe_name, m.name + ".nam")) - print(pf.mult_files) - print(pf.org_files) - - - # build pest - pst = pf.build_pst('freyberg.pst') - cov = pf.build_prior(fmt="ascii") - pe = pf.draw(100, use_specsim=True) - # check mult files are in pst input files - csv = os.path.join(template_ws, "mult2model_info.csv") - df = pd.read_csv(csv, index_col=0) - pst_input_files = {str(f) for f in pst.input_files} - mults_not_linked_to_pst = ((set(df.mlt_file.unique()) - - pst_input_files) - - set(df.loc[df.pp_file.notna()].mlt_file)) - assert len(mults_not_linked_to_pst) == 0, print(mults_not_linked_to_pst) - - pst.write_input_files(pst_path=pf.new_d) - # test par mults are working - os.chdir(pf.new_d) - pyemu.helpers.apply_list_and_array_pars( - arr_par_file="mult2model_info.csv") - os.chdir(tmp_path) + par_name_base="rch", alt_inst_str='kper', + datetime=date, geostruct=t_geostruct, + pargp='rch_t', lower_bound=0.9, upper_bound=1.1) + date = (pd.to_datetime(date) + + pd.DateOffset(m.dis.perlen.array[t], 'day')) + # spatially distributed array style pars - cell-by-cell + # pf.add_parameters(filenames=rch_mfiles, par_type="grid", + # zone_array=m.bas6.ibound[0].array, + # par_name_base="rch", + # geostruct=geostruct) + pf.add_parameters(filenames=rch_mfiles, par_type="pilot_point", + zone_array=m.bas6.ibound[0].array, + par_name_base="rch", pp_space=1, + ult_ubound=None, ult_lbound=None, + geostruct=geostruct, lower_bound=0.9, upper_bound=1.1) + # global constant recharge par + pf.add_parameters(filenames=rch_mfiles, par_type="constant", + zone_array=m.bas6.ibound[0].array, + par_name_base="rch_global", lower_bound=0.9, + upper_bound=1.1) + # zonal recharge pars + pf.add_parameters(filenames=rch_mfiles, + par_type="zone", par_name_base='rch_zone', + lower_bound=0.9, upper_bound=1.1, ult_lbound=1.e-6, + ult_ubound=100.) - pst.control_data.noptmax = 0 - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - - res_file = os.path.join(pf.new_d, "freyberg.base.rei") - assert os.path.exists(res_file), res_file - pst.set_res(res_file) - print(pst.phi) - assert np.isclose(pst.phi, 0), pst.phi - - pe.to_binary(os.path.join(pf.new_d, 'par.jcb')) - - # quick sweep test? - pst.pestpp_options["ies_par_en"] = 'par.jcb' - pst.pestpp_options["ies_num_reals"] = 10 - pst.control_data.noptmax = -1 - # par = pst.parameter_data - # par.loc[:, 'parval1'] = pe.iloc[0].T - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - # pyemu.os_utils.start_workers(pf.new_d, - # exe_rel_path="pestpp-ies", - # pst_rel_path="freyberg.pst", - # num_workers=20, master_dir="master", - # cleanup=False, port=4005) - except Exception as e: - os.chdir(bd) - raise Exception(str(e)) - os.chdir(bd) + + # add model run command + pf.mod_sys_cmds.append("{0} {1}".format(mf_exe_name, m.name + ".nam")) + print(pf.mult_files) + print(pf.org_files) + + + # build pest + pst = pf.build_pst('freyberg.pst') + cov = pf.build_prior(fmt="ascii") + pe = pf.draw(100, use_specsim=True) + # check mult files are in pst input files + csv = os.path.join(template_ws, "mult2model_info.csv") + df = pd.read_csv(csv, index_col=0) + pst_input_files = {str(f) for f in pst.input_files} + mults_not_linked_to_pst = ((set(df.mlt_file.unique()) - + pst_input_files) - + set(df.loc[df.pp_file.notna()].mlt_file)) + assert len(mults_not_linked_to_pst) == 0, print(mults_not_linked_to_pst) + + check_apply(pf) + + pst.control_data.noptmax = 0 + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + + res_file = os.path.join(pf.new_d, "freyberg.base.rei") + assert os.path.exists(res_file), res_file + pst.set_res(res_file) + print(pst.phi) + assert np.isclose(pst.phi, 0), pst.phi + + pe.to_binary(os.path.join(pf.new_d, 'par.jcb')) + + # quick sweep test? + pst.pestpp_options["ies_par_en"] = 'par.jcb' + pst.pestpp_options["ies_num_reals"] = 10 + pst.control_data.noptmax = -1 + # par = pst.parameter_data + # par.loc[:, 'parval1'] = pe.iloc[0].T + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + # pyemu.os_utils.start_workers(pf.new_d, + # exe_rel_path="pestpp-ies", + # pst_rel_path="freyberg.pst", + # num_workers=20, master_dir="master", + # cleanup=False, port=4005) def generic_function(wd='.'): @@ -684,8 +672,12 @@ def another_generic_function(some_arg): print(some_arg) -def mf6_freyberg_test(setup_freyberg_mf6): - pf, sim = setup_freyberg_mf6 +def mf6_freyberg_test(tmp_path): + # import sys + # sys.path.insert(0, os.path.join("..", "..", "pypestutils")) + import pypestutils as ppu + + pf,sim = setup_freyberg_mf6(tmp_path) m = sim.get_model() mg = m.modelgrid template_ws = pf.new_d @@ -790,6 +782,7 @@ def mf6_freyberg_test(setup_freyberg_mf6): upper_bound=10, lower_bound=0.1, par_type="grid", + apply_order=0 ) with open(Path(template_ws, "inflow2.txt"), 'w') as fp: @@ -817,7 +810,7 @@ def mf6_freyberg_test(setup_freyberg_mf6): lower_bound=0.1, par_type="grid", use_rows=[[205, 'infl'], [206, 'infl']], - ) + apply_order=1) pf.add_parameters(filenames=['inflow2.txt', "inflow3.txt"], pargp='inflow2', comment_char='#', @@ -828,7 +821,8 @@ def mf6_freyberg_test(setup_freyberg_mf6): par_type="grid", use_rows=[[205, 'div'], [206, 'div']], par_style='a', - transform='none' + transform='none', + apply_order=0 ) with open(Path(template_ws, "inflow4.txt"), 'w') as fp: fp.write("# rid type rate idx0 idx1\n") @@ -848,7 +842,7 @@ def mf6_freyberg_test(setup_freyberg_mf6): lower_bound=0.1, par_type="grid", use_rows=[("204_1", "infl")], - ) + apply_order=2) pf.add_parameters(filenames="inflow4.txt", pargp='inflow5', comment_char='#', @@ -858,6 +852,7 @@ def mf6_freyberg_test(setup_freyberg_mf6): lower_bound=0.1, par_type="grid", use_rows=(1, 3), + apply_order=0 ) # pf.add_parameters(filenames=['inflow2.txt'], # pargp='inflow3', @@ -872,10 +867,10 @@ def mf6_freyberg_test(setup_freyberg_mf6): ft, ftd = _gen_dummy_obs_file(pf.new_d, sep=',', ext='txt') pf.add_parameters(filenames=f, par_type="grid", mfile_skip=1, index_cols=0, use_cols=[2], par_name_base="tmp", - pargp="tmp") + pargp="tmp",apply_order=10) pf.add_parameters(filenames=ft, par_type="grid", mfile_skip=1, index_cols=0, use_cols=[1, 2], par_name_base=["tmp2_1", "tmp2_2"], - pargp="tmp2", mfile_sep=',', par_style='direct') + pargp="tmp2", mfile_sep=',', par_style='direct',apply_order=0) tags = {"npf_k_":[0.1,10.],"npf_k33_":[.1,10],"sto_ss":[.1,10],"sto_sy":[.9,1.1],"rch_recharge":[.5,1.5]} dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]),unit="d") print(dts) @@ -885,12 +880,12 @@ def mf6_freyberg_test(setup_freyberg_mf6): if "rch" in tag: pf.add_parameters(filenames=arr_files, par_type="grid", par_name_base="rch_gr", pargp="rch_gr", zone_array=ib, upper_bound=ub, lower_bound=lb, - geostruct=gr_gs) + geostruct=gr_gs,apply_order=10) for arr_file in arr_files: kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 pf.add_parameters(filenames=arr_file,par_type="constant",par_name_base=arr_file.split('.')[1]+"_cn", pargp="rch_const",zone_array=ib,upper_bound=ub,lower_bound=lb,geostruct=rch_temporal_gs, - datetime=dts[kper]) + datetime=dts[kper],apply_order=10) else: for arr_file in arr_files: # these ult bounds are used later in an assert @@ -909,7 +904,8 @@ def mf6_freyberg_test(setup_freyberg_mf6): pf.add_parameters(filenames=arr_file, par_type="pilotpoints", par_name_base=arr_file.split('.')[1]+"_pp", pargp=arr_file.split('.')[1]+"_pp", zone_array=ib,upper_bound=ub,lower_bound=lb, ult_ubound=None if ult_ub is None else ult_ub - 1, - ult_lbound=None if ult_lb is None else ult_lb - 1,geostruct=gr_gs) + ult_lbound=None if ult_lb is None else ult_lb - 1,geostruct=gr_gs, + pp_options=dict(try_use_ppu=False)) # use a slightly lower ult bound here pf.add_parameters(filenames=arr_file, par_type="constant", @@ -1054,10 +1050,7 @@ def mf6_freyberg_test(setup_freyberg_mf6): pars = pst.parameter_data pst.parameter_data.loc[pars.index.str.contains('_pp'), 'parval1'] = dummymult check_apply(pf) - # os.chdir(pf.new_d) - # pst.write_input_files() - # pyemu.helpers.apply_list_and_array_pars() - # os.chdir(tmp_path) + # verify apply inflow2_df = pd.read_csv(Path(pf.new_d, "inflow2.txt"), header=None, sep=' ', skiprows=1) @@ -1077,13 +1070,18 @@ def mf6_freyberg_test(setup_freyberg_mf6): org = np.loadtxt(Path(pf.new_d, subinfo.org_file.values[0])) m = dummymult ** len(subinfo) check = org * m - check[ib == 0] = org[ib == 0] + #check[ib == 0] = org[ib == 0] ult_u = subinfo.upper_bound.astype(float).values[0] ult_l = subinfo.lower_bound.astype(float).values[0] check[check < ult_l] = ult_l check[check > ult_u] = ult_u + check[ib==0] = np.nan + result = np.loadtxt(Path(pf.new_d, mfile)) - assert np.isclose(check, result).all(), (f"Problem with par apply for " + result[ib==0] = np.nan + diff = np.abs(check - result) + diff[ib==0] = 0.0 + assert np.isclose(diff,0.0,rtol=0.0001,atol=0.0001).all(), (f"Problem with par apply for " f"{mfile}") df = pd.read_csv(Path(pf.new_d, "freyberg6.sfr_packagedata_test.txt"), sep=r'\s+', index_col=0) @@ -1168,7 +1166,7 @@ def mf6_freyberg_test(setup_freyberg_mf6): assert np.isclose(np.abs(float(df.lower_bound.max()) - -0.3), 0.), df.lower_bound.max() -def mf6_freyberg_shortnames_test(setup_freyberg_mf6): +def mf6_freyberg_shortnames_test(tmp_path): import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) @@ -1180,7 +1178,7 @@ def mf6_freyberg_shortnames_test(setup_freyberg_mf6): # (generated by pyemu.gw_utils.setup_hds_obs()) # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', # index_cols='obsnme', use_cols='obsval', prefix='hds') - pf, sim = setup_freyberg_mf6 + pf, sim = setup_freyberg_mf6(tmp_path) m = sim.get_model() tmp_model_ws = m.model_ws template_ws = pf.new_d @@ -1349,218 +1347,208 @@ def mf6_freyberg_da_test(tmp_path): org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6_da') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) - bd = Path.cwd() os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) - m = sim.get_model("freyberg6") + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + m = sim.get_model("freyberg6") - # SETUP pest stuff... - os_utils.run("{0} ".format("mf6"), cwd=tmp_model_ws) + # SETUP pest stuff... + os_utils.run("{0} ".format("mf6"), cwd=tmp_model_ws) - template_ws = "new_temp_da" - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - # sr0 = m.sr - sr = pyemu.helpers.SpatialReference.from_namfile( - os.path.join(tmp_model_ws, "freyberg6.nam"), - delr=m.dis.delr.array, delc=m.dis.delc.array) - # set up PstFrom object - pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, - remove_existing=True, - longnames=True, spatial_reference=sr, - zero_based=False,start_datetime="1-1-2018") - # obs - # using tabular style model output - # (generated by pyemu.gw_utils.setup_hds_obs()) - # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', - # index_cols='obsnme', use_cols='obsval', prefix='hds') - - df = pd.read_csv(os.path.join(tmp_model_ws,"heads.csv"),index_col=0) - pf.add_observations("heads.csv",insfile="heads.csv.ins",index_cols="time",use_cols=list(df.columns.values),prefix="hds") - df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) - pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", use_cols=list(df.columns.values)) - v = pyemu.geostats.ExpVario(contribution=1.0,a=1000) - gr_gs = pyemu.geostats.GeoStruct(variograms=v) - rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0,a=60)) - pf.extra_py_imports.append('flopy') - ib = m.dis.idomain[0].array - tags = {"npf_k_":[0.1,10.],"npf_k33_":[.1,10],"sto_ss":[.1,10],"sto_sy":[.9,1.1],"rch_recharge":[.5,1.5]} - dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]),unit="d") - print(dts) - for tag,bnd in tags.items(): - lb,ub = bnd[0],bnd[1] - arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] - if "rch" in tag: - pf.add_parameters(filenames=arr_files, par_type="grid", par_name_base="rch_gr", - pargp="rch_gr", zone_array=ib, upper_bound=ub, lower_bound=lb, + template_ws = "new_temp_da" + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + # sr0 = m.sr + sr = pyemu.helpers.SpatialReference.from_namfile( + os.path.join(tmp_model_ws, "freyberg6.nam"), + delr=m.dis.delr.array, delc=m.dis.delc.array) + # set up PstFrom object + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + remove_existing=True, + longnames=True, spatial_reference=sr, + zero_based=False,start_datetime="1-1-2018") + # obs + # using tabular style model output + # (generated by pyemu.gw_utils.setup_hds_obs()) + # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', + # index_cols='obsnme', use_cols='obsval', prefix='hds') + + df = pd.read_csv(os.path.join(tmp_model_ws,"heads.csv"),index_col=0) + pf.add_observations("heads.csv",insfile="heads.csv.ins",index_cols="time",use_cols=list(df.columns.values),prefix="hds") + df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) + pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", use_cols=list(df.columns.values)) + v = pyemu.geostats.ExpVario(contribution=1.0,a=1000) + gr_gs = pyemu.geostats.GeoStruct(variograms=v) + rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0,a=60)) + pf.extra_py_imports.append('flopy') + ib = m.dis.idomain[0].array + tags = {"npf_k_":[0.1,10.],"npf_k33_":[.1,10],"sto_ss":[.1,10],"sto_sy":[.9,1.1],"rch_recharge":[.5,1.5]} + dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]),unit="d") + print(dts) + for tag,bnd in tags.items(): + lb,ub = bnd[0],bnd[1] + arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] + if "rch" in tag: + pf.add_parameters(filenames=arr_files, par_type="grid", par_name_base="rch_gr", + pargp="rch_gr", zone_array=ib, upper_bound=ub, lower_bound=lb, + geostruct=gr_gs) + for arr_file in arr_files: + kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 + pf.add_parameters(filenames=arr_file,par_type="constant",par_name_base=arr_file.split('.')[1]+"_cn", + pargp="rch_const",zone_array=ib,upper_bound=ub,lower_bound=lb,geostruct=rch_temporal_gs, + datetime=dts[kper]) + else: + for arr_file in arr_files: + pf.add_parameters(filenames=arr_file,par_type="grid",par_name_base=arr_file.split('.')[1]+"_gr", + pargp=arr_file.split('.')[1]+"_gr",zone_array=ib,upper_bound=ub,lower_bound=lb, geostruct=gr_gs) - for arr_file in arr_files: - kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 - pf.add_parameters(filenames=arr_file,par_type="constant",par_name_base=arr_file.split('.')[1]+"_cn", - pargp="rch_const",zone_array=ib,upper_bound=ub,lower_bound=lb,geostruct=rch_temporal_gs, - datetime=dts[kper]) - else: - for arr_file in arr_files: - pf.add_parameters(filenames=arr_file,par_type="grid",par_name_base=arr_file.split('.')[1]+"_gr", - pargp=arr_file.split('.')[1]+"_gr",zone_array=ib,upper_bound=ub,lower_bound=lb, - geostruct=gr_gs) - pf.add_parameters(filenames=arr_file, par_type="pilotpoints", par_name_base=arr_file.split('.')[1]+"_pp", - pargp=arr_file.split('.')[1]+"_pp", zone_array=ib,upper_bound=ub,lower_bound=lb,) - - - list_files = [f for f in os.listdir(tmp_model_ws) if "wel_stress_period_data" in f] - for list_file in list_files: - kper = int(list_file.split(".")[1].split('_')[-1]) - 1 - # add spatially constant, but temporally correlated wel flux pars - pf.add_parameters(filenames=list_file,par_type="constant",par_name_base="twel_mlt_{0}".format(kper), - pargp="twel_mlt".format(kper),index_cols=[0,1,2],use_cols=[3], - upper_bound=1.5,lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs) - - # add temporally indep, but spatially correlated wel flux pars - pf.add_parameters(filenames=list_file, par_type="grid", par_name_base="wel_grid_{0}".format(kper), - pargp="wel_{0}".format(kper), index_cols=[0, 1, 2], use_cols=[3], - upper_bound=1.5, lower_bound=0.5, geostruct=gr_gs) - - pf.add_parameters(filenames="freyberg6.sfr_packagedata.txt",par_name_base="sfr_rhk", - pargp="sfr_rhk",index_cols={'k':1,'i':2,'j':3},use_cols=[9],upper_bound=10.,lower_bound=0.1, - par_type="grid") - - # add model run command - pf.mod_sys_cmds.append("mf6") - print(pf.mult_files) - print(pf.org_files) - - # build pest - pst = pf.build_pst('freyberg.pst', version=2) - pst.write(os.path.join(template_ws,"freyberg6_da.pst"),version=2) - - - # setup direct (non mult) pars on the IC files with par names that match the obs names - obs = pst.observation_data - hobs = obs.loc[obs.obsnme.str.startswith("hds"),:].copy() - hobs.loc[:,"k"] = hobs.obsnme.apply(lambda x: int(x.split(':')[1].split("_")[1])) - hobs.loc[:, "i"] = hobs.obsnme.apply(lambda x: int(x.split(':')[1].split("_")[2])) - hobs.loc[:, "j"] = hobs.obsnme.apply(lambda x: int(x.split(':')[1].split("_")[3])) - hobs_set = set(hobs.obsnme.to_list()) - ic_files = [f for f in os.listdir(template_ws) if "ic_strt" in f and f.endswith(".txt")] - print(ic_files) - ib = m.dis.idomain[0].array - tpl_files = [] - for ic_file in ic_files: - tpl_file = os.path.join(template_ws,ic_file+".tpl") - vals,names = [],[] - with open(tpl_file,'w') as f: - f.write("ptf ~\n") - k = int(ic_file.split('.')[1][-1]) - 1 - org_arr = np.loadtxt(os.path.join(template_ws,ic_file)) - for i in range(org_arr.shape[0]): - for j in range(org_arr.shape[1]): - if ib[i,j] < 1: - f.write(" -1.0e+30 ") - else: - pname = "hds_usecol:trgw_{0}_{1}_{2}_time:31.0".format(k,i,j) - if pname not in hobs_set and ib[i,j] > 0: - print(k,i,j,pname,ib[i,j]) - f.write(" ~ {0} ~".format(pname)) - vals.append(org_arr[i,j]) - names.append(pname) - f.write("\n") - df = pf.pst.add_parameters(tpl_file,pst_path=".") - pf.pst.parameter_data.loc[df.parnme,"partrans"] = "fixed" - pf.pst.parameter_data.loc[names,"parval1"] = vals - - pf.pst.write(os.path.join(template_ws,"freyberg6_da.pst"),version=2) - - num_reals = 100 - pe = pf.draw(num_reals, use_specsim=True) - pe.to_binary(os.path.join(template_ws, "prior.jcb")) - assert pe.shape[1] == pst.npar_adj, "{0} vs {1}".format(pe.shape[0], pst.npar_adj) - assert pe.shape[0] == num_reals - - # test par mults are working - os.chdir(pf.new_d) - pst.write_input_files() - pyemu.helpers.apply_list_and_array_pars( - arr_par_file="mult2model_info.csv") - os.chdir(tmp_path) + pf.add_parameters(filenames=arr_file, par_type="pilotpoints", par_name_base=arr_file.split('.')[1]+"_pp", + pargp=arr_file.split('.')[1]+"_pp", zone_array=ib,upper_bound=ub,lower_bound=lb,) - pst.control_data.noptmax = 0 - pst.pestpp_options["additional_ins_delimiters"] = "," - - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - - res_file = os.path.join(pf.new_d, "freyberg.base.rei") - assert os.path.exists(res_file), res_file - pst.set_res(res_file) - print(pst.phi) - assert np.isclose(pst.phi, 0), pst.phi - - # check mult files are in pst input files - csv = os.path.join(template_ws, "mult2model_info.csv") - df = pd.read_csv(csv, index_col=0) - pst_input_files = {str(f) for f in pst.input_files} - mults_not_linked_to_pst = ((set(df.mlt_file.unique()) - - pst_input_files) - - set(df.loc[df.pp_file.notna()].mlt_file)) - assert len(mults_not_linked_to_pst) == 0, print(mults_not_linked_to_pst) - except Exception as e: - os.chdir(bd) - raise e - os.chdir(bd) + list_files = [f for f in os.listdir(tmp_model_ws) if "wel_stress_period_data" in f] + for list_file in list_files: + kper = int(list_file.split(".")[1].split('_')[-1]) - 1 + # add spatially constant, but temporally correlated wel flux pars + pf.add_parameters(filenames=list_file,par_type="constant",par_name_base="twel_mlt_{0}".format(kper), + pargp="twel_mlt".format(kper),index_cols=[0,1,2],use_cols=[3], + upper_bound=1.5,lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs) -@pytest.fixture -def setup_freyberg_mf6(tmp_path, request): - try: - import flopy - except: - return - try: - model = request.param - except AttributeError: - model = "freyberg_mf6" - org_model_ws = os.path.join('..', 'examples', model) - tmp_model_ws = setup_tmp(org_model_ws, tmp_path) - # print(tmp_model_ws) - dup_file = "freyberg6.wel_stress_period_data_with_dups.txt" - shutil.copy2(os.path.join("utils", dup_file), tmp_model_ws) - bd = Path.cwd() - os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) - m = sim.get_model() - sim.set_all_data_external() - sim.write_simulation() + # add temporally indep, but spatially correlated wel flux pars + pf.add_parameters(filenames=list_file, par_type="grid", par_name_base="wel_grid_{0}".format(kper), + pargp="wel_{0}".format(kper), index_cols=[0, 1, 2], use_cols=[3], + upper_bound=1.5, lower_bound=0.5, geostruct=gr_gs) - # SETUP pest stuff... - os_utils.run("{0} ".format("mf6"), cwd=tmp_model_ws) + pf.add_parameters(filenames="freyberg6.sfr_packagedata.txt",par_name_base="sfr_rhk", + pargp="sfr_rhk",index_cols={'k':1,'i':2,'j':3},use_cols=[9],upper_bound=10.,lower_bound=0.1, + par_type="grid") - template_ws = Path(tmp_path, "template") - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - sr = m.modelgrid - # set up PstFrom object + # add model run command + pf.mod_sys_cmds.append("mf6") + print(pf.mult_files) + print(pf.org_files) + + # build pest + pst = pf.build_pst('freyberg.pst', version=2) + pst.write(os.path.join(template_ws,"freyberg6_da.pst"),version=2) + + + # setup direct (non mult) pars on the IC files with par names that match the obs names + obs = pst.observation_data + hobs = obs.loc[obs.obsnme.str.startswith("hds"),:].copy() + hobs.loc[:,"k"] = hobs.obsnme.apply(lambda x: int(x.split(':')[1].split("_")[1])) + hobs.loc[:, "i"] = hobs.obsnme.apply(lambda x: int(x.split(':')[1].split("_")[2])) + hobs.loc[:, "j"] = hobs.obsnme.apply(lambda x: int(x.split(':')[1].split("_")[3])) + hobs_set = set(hobs.obsnme.to_list()) + ic_files = [f for f in os.listdir(template_ws) if "ic_strt" in f and f.endswith(".txt")] + print(ic_files) + ib = m.dis.idomain[0].array + tpl_files = [] + for ic_file in ic_files: + tpl_file = os.path.join(template_ws,ic_file+".tpl") + vals,names = [],[] + with open(tpl_file,'w') as f: + f.write("ptf ~\n") + k = int(ic_file.split('.')[1][-1]) - 1 + org_arr = np.loadtxt(os.path.join(template_ws,ic_file)) + for i in range(org_arr.shape[0]): + for j in range(org_arr.shape[1]): + if ib[i,j] < 1: + f.write(" -1.0e+30 ") + else: + pname = "hds_usecol:trgw_{0}_{1}_{2}_time:31.0".format(k,i,j) + if pname not in hobs_set and ib[i,j] > 0: + print(k,i,j,pname,ib[i,j]) + f.write(" ~ {0} ~".format(pname)) + vals.append(org_arr[i,j]) + names.append(pname) + f.write("\n") + df = pf.pst.add_parameters(tpl_file,pst_path=".") + pf.pst.parameter_data.loc[df.parnme,"partrans"] = "fixed" + pf.pst.parameter_data.loc[names,"parval1"] = vals + + pf.pst.write(os.path.join(template_ws,"freyberg6_da.pst"),version=2) + + num_reals = 100 + pe = pf.draw(num_reals, use_specsim=True) + pe.to_binary(os.path.join(template_ws, "prior.jcb")) + assert pe.shape[1] == pst.npar_adj, "{0} vs {1}".format(pe.shape[0], pst.npar_adj) + assert pe.shape[0] == num_reals + + # test par mults are working + check_apply(pf) + + pst.control_data.noptmax = 0 + pst.pestpp_options["additional_ins_delimiters"] = "," + + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + + res_file = os.path.join(pf.new_d, "freyberg.base.rei") + assert os.path.exists(res_file), res_file + pst.set_res(res_file) + print(pst.phi) + assert np.isclose(pst.phi, 0), pst.phi + + # check mult files are in pst input files + csv = os.path.join(template_ws, "mult2model_info.csv") + df = pd.read_csv(csv, index_col=0) + pst_input_files = {str(f) for f in pst.input_files} + mults_not_linked_to_pst = ((set(df.mlt_file.unique()) - + pst_input_files) - + set(df.loc[df.pp_file.notna()].mlt_file)) + assert len(mults_not_linked_to_pst) == 0, print(mults_not_linked_to_pst) + + +def setup_freyberg_mf6(wd, model="freyberg_mf6", **kwargs): + try: + import flopy + except: + return + # try: + # model = request.param + # except AttributeError: + # model = "freyberg_mf6" + org_model_ws = os.path.join('..', 'examples', model) + tmp_model_ws = setup_tmp(org_model_ws, wd) + # print(tmp_model_ws) + dup_file = "freyberg6.wel_stress_period_data_with_dups.txt" + shutil.copy2(os.path.join("utils", dup_file), tmp_model_ws) + bd = Path.cwd() + os.chdir(wd) + try: + tmp_model_ws = tmp_model_ws.relative_to(wd) + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + m = sim.get_model() + sim.set_all_data_external() + sim.write_simulation() + + # SETUP pest stuff... + os_utils.run("{0} ".format("mf6"), cwd=tmp_model_ws) + + template_ws = Path(wd, "template") + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + sr = m.modelgrid + # set up PstFrom object pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, remove_existing=True, longnames=True, spatial_reference=sr, zero_based=False, - start_datetime="1-1-2018") + start_datetime="1-1-2018", + **kwargs) if model == 'freyberg_mf6': pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", use_cols=["GAGE_1", "HEADWATER", "TAILWATER"], ofile_sep=",") - yield pf, sim + return pf, sim except Exception as e: os.chdir(bd) raise e - os.chdir(bd) + # os.chdir(bd) def build_direct(pf): @@ -1580,22 +1568,22 @@ def build_direct(pf): return pf, pst -def check_apply(pf): +def check_apply(pf, chunklen=50): # test par mults are working bd = Path.cwd() os.chdir(pf.new_d) try: pf.pst.write_input_files() pyemu.helpers.apply_list_and_array_pars( - arr_par_file="mult2model_info.csv", chunk_len=100) + arr_par_file="mult2model_info.csv", chunk_len=chunklen) except Exception as e: os.chdir(bd) raise e os.chdir(bd) -def direct_quickfull_test(setup_freyberg_mf6): - pf, sim = setup_freyberg_mf6 +def direct_quickfull_test(tmp_path): + pf, sim = setup_freyberg_mf6(tmp_path) m = sim.get_model() mg = m.modelgrid # Setup geostruct for spatial pars @@ -1690,9 +1678,9 @@ def direct_quickfull_test(setup_freyberg_mf6): ) -def direct_multadd_combo_test(setup_freyberg_mf6): +def direct_multadd_combo_test(tmp_path): dup_file = "freyberg6.wel_stress_period_data_with_dups.txt" - pf, _ = setup_freyberg_mf6 + pf, _ = setup_freyberg_mf6(tmp_path) pf.longnames = False ghb_files = [f for f in os.listdir(pf.new_d) if ".ghb_stress" in f and f.endswith("txt")] pf.add_parameters(ghb_files, par_type="grid", par_style="add", use_cols=3, par_name_base="ghbstage", @@ -1783,8 +1771,8 @@ def direct_multadd_combo_test(setup_freyberg_mf6): assert d.sum() == 0.0, d.sum() -def direct_arraypars_test(setup_freyberg_mf6): - pf, sim = setup_freyberg_mf6 +def direct_arraypars_test(tmp_path): + pf, sim = setup_freyberg_mf6(tmp_path) m = sim.get_model() mg = m.modelgrid # Setup geostruct for spatial pars @@ -1850,9 +1838,9 @@ def direct_arraypars_test(setup_freyberg_mf6): raise Exception("recharge too diff") -def direct_listpars_test(setup_freyberg_mf6): +def direct_listpars_test(tmp_path): import flopy - pf, sim = setup_freyberg_mf6 + pf, sim = setup_freyberg_mf6(tmp_path) m = sim.get_model() template_ws = pf.new_d # Setup geostruct for spatial pars @@ -2013,497 +2001,487 @@ def mf6_freyberg_direct_test(tmp_path): tmp_model_ws = setup_tmp(org_model_ws, tmp_path) dup_file = "freyberg6.wel_stress_period_data_with_dups.txt" shutil.copy2(os.path.join("utils", dup_file), tmp_model_ws) - bd = Path.cwd() - os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) - m = sim.get_model("freyberg6") - sim.set_all_data_external() - sim.write_simulation() - - # SETUP pest stuff... - os_utils.run("{0} ".format("mf6"), cwd=tmp_model_ws) - - template_ws = Path(tmp_path, "new_temp_direct") - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - sr = m.modelgrid - # set up PstFrom object - pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, - remove_existing=True, - longnames=True, spatial_reference=sr, - zero_based=False, start_datetime="1-1-2018") - # obs - # using tabular style model output - # (generated by pyemu.gw_utils.setup_hds_obs()) - # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', - # index_cols='obsnme', use_cols='obsval', prefix='hds') - - - ghb_files = [f for f in os.listdir(template_ws) if ".ghb_stress" in f and f.endswith("txt")] - pf.add_parameters(ghb_files,par_type="grid",par_style="add",use_cols=3,par_name_base="ghbstage", - pargp="ghbstage",index_cols=[0,1,2],transform="none",lower_bound=-5,upper_bound=5) - - pf.add_parameters(ghb_files, par_type="grid", par_style="multiplier", use_cols=3, par_name_base="mltstage", - pargp="ghbstage", index_cols=[0, 1, 2], transform="log", lower_bound=0.5, - upper_bound=1.5) - - # Add stream flow observation - # df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) - pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", - use_cols=["GAGE_1","HEADWATER","TAILWATER"],ofile_sep=",") - # Setup geostruct for spatial pars - gr_v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) - gr_gs = pyemu.geostats.GeoStruct(variograms=gr_v, transform="log") - pp_v = pyemu.geostats.ExpVario(contribution=1.0, a=5000) - pp_gs = pyemu.geostats.GeoStruct(variograms=pp_v, transform="log") - rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60)) - pf.extra_py_imports.append('flopy') - ib = m.dis.idomain[0].array - tags = {"npf_k_": [0.1, 10.], "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1], - "rch_recharge": [.5, 1.5]} - dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") - print(dts) - #ib = m.dis.idomain.array[0,:,:] - # setup from array style pars - for tag, bnd in tags.items(): - lb, ub = bnd[0], bnd[1] - arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] - if "rch" in tag: - for arr_file in arr_files: - # indy direct grid pars for each array type file - recharge_files = ["recharge_1.txt","recharge_2.txt","recharge_3.txt"] - pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base="rch_gr", - pargp="rch_gr", zone_array=ib, upper_bound=1.0e-3, lower_bound=1.0e-7, - par_style="direct") - # additional constant mults - kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 - pf.add_parameters(filenames=arr_file, par_type="constant", - par_name_base=arr_file.split('.')[1] + "_cn", - pargp="rch_const", zone_array=ib, upper_bound=ub, lower_bound=lb, - geostruct=rch_temporal_gs, - datetime=dts[kper]) - else: - for arr_file in arr_files: - # grid mults pure and simple - pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base=arr_file.split('.')[1] + "_gr", - pargp=arr_file.split('.')[1] + "_gr", zone_array=ib, upper_bound=ub, - lower_bound=lb, - geostruct=gr_gs) - - # Add a variety of list style pars - list_files = ["freyberg6.wel_stress_period_data_{0}.txt".format(t) - for t in range(1, m.nper + 1)] - # make dummy versions with headers - for fl in list_files[0:2]: # this adds a header to well file - with open(os.path.join(template_ws, fl), 'r') as fr: - lines = [line for line in fr] - with open(os.path.join(template_ws, f"new_{fl}"), 'w') as fw: - fw.write("k i j flux \n") - for line in lines: - fw.write(line) - - # fl = "freyberg6.wel_stress_period_data_3.txt" # Add extra string col_id - for fl in list_files[2:7]: - with open(os.path.join(template_ws, fl), 'r') as fr: - lines = [line for line in fr] - with open(os.path.join(template_ws, f"new_{fl}"), 'w') as fw: - fw.write("well k i j flux \n") - for i, line in enumerate(lines): - fw.write(f"well{i}" + line) - - list_files.sort() - for list_file in list_files: - kper = int(list_file.split(".")[1].split('_')[-1]) - 1 - #add spatially constant, but temporally correlated wel flux pars - pf.add_parameters(filenames=list_file, par_type="constant", par_name_base="twel_mlt_{0}".format(kper), - pargp="twel_mlt_{0}".format(kper), index_cols=[0, 1, 2], use_cols=[3], - upper_bound=1.5, lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs) - - # add temporally indep, but spatially correlated wel flux pars - pf.add_parameters(filenames=list_file, par_type="grid", par_name_base="wel_grid_{0}".format(kper), - pargp="wel_{0}".format(kper), index_cols=[0, 1, 2], use_cols=[3], - upper_bound=0.0, lower_bound=-1000, geostruct=gr_gs, par_style="direct", - transform="none") - # Adding dummy list pars with different file structures - list_file = "new_freyberg6.wel_stress_period_data_1.txt" # with a header - pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nwell_mlt', - pargp='nwell_mult', index_cols=['k', 'i', 'j'], use_cols='flux', - upper_bound=10, lower_bound=-10, geostruct=gr_gs, - transform="none") - pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nwell_grid', - pargp='nwell', index_cols=['k', 'i', 'j'], use_cols='flux', - upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", - transform="none") - # with skip instead - list_file = "new_freyberg6.wel_stress_period_data_2.txt" - pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nwell_grid', - pargp='nwell', index_cols=[0, 1, 2], use_cols=3, - upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", - transform="none", mfile_skip=1) - - list_file = "new_freyberg6.wel_stress_period_data_3.txt" - pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nwell_mlt', - pargp='nwell_mult', index_cols=['well', 'k', 'i', 'j'], use_cols='flux', - upper_bound=10, lower_bound=-10, geostruct=gr_gs, - transform="none") - pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nwell_grid', - pargp='nwell', index_cols=['well', 'k', 'i', 'j'], use_cols='flux', - upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", - transform="none") - # with skip instead - list_file = "new_freyberg6.wel_stress_period_data_4.txt" - pf.add_parameters(filenames=list_file, par_type="grid", - par_name_base='nwell_grid', pargp='nwell', - index_cols=[0, 1, 2, 3], # or... {'well': 0, 'k': 1, 'i': 2, 'j': 3}, - use_cols=4, upper_bound=10, lower_bound=-10, - geostruct=gr_gs, par_style="direct", transform="none", - mfile_skip=1) - - list_file = "freyberg6.ghb_stress_period_data_1.txt" - pf.add_parameters(filenames=list_file, par_type="constant", par_name_base=["ghb_stage","ghb_cond"], - pargp=["ghb_stage","ghb_cond"], index_cols=[0, 1, 2], use_cols=[3,4], - upper_bound=[35,150], lower_bound=[32,50], par_style="direct", - transform="none") - pf.add_parameters(filenames=dup_file, par_type="grid", par_name_base="dups", - pargp="dups", index_cols=[0, 1, 2], use_cols=[3], - upper_bound=0.0, lower_bound=-500,par_style="direct", - transform="none") - - list_file = "new_freyberg6.wel_stress_period_data_5.txt" - pf.add_parameters(filenames=list_file, par_type="grid", - par_name_base=['nwell5_k', 'nwell5_q'], - pargp='nwell5', - index_cols=['well', 'i', 'j'], - use_cols=['k', 'flux'], upper_bound=10, lower_bound=-10, - geostruct=gr_gs, par_style="direct", transform="none", - mfile_skip=0, use_rows=[3, 4]) - - list_file = "new_freyberg6.wel_stress_period_data_6.txt" - pf.add_parameters(filenames=list_file, par_type="grid", - par_name_base=['nwell6_k', 'nwell6_q'], - pargp='nwell6', - index_cols=['well', 'i', 'j'], - use_cols=['k', 'flux'], upper_bound=10, lower_bound=-10, - geostruct=gr_gs, par_style="direct", transform="none", - mfile_skip=0, use_rows=[(3, 21, 15), (3, 30, 7)]) - # use_rows should match so all should be setup 2 cols 6 rows - assert len(pf.par_dfs[-1]) == 2*6 # should be - list_file = "new_freyberg6.wel_stress_period_data_7.txt" - pf.add_parameters(filenames=list_file, par_type="grid", - par_name_base=['nwell6_k', 'nwell6_q'], - pargp='nwell6', - index_cols=['well', 'i', 'j'], - use_cols=['k', 'flux'], upper_bound=10, lower_bound=-10, - geostruct=gr_gs, par_style="direct", transform="none", - mfile_skip=0, - use_rows=[('well2', 21, 15), ('well4', 30, 7)]) - assert len(pf.par_dfs[-1]) == 2 * 2 # should be - # add model run command - pf.mod_sys_cmds.append("mf6") - print(pf.mult_files) - print(pf.org_files) - - # build pest - pst = pf.build_pst('freyberg.pst') - cov = pf.build_prior(fmt="non") - cov.to_coo(os.path.join(template_ws, "prior.jcb")) - pst.try_parse_name_metadata() - df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0) - pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", - use_cols=list(df.columns.values), - prefix="hds", rebuild_pst=True) - - # test par mults are working - - os.chdir(pf.new_d) - pst.write_input_files() - pyemu.helpers.apply_list_and_array_pars( - arr_par_file="mult2model_info.csv", chunk_len=1) - # TODO Some checks on resultant par files... - list_files = [f for f in os.listdir('.') - if f.startswith('new_') and f.endswith('txt')] - # check on that those dummy pars compare to the model versions. - for f in list_files: - n_df = pd.read_csv(f, sep=r"\s+") - o_df = pd.read_csv(f.strip('new_'), sep=r"\s+", header=None) - o_df.columns = ['k', 'i', 'j', 'flux'] - assert np.isclose(n_df.loc[:, o_df.columns], o_df).all(), ( - "Something broke with alternative style model files" - ) - os.chdir(tmp_path) - - num_reals = 100 - pe = pf.draw(num_reals, use_specsim=True) - pe.to_binary(os.path.join(template_ws, "prior.jcb")) - assert pe.shape[1] == pst.npar_adj, "{0} vs {1}".format(pe.shape[0], pst.npar_adj) - assert pe.shape[0] == num_reals - - pst.control_data.noptmax = 0 - pst.pestpp_options["additional_ins_delimiters"] = "," - - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - - res_file = os.path.join(pf.new_d, "freyberg.base.rei") - assert os.path.exists(res_file), res_file - pst.set_res(res_file) - print(pst.phi) - assert pst.phi < 0.1, pst.phi - - org_ghb = pd.read_csv(os.path.join(pf.new_d,"org","freyberg6.ghb_stress_period_data_1.txt"), - header=None,names=["l","r","c","stage","cond"]) - new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), - sep=r"\s+", - header=None, names=["l", "r", "c", "stage", "cond"]) - d = org_ghb.stage - new_ghb.stage - print(d) - assert d.sum() == 0, d.sum() - - - # test the additive ghb stage pars - par = pst.parameter_data - par.loc[par.parnme.str.contains("ghbstage_inst:0"),"parval1"] = 3.0 - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - org_ghb = pd.read_csv(os.path.join(pf.new_d, "org", "freyberg6.ghb_stress_period_data_1.txt"), - header=None, names=["l", "r", "c", "stage", "cond"]) - new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), - sep=r"\s+", - header=None, names=["l", "r", "c", "stage", "cond"]) - d = (org_ghb.stage - new_ghb.stage).apply(np.abs) - print(d) - assert d.mean() == 3.0, d.mean() - - # check that the interaction between the direct ghb stage par and the additive ghb stage pars - # is working - par.loc[par.parnme.str.contains("ghb_stage"),"parval1"] -= 3.0 - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - org_ghb = pd.read_csv(os.path.join(tmp_model_ws,"freyberg6.ghb_stress_period_data_1.txt"), - header=None, names=["l", "r", "c", "stage", "cond"],sep=r"\s+") - new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), - sep=r"\s+", - header=None, names=["l", "r", "c", "stage", "cond"]) - d = org_ghb.stage - new_ghb.stage - print(new_ghb.stage) - print(org_ghb.stage) - print(d) - assert d.sum() == 0.0, d.sum() - - # check the interaction with multiplicative ghb stage, direct ghb stage and additive ghb stage - par.loc[par.parnme.str.contains("mltstage"), "parval1"] = 1.1 - #par.loc[par.parnme.str.contains("ghbstage_inst:0"), "parval1"] = 0.0 - #par.loc[par.parnme.str.contains("ghb_stage"), "parval1"] += 3.0 - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - org_ghb = pd.read_csv(os.path.join(tmp_model_ws, "freyberg6.ghb_stress_period_data_1.txt"), - header=None, names=["l", "r", "c", "stage", "cond"], sep=r"\s+") - new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), - sep=r"\s+", - header=None, names=["l", "r", "c", "stage", "cond"]) - d = (org_ghb.stage * 1.1) - new_ghb.stage - print(new_ghb.stage) - print(org_ghb.stage) - print(d) - assert d.sum() == 0.0, d.sum() - - # turn direct recharge to min and direct wel to min and - # check that the model results are consistent - par = pst.parameter_data - rch_par = par.loc[par.parnme.apply( - lambda x: "pname:rch_gr" in x and "ptype:gr_pstyle:d" in x ), "parnme"] - wel_par = par.loc[par.parnme.apply( - lambda x: "pname:wel_grid" in x and "ptype:gr_usecol:3_pstyle:d" in x), "parnme"] - par.loc[rch_par,"parval1"] = par.loc[rch_par, "parlbnd"] - # this should set wells to zero since they are negative values in the control file - par.loc[wel_par,"parval1"] = par.loc[wel_par, "parubnd"] - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - lst = flopy.utils.Mf6ListBudget(os.path.join(pf.new_d, "freyberg6.lst")) - flx, cum = lst.get_dataframes(diff=True) - wel_tot = flx.wel.apply(np.abs).sum() - print(flx.wel) - assert wel_tot < 1.0e-6, wel_tot - - rch_files = [f for f in os.listdir(pf.new_d) - if ".rch_recharge" in f and f.endswith(".txt")] - rch_val = par.loc[rch_par,"parval1"][0] - i, j = par.loc[rch_par, ["i", 'j']].astype(int).values.T - for rch_file in rch_files: - arr = np.loadtxt(os.path.join(pf.new_d, rch_file))[i, j] - print(rch_file, rch_val, arr.mean(), arr.max(), arr.min()) - if np.abs(arr.max() - rch_val) > 1.0e-6 or np.abs(arr.min() - rch_val) > 1.0e-6: - raise Exception("recharge too diff") - except Exception as e: - os.chdir(bd) - raise e - os.chdir(bd) - - -def mf6_freyberg_varying_idomain(tmp_path): - import numpy as np - import pandas as pd - pd.set_option('display.max_rows', 500) - pd.set_option('display.max_columns', 500) - pd.set_option('display.width', 1000) - try: - import flopy - except: - return - org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') - tmp_model_ws = setup_tmp(org_model_ws, tmp_path) - bd = Path.cwd() os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) - # sim.set_all_data_external() - sim.set_sim_path(str(tmp_model_ws)) - # sim.set_all_data_external() - m = sim.get_model("freyberg6") - sim.set_all_data_external(check_data=False) - sim.write_simulation() - - #sim = None - ib_file = os.path.join(tmp_model_ws,"freyberg6.dis_idomain_layer1.txt") - arr = np.loadtxt(ib_file,dtype=np.int64) - - arr[:2,:14] = 0 - np.savetxt(ib_file,arr,fmt="%2d") - print(arr) - - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) - m = sim.get_model("freyberg6") + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + m = sim.get_model("freyberg6") + sim.set_all_data_external() + sim.write_simulation() - # SETUP pest stuff... - os_utils.run("{0} ".format(mf6_exe_path), cwd=str(tmp_model_ws)) + # SETUP pest stuff... + os_utils.run("{0} ".format("mf6"), cwd=tmp_model_ws) + template_ws = Path(tmp_path, "new_temp_direct") + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + sr = m.modelgrid + # set up PstFrom object + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + remove_existing=True, + longnames=True, spatial_reference=sr, + zero_based=False, start_datetime="1-1-2018") + # obs + # using tabular style model output + # (generated by pyemu.gw_utils.setup_hds_obs()) + # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', + # index_cols='obsnme', use_cols='obsval', prefix='hds') - template_ws = "new_temp" - if os.path.exists(template_ws): - shutil.rmtree(template_ws) + ghb_files = [f for f in os.listdir(template_ws) if ".ghb_stress" in f and f.endswith("txt")] + pf.add_parameters(ghb_files,par_type="grid",par_style="add",use_cols=3,par_name_base="ghbstage", + pargp="ghbstage",index_cols=[0,1,2],transform="none",lower_bound=-5,upper_bound=5) - # if os.path.exists(template_ws): - # shutil.rmtree(template_ws) - # shutil.copytree(tmp_model_ws,template_ws) - # hk_file = os.path.join(template_ws, "freyberg6.npf_k_layer1.txt") - # hk = np.loadtxt(hk_file) - # - # hk[arr == 0] = 1.0e+30 - # np.savetxt(hk_file,hk,fmt="%50.45f") - # os_utils.run("{0} ".format(mf6_exe_path), cwd=template_ws) - # import matplotlib.pyplot as plt - # hds1 = flopy.utils.HeadFile(os.path.join(tmp_model_ws, "freyberg6_freyberg.hds")) - # hds2 = flopy.utils.HeadFile(os.path.join(template_ws, "freyberg6_freyberg.hds")) - # - # d = hds1.get_data() - hds2.get_data() - # for dd in d: - # cb = plt.imshow(dd) - # plt.colorbar(cb) - # plt.show() - # return - - # sr0 = m.sr - # sr = pyemu.helpers.SpatialReference.from_namfile( - # os.path.join(tmp_model_ws, "freyberg6.nam"), - # delr=m.dis.delr.array, delc=m.dis.delc.array) + pf.add_parameters(ghb_files, par_type="grid", par_style="multiplier", use_cols=3, par_name_base="mltstage", + pargp="ghbstage", index_cols=[0, 1, 2], transform="log", lower_bound=0.5, + upper_bound=1.5) - sr = m.modelgrid - # set up PstFrom object - pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, - remove_existing=True, - longnames=True, spatial_reference=sr, - zero_based=False, start_datetime="1-1-2018") + # Add stream flow observation + # df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) + pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", + use_cols=["GAGE_1","HEADWATER","TAILWATER"],ofile_sep=",") + # Setup geostruct for spatial pars + gr_v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) + gr_gs = pyemu.geostats.GeoStruct(variograms=gr_v, transform="log") + pp_v = pyemu.geostats.ExpVario(contribution=1.0, a=5000) + pp_gs = pyemu.geostats.GeoStruct(variograms=pp_v, transform="log") + rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60)) + pf.extra_py_imports.append('flopy') + ib = m.dis.idomain[0].array + tags = {"npf_k_": [0.1, 10.], "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1], + "rch_recharge": [.5, 1.5]} + dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") + print(dts) + #ib = m.dis.idomain.array[0,:,:] + # setup from array style pars + for tag, bnd in tags.items(): + lb, ub = bnd[0], bnd[1] + arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] + if "rch" in tag: + for arr_file in arr_files: + # indy direct grid pars for each array type file + recharge_files = ["recharge_1.txt","recharge_2.txt","recharge_3.txt"] + pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base="rch_gr", + pargp="rch_gr", zone_array=ib, upper_bound=1.0e-3, lower_bound=1.0e-7, + par_style="direct") + # additional constant mults + kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 + pf.add_parameters(filenames=arr_file, par_type="constant", + par_name_base=arr_file.split('.')[1] + "_cn", + pargp="rch_const", zone_array=ib, upper_bound=ub, lower_bound=lb, + geostruct=rch_temporal_gs, + datetime=dts[kper]) + else: + for arr_file in arr_files: + # grid mults pure and simple + pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base=arr_file.split('.')[1] + "_gr", + pargp=arr_file.split('.')[1] + "_gr", zone_array=ib, upper_bound=ub, + lower_bound=lb, + geostruct=gr_gs) + # Add a variety of list style pars + list_files = ["freyberg6.wel_stress_period_data_{0}.txt".format(t) + for t in range(1, m.nper + 1)] + # make dummy versions with headers + for fl in list_files[0:2]: # this adds a header to well file + with open(os.path.join(template_ws, fl), 'r') as fr: + lines = [line for line in fr] + with open(os.path.join(template_ws, f"new_{fl}"), 'w') as fw: + fw.write("k i j flux \n") + for line in lines: + fw.write(line) - # pf.post_py_cmds.append("generic_function()") - df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) - pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", use_cols=list(df.columns.values), - ofile_sep=",") - v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) - gr_gs = pyemu.geostats.GeoStruct(variograms=v) - rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60)) - pf.extra_py_imports.append('flopy') - - ib = {} - for k in range(m.dis.nlay.data): - a = m.dis.idomain.array[k,:,:].copy() - print(a) - ib[k] = a - - tags = {"npf_k_": [0.1, 10.,0.003,35]}#, "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1]} - dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") - print(dts) - for tag, bnd in tags.items(): - lb, ub = bnd[0], bnd[1] - ult_lb = bnd[2] - ult_ub = bnd[3] - arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] + # fl = "freyberg6.wel_stress_period_data_3.txt" # Add extra string col_id + for fl in list_files[2:7]: + with open(os.path.join(template_ws, fl), 'r') as fr: + lines = [line for line in fr] + with open(os.path.join(template_ws, f"new_{fl}"), 'w') as fw: + fw.write("well k i j flux \n") + for i, line in enumerate(lines): + fw.write(f"well{i}" + line) - for arr_file in arr_files: + list_files.sort() + for list_file in list_files: + kper = int(list_file.split(".")[1].split('_')[-1]) - 1 + #add spatially constant, but temporally correlated wel flux pars + pf.add_parameters(filenames=list_file, par_type="constant", par_name_base="twel_mlt_{0}".format(kper), + pargp="twel_mlt_{0}".format(kper), index_cols=[0, 1, 2], use_cols=[3], + upper_bound=1.5, lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs) - # these ult bounds are used later in an assert + # add temporally indep, but spatially correlated wel flux pars + pf.add_parameters(filenames=list_file, par_type="grid", par_name_base="wel_grid_{0}".format(kper), + pargp="wel_{0}".format(kper), index_cols=[0, 1, 2], use_cols=[3], + upper_bound=0.0, lower_bound=-1000, geostruct=gr_gs, par_style="direct", + transform="none") + # Adding dummy list pars with different file structures + list_file = "new_freyberg6.wel_stress_period_data_1.txt" # with a header + pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nwell_mlt', + pargp='nwell_mult', index_cols=['k', 'i', 'j'], use_cols='flux', + upper_bound=10, lower_bound=-10, geostruct=gr_gs, + transform="none") + pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nwell_grid', + pargp='nwell', index_cols=['k', 'i', 'j'], use_cols='flux', + upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", + transform="none") + # with skip instead + list_file = "new_freyberg6.wel_stress_period_data_2.txt" + pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nwell_grid', + pargp='nwell', index_cols=[0, 1, 2], use_cols=3, + upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", + transform="none", mfile_skip=1) - k = int(arr_file.split(".")[-2].split("layer")[1].split("_")[0]) - 1 - pf.add_parameters(filenames=arr_file, par_type="pilotpoints", par_name_base=arr_file.split('.')[1] + "_pp", - pargp=arr_file.split('.')[1] + "_pp", upper_bound=ub, lower_bound=lb, - geostruct=gr_gs, zone_array=ib[k],ult_lbound=ult_lb,ult_ubound=ult_ub) + list_file = "new_freyberg6.wel_stress_period_data_3.txt" + pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nwell_mlt', + pargp='nwell_mult', index_cols=['well', 'k', 'i', 'j'], use_cols='flux', + upper_bound=10, lower_bound=-10, geostruct=gr_gs, + transform="none") + pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nwell_grid', + pargp='nwell', index_cols=['well', 'k', 'i', 'j'], use_cols='flux', + upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", + transform="none") + # with skip instead + list_file = "new_freyberg6.wel_stress_period_data_4.txt" + pf.add_parameters(filenames=list_file, par_type="grid", + par_name_base='nwell_grid', pargp='nwell', + index_cols=[0, 1, 2, 3], # or... {'well': 0, 'k': 1, 'i': 2, 'j': 3}, + use_cols=4, upper_bound=10, lower_bound=-10, + geostruct=gr_gs, par_style="direct", transform="none", + mfile_skip=1) - # add model run command - pf.mod_sys_cmds.append("mf6") - df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0) - df = pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", use_cols=list(df.columns.values), - prefix="hds", ofile_sep=",") + list_file = "freyberg6.ghb_stress_period_data_1.txt" + pf.add_parameters(filenames=list_file, par_type="constant", par_name_base=["ghb_stage","ghb_cond"], + pargp=["ghb_stage","ghb_cond"], index_cols=[0, 1, 2], use_cols=[3,4], + upper_bound=[35,150], lower_bound=[32,50], par_style="direct", + transform="none") + pf.add_parameters(filenames=dup_file, par_type="grid", par_name_base="dups", + pargp="dups", index_cols=[0, 1, 2], use_cols=[3], + upper_bound=0.0, lower_bound=-500,par_style="direct", + transform="none") + list_file = "new_freyberg6.wel_stress_period_data_5.txt" + pf.add_parameters(filenames=list_file, par_type="grid", + par_name_base=['nwell5_k', 'nwell5_q'], + pargp='nwell5', + index_cols=['well', 'i', 'j'], + use_cols=['k', 'flux'], upper_bound=10, lower_bound=-10, + geostruct=gr_gs, par_style="direct", transform="none", + mfile_skip=0, use_rows=[3, 4]) + + list_file = "new_freyberg6.wel_stress_period_data_6.txt" + pf.add_parameters(filenames=list_file, par_type="grid", + par_name_base=['nwell6_k', 'nwell6_q'], + pargp='nwell6', + index_cols=['well', 'i', 'j'], + use_cols=['k', 'flux'], upper_bound=10, lower_bound=-10, + geostruct=gr_gs, par_style="direct", transform="none", + mfile_skip=0, use_rows=[(3, 21, 15), (3, 30, 7)]) + # use_rows should match so all should be setup 2 cols 6 rows + assert len(pf.par_dfs[-1]) == 2*6 # should be + list_file = "new_freyberg6.wel_stress_period_data_7.txt" + pf.add_parameters(filenames=list_file, par_type="grid", + par_name_base=['nwell6_k', 'nwell6_q'], + pargp='nwell6', + index_cols=['well', 'i', 'j'], + use_cols=['k', 'flux'], upper_bound=10, lower_bound=-10, + geostruct=gr_gs, par_style="direct", transform="none", + mfile_skip=0, + use_rows=[('well2', 21, 15), ('well4', 30, 7)]) + assert len(pf.par_dfs[-1]) == 2 * 2 # should be + # add model run command + pf.mod_sys_cmds.append("mf6") + print(pf.mult_files) + print(pf.org_files) + + # build pest + pst = pf.build_pst('freyberg.pst') + cov = pf.build_prior(fmt="non") + cov.to_coo(os.path.join(template_ws, "prior.jcb")) + pst.try_parse_name_metadata() + df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0) + pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", + use_cols=list(df.columns.values), + prefix="hds", rebuild_pst=True) + + # test par mults are working + + os.chdir(pf.new_d) + pst.write_input_files() + pyemu.helpers.apply_list_and_array_pars( + arr_par_file="mult2model_info.csv", chunk_len=10) + # TODO Some checks on resultant par files... + list_files = [f for f in os.listdir('.') + if f.startswith('new_') and f.endswith('txt')] + # check on that those dummy pars compare to the model versions. + for f in list_files: + n_df = pd.read_csv(f, sep=r"\s+") + o_df = pd.read_csv(f.strip('new_'), sep=r"\s+", header=None) + o_df.columns = ['k', 'i', 'j', 'flux'] + assert np.isclose(n_df.loc[:, o_df.columns], o_df).all(), ( + "Something broke with alternative style model files" + ) + os.chdir(tmp_path) + + num_reals = 100 + pe = pf.draw(num_reals, use_specsim=True) + pe.to_binary(os.path.join(template_ws, "prior.jcb")) + assert pe.shape[1] == pst.npar_adj, "{0} vs {1}".format(pe.shape[0], pst.npar_adj) + assert pe.shape[0] == num_reals + + pst.control_data.noptmax = 0 + pst.pestpp_options["additional_ins_delimiters"] = "," + + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + + res_file = os.path.join(pf.new_d, "freyberg.base.rei") + assert os.path.exists(res_file), res_file + pst.set_res(res_file) + print(pst.phi) + assert pst.phi < 0.1, pst.phi + + org_ghb = pd.read_csv(os.path.join(pf.new_d,"org","freyberg6.ghb_stress_period_data_1.txt"), + header=None,names=["l","r","c","stage","cond"]) + new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), + sep=r"\s+", + header=None, names=["l", "r", "c", "stage", "cond"]) + d = org_ghb.stage - new_ghb.stage + print(d) + assert d.sum() == 0, d.sum() + + + # test the additive ghb stage pars + par = pst.parameter_data + par.loc[par.parnme.str.contains("ghbstage_inst:0"),"parval1"] = 3.0 + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + org_ghb = pd.read_csv(os.path.join(pf.new_d, "org", "freyberg6.ghb_stress_period_data_1.txt"), + header=None, names=["l", "r", "c", "stage", "cond"]) + new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), + sep=r"\s+", + header=None, names=["l", "r", "c", "stage", "cond"]) + d = (org_ghb.stage - new_ghb.stage).apply(np.abs) + print(d) + assert d.mean() == 3.0, d.mean() + + # check that the interaction between the direct ghb stage par and the additive ghb stage pars + # is working + par.loc[par.parnme.str.contains("ghb_stage"),"parval1"] -= 3.0 + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + org_ghb = pd.read_csv(os.path.join(tmp_model_ws,"freyberg6.ghb_stress_period_data_1.txt"), + header=None, names=["l", "r", "c", "stage", "cond"],sep=r"\s+") + new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), + sep=r"\s+", + header=None, names=["l", "r", "c", "stage", "cond"]) + d = org_ghb.stage - new_ghb.stage + print(new_ghb.stage) + print(org_ghb.stage) + print(d) + assert d.sum() == 0.0, d.sum() + + # check the interaction with multiplicative ghb stage, direct ghb stage and additive ghb stage + par.loc[par.parnme.str.contains("mltstage"), "parval1"] = 1.1 + #par.loc[par.parnme.str.contains("ghbstage_inst:0"), "parval1"] = 0.0 + #par.loc[par.parnme.str.contains("ghb_stage"), "parval1"] += 3.0 + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + org_ghb = pd.read_csv(os.path.join(tmp_model_ws, "freyberg6.ghb_stress_period_data_1.txt"), + header=None, names=["l", "r", "c", "stage", "cond"], sep=r"\s+") + new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), + sep=r"\s+", + header=None, names=["l", "r", "c", "stage", "cond"]) + d = (org_ghb.stage * 1.1) - new_ghb.stage + print(new_ghb.stage) + print(org_ghb.stage) + print(d) + assert d.sum() == 0.0, d.sum() + + # turn direct recharge to min and direct wel to min and + # check that the model results are consistent + par = pst.parameter_data + rch_par = par.loc[par.parnme.apply( + lambda x: "pname:rch_gr" in x and "ptype:gr_pstyle:d" in x ), "parnme"] + wel_par = par.loc[par.parnme.apply( + lambda x: "pname:wel_grid" in x and "ptype:gr_usecol:3_pstyle:d" in x), "parnme"] + par.loc[rch_par,"parval1"] = par.loc[rch_par, "parlbnd"] + # this should set wells to zero since they are negative values in the control file + par.loc[wel_par,"parval1"] = par.loc[wel_par, "parubnd"] + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + lst = flopy.utils.Mf6ListBudget(os.path.join(pf.new_d, "freyberg6.lst")) + flx, cum = lst.get_dataframes(diff=True) + wel_tot = flx.wel.apply(np.abs).sum() + print(flx.wel) + assert wel_tot < 1.0e-6, wel_tot + + rch_files = [f for f in os.listdir(pf.new_d) + if ".rch_recharge" in f and f.endswith(".txt")] + rch_val = par.loc[rch_par,"parval1"][0] + i, j = par.loc[rch_par, ["i", 'j']].astype(int).values.T + for rch_file in rch_files: + arr = np.loadtxt(os.path.join(pf.new_d, rch_file))[i, j] + print(rch_file, rch_val, arr.mean(), arr.max(), arr.min()) + if np.abs(arr.max() - rch_val) > 1.0e-6 or np.abs(arr.min() - rch_val) > 1.0e-6: + raise Exception("recharge too diff") + + +def mf6_freyberg_varying_idomain(tmp_path): + import numpy as np + import pandas as pd + pd.set_option('display.max_rows', 500) + pd.set_option('display.max_columns', 500) + pd.set_option('display.width', 1000) + try: + import flopy + except: + return + + org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') + tmp_model_ws = setup_tmp(org_model_ws, tmp_path) + + os.chdir(tmp_path) + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + # sim.set_all_data_external() + sim.set_sim_path(str(tmp_model_ws)) + # sim.set_all_data_external() + m = sim.get_model("freyberg6") + sim.set_all_data_external(check_data=False) + sim.write_simulation() + + #sim = None + ib_file = os.path.join(tmp_model_ws,"freyberg6.dis_idomain_layer1.txt") + arr = np.loadtxt(ib_file,dtype=np.int64) + + arr[:2,:14] = 0 + np.savetxt(ib_file,arr,fmt="%2d") + print(arr) + + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + m = sim.get_model("freyberg6") + + # SETUP pest stuff... + os_utils.run("{0} ".format(mf6_exe_path), cwd=str(tmp_model_ws)) - #pst = pf.build_pst('freyberg.pst') - pf.parfile_relations.to_csv(os.path.join(pf.new_d, "mult2model_info.csv")) - os.chdir(pf.new_d) - df = pyemu.helpers.calc_array_par_summary_stats() - os.chdir(tmp_path) - pf.post_py_cmds.append("pyemu.helpers.calc_array_par_summary_stats()") - pf.add_observations("arr_par_summary.csv",index_cols=["model_file"],use_cols=df.columns.tolist(), - obsgp=["arr_par_summary" for _ in df.columns],prefix=["arr_par_summary" for _ in df.columns]) - pst = pf.build_pst('freyberg.pst') - pst.control_data.noptmax = 0 - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - - res_file = os.path.join(pf.new_d, "freyberg.base.rei") - assert os.path.exists(res_file), res_file - pst.set_res(res_file) - print(pst.phi) - assert pst.phi < 1.0e-6 - - pe = pf.draw(10,use_specsim=True) - pe.enforce() - pst.parameter_data.loc[:,"parval1"] = pe.loc[pe.index[0],pst.par_names] - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - - res_file = os.path.join(pf.new_d, "freyberg.base.rei") - assert os.path.exists(res_file), res_file - pst.set_res(res_file) - print(pst.phi) - - df = pd.read_csv(os.path.join(pf.new_d,"mult2model_info.csv"), index_col=0) - arr_pars = df.loc[df.index_cols.isna()].copy() - model_files = arr_pars.model_file.unique() - pst.try_parse_name_metadata() - for model_file in model_files: - arr = np.loadtxt(os.path.join(pf.new_d,model_file)) - clean_name = model_file.replace(".","_").replace("\\","_").replace("/","_") - sim_val = pst.res.loc[pst.res.name.apply(lambda x: clean_name in x ),"modelled"] - sim_val = sim_val.loc[sim_val.index.map(lambda x: "mean_model_file" in x)] - print(model_file,sim_val,arr.mean()) - except Exception as e: - os.chdir(bd) - raise e - os.chdir(bd) + + + template_ws = "new_temp" + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + + # if os.path.exists(template_ws): + # shutil.rmtree(template_ws) + # shutil.copytree(tmp_model_ws,template_ws) + # hk_file = os.path.join(template_ws, "freyberg6.npf_k_layer1.txt") + # hk = np.loadtxt(hk_file) + # + # hk[arr == 0] = 1.0e+30 + # np.savetxt(hk_file,hk,fmt="%50.45f") + # os_utils.run("{0} ".format(mf6_exe_path), cwd=template_ws) + # import matplotlib.pyplot as plt + # hds1 = flopy.utils.HeadFile(os.path.join(tmp_model_ws, "freyberg6_freyberg.hds")) + # hds2 = flopy.utils.HeadFile(os.path.join(template_ws, "freyberg6_freyberg.hds")) + # + # d = hds1.get_data() - hds2.get_data() + # for dd in d: + # cb = plt.imshow(dd) + # plt.colorbar(cb) + # plt.show() + # return + + # sr0 = m.sr + # sr = pyemu.helpers.SpatialReference.from_namfile( + # os.path.join(tmp_model_ws, "freyberg6.nam"), + # delr=m.dis.delr.array, delc=m.dis.delc.array) + + sr = m.modelgrid + # set up PstFrom object + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + remove_existing=True, + longnames=True, spatial_reference=sr, + zero_based=False, start_datetime="1-1-2018") + + + # pf.post_py_cmds.append("generic_function()") + df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) + pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", use_cols=list(df.columns.values), + ofile_sep=",") + v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) + gr_gs = pyemu.geostats.GeoStruct(variograms=v) + rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60)) + pf.extra_py_imports.append('flopy') + + ib = {} + for k in range(m.dis.nlay.data): + a = m.dis.idomain.array[k,:,:].copy() + print(a) + ib[k] = a + + tags = {"npf_k_": [0.1, 10.,0.003,35]}#, "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1]} + dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") + print(dts) + for tag, bnd in tags.items(): + lb, ub = bnd[0], bnd[1] + ult_lb = bnd[2] + ult_ub = bnd[3] + arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] + + for arr_file in arr_files: + + # these ult bounds are used later in an assert + + k = int(arr_file.split(".")[-2].split("layer")[1].split("_")[0]) - 1 + pf.add_parameters(filenames=arr_file, par_type="pilotpoints", par_name_base=arr_file.split('.')[1] + "_pp", + pargp=arr_file.split('.')[1] + "_pp", upper_bound=ub, lower_bound=lb, + geostruct=gr_gs, zone_array=ib[k],ult_lbound=ult_lb,ult_ubound=ult_ub) + + # add model run command + pf.mod_sys_cmds.append("mf6") + df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0) + df = pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", use_cols=list(df.columns.values), + prefix="hds", ofile_sep=",") + + + #pst = pf.build_pst('freyberg.pst') + pf.parfile_relations.to_csv(os.path.join(pf.new_d, "mult2model_info.csv")) + os.chdir(pf.new_d) + df = pyemu.helpers.calc_array_par_summary_stats() + os.chdir(tmp_path) + pf.post_py_cmds.append("pyemu.helpers.calc_array_par_summary_stats()") + pf.add_observations("arr_par_summary.csv",index_cols=["model_file"],use_cols=df.columns.tolist(), + obsgp=["arr_par_summary" for _ in df.columns],prefix=["arr_par_summary" for _ in df.columns]) + pst = pf.build_pst('freyberg.pst') + pst.control_data.noptmax = 0 + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + + res_file = os.path.join(pf.new_d, "freyberg.base.rei") + assert os.path.exists(res_file), res_file + pst.set_res(res_file) + print(pst.phi) + assert pst.phi < 1.0e-6 + + pe = pf.draw(10,use_specsim=True) + pe.enforce() + pst.parameter_data.loc[:,"parval1"] = pe.loc[pe.index[0],pst.par_names] + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + + res_file = os.path.join(pf.new_d, "freyberg.base.rei") + assert os.path.exists(res_file), res_file + pst.set_res(res_file) + print(pst.phi) + + df = pd.read_csv(os.path.join(pf.new_d,"mult2model_info.csv"), index_col=0) + arr_pars = df.loc[df.index_cols.isna()].copy() + model_files = arr_pars.model_file.unique() + pst.try_parse_name_metadata() + for model_file in model_files: + arr = np.loadtxt(os.path.join(pf.new_d,model_file)) + clean_name = model_file.replace(".","_").replace("\\","_").replace("/","_") + sim_val = pst.res.loc[pst.res.name.apply(lambda x: clean_name in x ),"modelled"] + sim_val = sim_val.loc[sim_val.index.map(lambda x: "mean_model_file" in x)] + print(model_file,sim_val,arr.mean()) def xsec_test(tmp_path): @@ -2522,42 +2500,37 @@ def xsec_test(tmp_path): # SETUP pest stuff... nam_file = "10par_xsec.nam" os_utils.run("{0} {1}".format(mf_exe_path,nam_file), cwd=tmp_model_ws) - bd = Path.cwd() - os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - m = flopy.modflow.Modflow.load(nam_file,model_ws=tmp_model_ws,version="mfnwt") - sr = m.modelgrid - t_d = "template_xsec" - pf = pyemu.utils.PstFrom(tmp_model_ws,t_d,remove_existing=True,spatial_reference=sr) - pf.add_parameters("hk_Layer_1.ref",par_type="grid",par_style="direct",upper_bound=25, - lower_bound=0.25) - pf.add_parameters("hk_Layer_1.ref", par_type="grid", par_style="multiplier", upper_bound=10.0, - lower_bound=0.1) - - hds_arr = np.loadtxt(os.path.join(t_d,"10par_xsec.hds")) - with open(os.path.join(t_d,"10par_xsec.hds.ins"),'w') as f: - f.write("pif ~\n") - for kper in range(hds_arr.shape[0]): - f.write("l1 ") - for j in range(hds_arr.shape[1]): - oname = "hds_{0}_{1}".format(kper,j) - f.write(" !{0}! ".format(oname)) - f.write("\n") - pf.add_observations_from_ins(os.path.join(t_d,"10par_xsec.hds.ins"),pst_path=".") - - pf.mod_sys_cmds.append("mfnwt {0}".format(nam_file)) - pf.build_pst(os.path.join(t_d,"pest.pst")) - - pyemu.os_utils.run("{0} {1}".format(ies_exe_path,"pest.pst"),cwd=t_d) - pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) - print(pst.phi) - assert np.isclose(pst.phi, 0), pst.phi - except Exception as e: - os.chdir(bd) - raise e - os.chdir(bd) + os.chdir(tmp_path) + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + m = flopy.modflow.Modflow.load(nam_file,model_ws=tmp_model_ws,version="mfnwt") + sr = m.modelgrid + t_d = "template_xsec" + pf = pyemu.utils.PstFrom(tmp_model_ws,t_d,remove_existing=True,spatial_reference=sr) + pf.add_parameters("hk_Layer_1.ref",par_type="grid",par_style="direct",upper_bound=25, + lower_bound=0.25) + pf.add_parameters("hk_Layer_1.ref", par_type="grid", par_style="multiplier", upper_bound=10.0, + lower_bound=0.1) + + hds_arr = np.loadtxt(os.path.join(t_d,"10par_xsec.hds")) + with open(os.path.join(t_d,"10par_xsec.hds.ins"),'w') as f: + f.write("pif ~\n") + for kper in range(hds_arr.shape[0]): + f.write("l1 ") + for j in range(hds_arr.shape[1]): + oname = "hds_{0}_{1}".format(kper,j) + f.write(" !{0}! ".format(oname)) + f.write("\n") + pf.add_observations_from_ins(os.path.join(t_d,"10par_xsec.hds.ins"),pst_path=".") + + pf.mod_sys_cmds.append("mfnwt {0}".format(nam_file)) + + pf.build_pst(os.path.join(t_d,"pest.pst")) + + pyemu.os_utils.run("{0} {1}".format(ies_exe_path,"pest.pst"),cwd=t_d) + pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) + print(pst.phi) + assert np.isclose(pst.phi, 0), pst.phi def mf6_freyberg_short_direct_test(tmp_path): @@ -2575,248 +2548,236 @@ def mf6_freyberg_short_direct_test(tmp_path): org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) - bd = Path.cwd() os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) - m = sim.get_model("freyberg6") - sim.set_all_data_external() - sim.write_simulation() + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + m = sim.get_model("freyberg6") + sim.set_all_data_external() + sim.write_simulation() - # SETUP pest stuff... - os_utils.run("{0} ".format("mf6"), cwd=tmp_model_ws) + # SETUP pest stuff... + os_utils.run("{0} ".format("mf6"), cwd=tmp_model_ws) - template_ws = "new_temp_direct" - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - sr = m.modelgrid - # set up PstFrom object - pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, - remove_existing=True, - longnames=False, spatial_reference=sr, - zero_based=False, start_datetime="1-1-2018") - # obs - # using tabular style model output - # (generated by pyemu.gw_utils.setup_hds_obs()) - # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', - # index_cols='obsnme', use_cols='obsval', prefix='hds') - - # Add stream flow observation - # df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) - pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", - use_cols=["GAGE_1","HEADWATER","TAILWATER"],ofile_sep=",") - # Setup geostruct for spatial pars - v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) - gr_gs = pyemu.geostats.GeoStruct(variograms=v, transform="log") - rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60)) - pf.extra_py_imports.append('flopy') - ib = m.dis.idomain[0].array - tags = { - "npf_k_": [0.1, 10.], - "npf_k33_": [.1, 10], - "sto_ss": [.1, 10], - "sto_sy": [.9, 1.1], - "rch_recharge": [.5, 1.5] - } - dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") - print(dts) - # setup from array style pars - for tag, bnd in tags.items(): - lb, ub = bnd[0], bnd[1] - arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] - if "rch" in tag: - for arr_file in arr_files: - nmebase = arr_file.split('.')[1].replace('layer','').replace('_','').replace("npf",'').replace("sto",'').replace("recharge",'') - # indy direct grid pars for each array type file - recharge_files = ["recharge_1.txt","recharge_2.txt","recharge_3.txt"] - pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base="rchg", - pargp="rch_gr", zone_array=ib, upper_bound=1.0e-3, lower_bound=1.0e-7, - par_style="direct") - # additional constant mults - kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 - pf.add_parameters(filenames=arr_file, par_type="constant", - par_name_base=nmebase + "cn", - pargp="rch_const", zone_array=ib, upper_bound=ub, lower_bound=lb, - geostruct=rch_temporal_gs, - datetime=dts[kper]) - else: - for arr_file in arr_files: - nmebase = arr_file.split('.')[1].replace( - 'layer', '').replace('_','').replace("npf",'').replace( - "sto",'').replace("recharge",'') - # grid mults pure and simple - pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base=nmebase, - pargp=arr_file.split('.')[1] + "_gr", zone_array=ib, upper_bound=ub, - lower_bound=lb, - geostruct=gr_gs) - - # Add a variety of list style pars - list_files = ["freyberg6.wel_stress_period_data_{0}.txt".format(t) - for t in range(1, m.nper + 1)] - # make dummy versions with headers - for fl in list_files[0:2]: # this adds a header to well file - with open(os.path.join(template_ws, fl), 'r') as fr: - lines = [line for line in fr] - with open(os.path.join(template_ws, f"new_{fl}"), 'w') as fw: - fw.write("k i j flx \n") - for line in lines: - fw.write(line) - - # fl = "freyberg6.wel_stress_period_data_3.txt" # Add extra string col_id - for fl in list_files[2:4]: - with open(os.path.join(template_ws, fl), 'r') as fr: - lines = [line for line in fr] - with open(os.path.join(template_ws, f"new_{fl}"), 'w') as fw: - fw.write("well k i j flx \n") - for i, line in enumerate(lines): - fw.write(f"w{i}" + line) - - list_files.sort() - for list_file in list_files: - kper = int(list_file.split(".")[1].split('_')[-1]) - 1 - #add spatially constant, but temporally correlated wel flux pars - pf.add_parameters(filenames=list_file, par_type="constant", par_name_base="wel{0}".format(kper), - pargp="twel_mlt_{0}".format(kper), index_cols=[0, 1, 2], use_cols=[3], - upper_bound=1.5, lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs) - - # add temporally indep, but spatially correlated wel flux pars - pf.add_parameters(filenames=list_file, par_type="grid", par_name_base="wel{0}".format(kper), - pargp="wel_{0}_direct".format(kper), index_cols=[0, 1, 2], use_cols=[3], - upper_bound=0.0, lower_bound=-1000, geostruct=gr_gs, par_style="direct", - transform="none") - # # Adding dummy list pars with different file structures - list_file = "new_freyberg6.wel_stress_period_data_1.txt" # with a header - pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nw', - pargp='nwell_mult', index_cols=['k', 'i', 'j'], use_cols='flx', - upper_bound=10, lower_bound=-10, geostruct=gr_gs, - transform="none") - pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nw', - pargp='nwell', index_cols=['k', 'i', 'j'], use_cols='flx', - upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", - transform="none") - # with skip instead - list_file = "new_freyberg6.wel_stress_period_data_2.txt" - pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nw', - pargp='nwell', index_cols=[0, 1, 2], use_cols=3, - upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", - transform="none", mfile_skip=1) - - list_file = "new_freyberg6.wel_stress_period_data_3.txt" - pf.add_parameters(filenames=list_file, par_type="grid", - pargp='nwell_mult', index_cols=['well', 'k', 'i', 'j'], use_cols='flx', - upper_bound=10, lower_bound=-10, geostruct=gr_gs, - transform="none") - pf.add_parameters(filenames=list_file, par_type="grid", - pargp='nwell', index_cols=['well', 'k', 'i', 'j'], use_cols='flx', - upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", - transform="none") - # with skip instead - list_file = "new_freyberg6.wel_stress_period_data_4.txt" - pf.add_parameters(filenames=list_file, par_type="grid", - par_name_base='nw', pargp='nwell', - index_cols=[0, 1, 2, 3], # or... {'well': 0, 'k': 1, 'i': 2, 'j': 3}, - use_cols=4, upper_bound=10, lower_bound=-10, - geostruct=gr_gs, par_style="direct", transform="none", - mfile_skip=1) - - list_file = "freyberg6.ghb_stress_period_data_1.txt" - pf.add_parameters(filenames=list_file, par_type="constant", par_name_base=["ghbst","ghbc"], - pargp=["ghb_stage","ghb_cond"], index_cols=[0, 1, 2], use_cols=[3,4], - upper_bound=[35,150], lower_bound=[32,50], par_style="direct", + template_ws = "new_temp_direct" + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + sr = m.modelgrid + # set up PstFrom object + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + remove_existing=True, + longnames=False, spatial_reference=sr, + zero_based=False, start_datetime="1-1-2018") + # obs + # using tabular style model output + # (generated by pyemu.gw_utils.setup_hds_obs()) + # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', + # index_cols='obsnme', use_cols='obsval', prefix='hds') + + # Add stream flow observation + # df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) + pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", + use_cols=["GAGE_1","HEADWATER","TAILWATER"],ofile_sep=",") + # Setup geostruct for spatial pars + v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) + gr_gs = pyemu.geostats.GeoStruct(variograms=v, transform="log") + rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60)) + pf.extra_py_imports.append('flopy') + ib = m.dis.idomain[0].array + tags = { + "npf_k_": [0.1, 10.], + "npf_k33_": [.1, 10], + "sto_ss": [.1, 10], + "sto_sy": [.9, 1.1], + "rch_recharge": [.5, 1.5] + } + dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") + print(dts) + # setup from array style pars + for tag, bnd in tags.items(): + lb, ub = bnd[0], bnd[1] + arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] + if "rch" in tag: + for arr_file in arr_files: + nmebase = arr_file.split('.')[1].replace('layer','').replace('_','').replace("npf",'').replace("sto",'').replace("recharge",'') + # indy direct grid pars for each array type file + recharge_files = ["recharge_1.txt","recharge_2.txt","recharge_3.txt"] + pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base="rchg", + pargp="rch_gr", zone_array=ib, upper_bound=1.0e-3, lower_bound=1.0e-7, + par_style="direct") + # additional constant mults + kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 + pf.add_parameters(filenames=arr_file, par_type="constant", + par_name_base=nmebase + "cn", + pargp="rch_const", zone_array=ib, upper_bound=ub, lower_bound=lb, + geostruct=rch_temporal_gs, + datetime=dts[kper]) + else: + for arr_file in arr_files: + nmebase = arr_file.split('.')[1].replace( + 'layer', '').replace('_','').replace("npf",'').replace( + "sto",'').replace("recharge",'') + # grid mults pure and simple + pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base=nmebase, + pargp=arr_file.split('.')[1] + "_gr", zone_array=ib, upper_bound=ub, + lower_bound=lb, + geostruct=gr_gs) + + # Add a variety of list style pars + list_files = ["freyberg6.wel_stress_period_data_{0}.txt".format(t) + for t in range(1, m.nper + 1)] + # make dummy versions with headers + for fl in list_files[0:2]: # this adds a header to well file + with open(os.path.join(template_ws, fl), 'r') as fr: + lines = [line for line in fr] + with open(os.path.join(template_ws, f"new_{fl}"), 'w') as fw: + fw.write("k i j flx \n") + for line in lines: + fw.write(line) + + # fl = "freyberg6.wel_stress_period_data_3.txt" # Add extra string col_id + for fl in list_files[2:4]: + with open(os.path.join(template_ws, fl), 'r') as fr: + lines = [line for line in fr] + with open(os.path.join(template_ws, f"new_{fl}"), 'w') as fw: + fw.write("well k i j flx \n") + for i, line in enumerate(lines): + fw.write(f"w{i}" + line) + + list_files.sort() + for list_file in list_files: + kper = int(list_file.split(".")[1].split('_')[-1]) - 1 + #add spatially constant, but temporally correlated wel flux pars + pf.add_parameters(filenames=list_file, par_type="constant", par_name_base="wel{0}".format(kper), + pargp="twel_mlt_{0}".format(kper), index_cols=[0, 1, 2], use_cols=[3], + upper_bound=1.5, lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs) + + # add temporally indep, but spatially correlated wel flux pars + pf.add_parameters(filenames=list_file, par_type="grid", par_name_base="wel{0}".format(kper), + pargp="wel_{0}_direct".format(kper), index_cols=[0, 1, 2], use_cols=[3], + upper_bound=0.0, lower_bound=-1000, geostruct=gr_gs, par_style="direct", transform="none") + # # Adding dummy list pars with different file structures + list_file = "new_freyberg6.wel_stress_period_data_1.txt" # with a header + pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nw', + pargp='nwell_mult', index_cols=['k', 'i', 'j'], use_cols='flx', + upper_bound=10, lower_bound=-10, geostruct=gr_gs, + transform="none") + pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nw', + pargp='nwell', index_cols=['k', 'i', 'j'], use_cols='flx', + upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", + transform="none") + # with skip instead + list_file = "new_freyberg6.wel_stress_period_data_2.txt" + pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nw', + pargp='nwell', index_cols=[0, 1, 2], use_cols=3, + upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", + transform="none", mfile_skip=1) - # add model run command - pf.mod_sys_cmds.append("mf6") - print(pf.mult_files) - print(pf.org_files) - - # build pest - pst = pf.build_pst('freyberg.pst') - #cov = pf.build_prior(fmt="non") - #cov.to_coo("prior.jcb") - pst.try_parse_name_metadata() - df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0) - pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", - use_cols=list(df.columns.values), - prefix="hds", rebuild_pst=True) - - # test par mults are working - - os.chdir(pf.new_d) - pst.write_input_files() - pyemu.helpers.apply_list_and_array_pars( - arr_par_file="mult2model_info.csv", chunk_len=1) - - # TODO Some checks on resultant par files... - list_files = [f for f in os.listdir('.') - if f.startswith('new_') and f.endswith('txt')] - # check on that those dummy pars compare to the model versions. - for f in list_files: - n_df = pd.read_csv(f, sep=r"\s+") - o_df = pd.read_csv(f.strip('new_'), sep=r"\s+", header=None) - o_df.columns = ['k', 'i', 'j', 'flx'] - assert np.allclose(o_df.values, - n_df.loc[:, o_df.columns].values, - rtol=1e-4), ( - f"Something broke with alternative style model file: {f}" - ) - os.chdir(tmp_path) + list_file = "new_freyberg6.wel_stress_period_data_3.txt" + pf.add_parameters(filenames=list_file, par_type="grid", + pargp='nwell_mult', index_cols=['well', 'k', 'i', 'j'], use_cols='flx', + upper_bound=10, lower_bound=-10, geostruct=gr_gs, + transform="none") + pf.add_parameters(filenames=list_file, par_type="grid", + pargp='nwell', index_cols=['well', 'k', 'i', 'j'], use_cols='flx', + upper_bound=10, lower_bound=-10, geostruct=gr_gs, par_style="direct", + transform="none") + # with skip instead + list_file = "new_freyberg6.wel_stress_period_data_4.txt" + pf.add_parameters(filenames=list_file, par_type="grid", + par_name_base='nw', pargp='nwell', + index_cols=[0, 1, 2, 3], # or... {'well': 0, 'k': 1, 'i': 2, 'j': 3}, + use_cols=4, upper_bound=10, lower_bound=-10, + geostruct=gr_gs, par_style="direct", transform="none", + mfile_skip=1) - num_reals = 100 - pe = pf.draw(num_reals, use_specsim=True) - pe.to_binary(os.path.join(template_ws, "prior.jcb")) - assert pe.shape[1] == pst.npar_adj, "{0} vs {1}".format(pe.shape[0], pst.npar_adj) - assert pe.shape[0] == num_reals - - pst.control_data.noptmax = 0 - pst.pestpp_options["additional_ins_delimiters"] = "," - - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - - res_file = os.path.join(pf.new_d, "freyberg.base.rei") - assert os.path.exists(res_file), res_file - pst.set_res(res_file) - print(pst.phi) - assert np.isclose(pst.phi, 0), pst.phi - - - # turn direct recharge to min and direct wel to min and - # check that the model results are consistent - par = pst.parameter_data - rch_par = par.loc[(par.pname == 'rch') & - (par.ptype == 'gr') & - (par.pstyle == 'd'), - "parnme"] - wel_par = par.loc[(par.pname.str.contains('wel')) & - (par.pstyle == 'd'), - "parnme"] - par.loc[rch_par, "parval1"] = par.loc[rch_par, "parlbnd"] - # this should set wells to zero since they are negative values in the control file - par.loc[wel_par, "parval1"] = par.loc[wel_par, "parubnd"] - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - lst = flopy.utils.Mf6ListBudget(os.path.join(pf.new_d, "freyberg6.lst")) - flx, cum = lst.get_dataframes(diff=True) - wel_tot = flx.wel.apply(np.abs).sum() - print(flx.wel) - assert np.isclose(wel_tot, 0), wel_tot - - # shortpars so not going to be able to get ij easily - # rch_files = [f for f in os.listdir(pf.new_d) - # if ".rch_recharge" in f and f.endswith(".txt")] - # rch_val = par.loc[rch_par,"parval1"][0] - # i, j = par.loc[rch_par, ["i", 'j']].astype(int).values.T - # for rch_file in rch_files: - # arr = np.loadtxt(os.path.join(pf.new_d, rch_file))[i, j] - # print(rch_file, rch_val, arr.mean(), arr.max(), arr.min()) - # if np.abs(arr.max() - rch_val) > 1.0e-6 or np.abs(arr.min() - rch_val) > 1.0e-6: - # raise Exception("recharge too diff") - except Exception as e: - os.chdir(bd) - raise e - os.chdir(bd) + list_file = "freyberg6.ghb_stress_period_data_1.txt" + pf.add_parameters(filenames=list_file, par_type="constant", par_name_base=["ghbst","ghbc"], + pargp=["ghb_stage","ghb_cond"], index_cols=[0, 1, 2], use_cols=[3,4], + upper_bound=[35,150], lower_bound=[32,50], par_style="direct", + transform="none") + + # add model run command + pf.mod_sys_cmds.append("mf6") + print(pf.mult_files) + print(pf.org_files) + + # build pest + pst = pf.build_pst('freyberg.pst') + #cov = pf.build_prior(fmt="non") + #cov.to_coo("prior.jcb") + pst.try_parse_name_metadata() + df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0) + pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", + use_cols=list(df.columns.values), + prefix="hds", rebuild_pst=True) + + # test par mults are working + check_apply(pf, 10) + + # TODO Some checks on resultant par files... + list_files = list(Path(pf.new_d).glob("new_*txt")) + # check on that those dummy pars compare to the model versions. + for f in list_files: + n_df = pd.read_csv(f, sep=r"\s+") + o_df = pd.read_csv(f.with_name(f.name.strip('new_')), sep=r"\s+", header=None) + o_df.columns = ['k', 'i', 'j', 'flx'] + assert np.allclose(o_df.values, + n_df.loc[:, o_df.columns].values, + rtol=1e-4), ( + f"Something broke with alternative style model file: {f}" + ) + + num_reals = 100 + pe = pf.draw(num_reals, use_specsim=True) + pe.to_binary(os.path.join(template_ws, "prior.jcb")) + assert pe.shape[1] == pst.npar_adj, "{0} vs {1}".format(pe.shape[0], pst.npar_adj) + assert pe.shape[0] == num_reals + + pst.control_data.noptmax = 0 + pst.pestpp_options["additional_ins_delimiters"] = "," + + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + + res_file = os.path.join(pf.new_d, "freyberg.base.rei") + assert os.path.exists(res_file), res_file + pst.set_res(res_file) + print(pst.phi) + assert np.isclose(pst.phi, 0), pst.phi + + + # turn direct recharge to min and direct wel to min and + # check that the model results are consistent + par = pst.parameter_data + rch_par = par.loc[(par.pname == 'rch') & + (par.ptype == 'gr') & + (par.pstyle == 'd'), + "parnme"] + wel_par = par.loc[(par.pname.str.contains('wel')) & + (par.pstyle == 'd'), + "parnme"] + par.loc[rch_par, "parval1"] = par.loc[rch_par, "parlbnd"] + # this should set wells to zero since they are negative values in the control file + par.loc[wel_par, "parval1"] = par.loc[wel_par, "parubnd"] + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + lst = flopy.utils.Mf6ListBudget(os.path.join(pf.new_d, "freyberg6.lst")) + flx, cum = lst.get_dataframes(diff=True) + wel_tot = flx.wel.apply(np.abs).sum() + print(flx.wel) + assert np.isclose(wel_tot, 0), wel_tot + + # shortpars so not going to be able to get ij easily + # rch_files = [f for f in os.listdir(pf.new_d) + # if ".rch_recharge" in f and f.endswith(".txt")] + # rch_val = par.loc[rch_par,"parval1"][0] + # i, j = par.loc[rch_par, ["i", 'j']].astype(int).values.T + # for rch_file in rch_files: + # arr = np.loadtxt(os.path.join(pf.new_d, rch_file))[i, j] + # print(rch_file, rch_val, arr.mean(), arr.max(), arr.min()) + # if np.abs(arr.max() - rch_val) > 1.0e-6 or np.abs(arr.min() - rch_val) > 1.0e-6: + # raise Exception("recharge too diff") class TestPstFrom(): @@ -2884,406 +2845,390 @@ def test_add_array_parameters(self): """test setting up array parameters with different external file configurations and path formats. """ - try: - tag = 'hk' - # test with different array input configurations - array_file_input = [ - Path('hk0.dat'), # sim_ws; just file name as Path instance - 'hk1.dat', # sim_ws; just file name as string - Path(self.sim_ws, 'hk2.dat'), # sim_ws; full path as Path instance - 'external/hk3.dat', # subfolder; relative file path as string - Path('external/hk4.dat'), # subfolder; relative path as Path instance - '../external_files/hk5.dat', # subfolder up one level - ] - for i, array_file in enumerate(array_file_input): - par_name_base = f'{tag}_{i:d}' - - # create the file - # dest_file is the data file relative to the sim or dest ws - dest_file = Path(array_file) - if self.sim_ws in dest_file.parents: - dest_file = dest_file.relative_to(self.sim_ws) - shutil.copy(self.array_file, Path(self.dest_ws, dest_file)) - - self.pf.add_parameters(filenames=array_file, par_type='zone', - zone_array=self.zone_array, - par_name_base=par_name_base, # basename for parameters that are set up - pargp=f'{tag}_zone', # Parameter group to assign pars to. - ) - - assert (self.dest_ws / dest_file).exists() - assert (self.dest_ws / f'org/{dest_file.name}').exists() - # mult file name is par_name_base + `instance` identifier + part_type - mult_filename = f'{par_name_base}_inst0_zone.csv' - assert (self.dest_ws / f'mult/{mult_filename}').exists() - # for now, assume tpl file should be in main folder - template_file = (self.pf.tpl_d / f'{mult_filename}.tpl') - assert template_file.exists() - - # make the PEST control file - pst = self.pf.build_pst() - assert pst.filename == Path(self.dest_ws, 'pst-from-small.pst') - assert pst.filename.exists() - rel_tpl = pyemu.utils.pst_from.get_relative_filepath(self.pf.new_d, template_file) - assert rel_tpl in pst.template_files - - # make the PEST control file (just filename) - pst = self.pf.build_pst('junk.pst') - assert pst.filename == Path(self.dest_ws, 'junk.pst') - assert pst.filename.exists() - - # make the PEST control file (file path) - pst = self.pf.build_pst(str(Path(self.dest_ws, 'junk2.pst'))) - assert pst.filename == Path(self.dest_ws, 'junk2.pst') - assert pst.filename.exists() - - # check the mult2model info - df = pd.read_csv(self.dest_ws / 'mult2model_info.csv') - # org data file relative to dest_ws - org_file = Path(df['org_file'].values[i]) - assert org_file == Path(f'org/{dest_file.name}') - # model file relative to dest_ws - model_file = Path(df['model_file'].values[i]) - assert model_file == dest_file - # mult file - mult_file = Path(df['mlt_file'].values[i]) - assert mult_file == Path(f'mult/{mult_filename}') - - # check applying the parameters (in the dest or template ws) - os.chdir(self.dest_ws) - # first delete the model file in the template ws - model_file.unlink() - # manually apply a multipler - mult = 4 - mult_values = np.loadtxt(mult_file) - mult_values[:] = mult - np.savetxt(mult_file, mult_values) - # apply the multiplier - pyemu.helpers.apply_list_and_array_pars(arr_par_file='mult2model_info.csv') - # model file should have been remade by apply_list_and_array_pars - assert model_file.exists() - result = np.loadtxt(model_file) - # results should be the same with default multipliers of 1 - # assume details of parameterization are handled by other tests - assert np.allclose(result, self.array_data * mult) - # revert to original wd - os.chdir(self.tmp_path) - except Exception as e: - os.chdir(self.original_wd) - raise e + tag = 'hk' + # test with different array input configurations + array_file_input = [ + Path('hk0.dat'), # sim_ws; just file name as Path instance + 'hk1.dat', # sim_ws; just file name as string + Path(self.sim_ws, 'hk2.dat'), # sim_ws; full path as Path instance + 'external/hk3.dat', # subfolder; relative file path as string + Path('external/hk4.dat'), # subfolder; relative path as Path instance + '../external_files/hk5.dat', # subfolder up one level + ] + for i, array_file in enumerate(array_file_input): + par_name_base = f'{tag}_{i:d}' + + # create the file + # dest_file is the data file relative to the sim or dest ws + dest_file = Path(array_file) + if self.sim_ws in dest_file.parents: + dest_file = dest_file.relative_to(self.sim_ws) + shutil.copy(self.array_file, Path(self.dest_ws, dest_file)) + + self.pf.add_parameters(filenames=array_file, par_type='zone', + zone_array=self.zone_array, + par_name_base=par_name_base, # basename for parameters that are set up + pargp=f'{tag}_zone', # Parameter group to assign pars to. + ) + + assert (self.dest_ws / dest_file).exists() + assert (self.dest_ws / f'org/{dest_file.name}').exists() + # mult file name is par_name_base + `instance` identifier + part_type + mult_filename = f'{par_name_base}_inst0_zone.csv' + assert (self.dest_ws / f'mult/{mult_filename}').exists() + # for now, assume tpl file should be in main folder + template_file = (self.pf.tpl_d / f'{mult_filename}.tpl') + assert template_file.exists() + + # make the PEST control file + pst = self.pf.build_pst() + assert pst.filename == Path(self.dest_ws, 'pst-from-small.pst') + assert pst.filename.exists() + rel_tpl = pyemu.utils.pst_from.get_relative_filepath(self.pf.new_d, template_file) + assert rel_tpl in pst.template_files + + # make the PEST control file (just filename) + pst = self.pf.build_pst('junk.pst') + assert pst.filename == Path(self.dest_ws, 'junk.pst') + assert pst.filename.exists() + + # make the PEST control file (file path) + pst = self.pf.build_pst(str(Path(self.dest_ws, 'junk2.pst'))) + assert pst.filename == Path(self.dest_ws, 'junk2.pst') + assert pst.filename.exists() + + # check the mult2model info + df = pd.read_csv(self.dest_ws / 'mult2model_info.csv') + # org data file relative to dest_ws + org_file = Path(df['org_file'].values[i]) + assert org_file == Path(f'org/{dest_file.name}') + # model file relative to dest_ws + model_file = Path(df['model_file'].values[i]) + assert model_file == dest_file + # mult file + mult_file = Path(df['mlt_file'].values[i]) + assert mult_file == Path(f'mult/{mult_filename}') + + # check applying the parameters (in the dest or template ws) + os.chdir(self.dest_ws) + # first delete the model file in the template ws + model_file.unlink() + # manually apply a multipler + mult = 4 + mult_values = np.loadtxt(mult_file) + mult_values[:] = mult + np.savetxt(mult_file, mult_values) + # apply the multiplier + pyemu.helpers.apply_list_and_array_pars(arr_par_file='mult2model_info.csv') + # model file should have been remade by apply_list_and_array_pars + assert model_file.exists() + result = np.loadtxt(model_file) + # results should be the same with default multipliers of 1 + # assume details of parameterization are handled by other tests + assert np.allclose(result, self.array_data * mult) + # revert to original wd + os.chdir(self.tmp_path) def test_add_list_parameters(self): """test setting up list parameters with different external file configurations and path formats. """ - try: - tag = 'wel' - # test with different array input configurations - list_file_input = [ - Path('wel0.dat'), # sim_ws; just file name as Path instance - 'wel1.dat', # sim_ws; just file name as string - Path(self.sim_ws, 'wel2.dat'), # sim_ws; full path as Path instance - 'external/wel3.dat', # subfolder; relative file path as string - Path('external/wel4.dat'), # subfolder; relative path as Path instance - '../external_files/wel5.dat', # subfolder up one level - ] - par_type = 'constant' - for i, list_file in enumerate(list_file_input): - par_name_base = f'{tag}_{i:d}' - - # create the file - # dest_file is the data file relative to the sim or dest ws - dest_file = Path(list_file) - if self.sim_ws in dest_file.parents: - dest_file = dest_file.relative_to(self.sim_ws) - shutil.copy(self.list_file, Path(self.dest_ws, dest_file)) - - self.pf.add_parameters(filenames=list_file, par_type=par_type, - par_name_base=par_name_base, - index_cols=[0, 1, 2], use_cols=[3], - pargp=f'{tag}_{i}', - comment_char='#', - ) - - assert (self.dest_ws / dest_file).exists() - assert (self.dest_ws / f'org/{dest_file.name}').exists() - # mult file name is par_name_base + `instance` identifier + part_type - mult_filename = f'{par_name_base}_inst0_{par_type}.csv' - assert (self.dest_ws / f'mult/{mult_filename}').exists() - # for now, assume tpl file should be in main folder - template_file = (self.pf.tpl_d / f'{mult_filename}.tpl') - assert template_file.exists() - - # make the PEST control file - pst = self.pf.build_pst() - rel_tpl = pyemu.utils.pst_from.get_relative_filepath(self.pf.new_d, template_file) - assert rel_tpl in pst.template_files - - # check the mult2model info - df = pd.read_csv(self.dest_ws / 'mult2model_info.csv') - # org data file relative to dest_ws - org_file = Path(df['org_file'].values[i]) - assert org_file == Path(f'org/{dest_file.name}') - # model file relative to dest_ws - model_file = Path(df['model_file'].values[i]) - assert model_file == dest_file - # mult file - mult_file = Path(df['mlt_file'].values[i]) - assert mult_file == Path(f'mult/{mult_filename}') - - # check applying the parameters (in the dest or template ws) - os.chdir(self.dest_ws) - # first delete the model file in the template ws - model_file.unlink() - # manually apply a multipler - mult = 4 - mult_df = pd.read_csv(mult_file) - # no idea why '3' is the column with multipliers and 'parval1_3' isn't - # what is the purpose of 'parval1_3'? - parval_col = '3' - mult_df[parval_col] = mult - mult_df.to_csv(mult_file, index=False) - # apply the multiplier - pyemu.helpers.apply_list_and_array_pars(arr_par_file='mult2model_info.csv') - # model file should have been remade by apply_list_and_array_pars - assert model_file.exists() - result = pd.read_csv(model_file, sep=r'\s+') - # results should be the same with default multipliers of 1 - # assume details of parameterization are handled by other tests - assert np.allclose(result['flux'], self.list_data['flux'] * mult) - - # revert to original wd - os.chdir(self.tmp_path) - new_list = self.list_data.copy() - new_list[['x', 'y']] = new_list[['i', 'j']].apply( - lambda r: pd.Series( - [self.pf._spatial_reference.ycentergrid[r.i - 1, r.j - 1], - self.pf._spatial_reference.xcentergrid[r.i - 1, r.j - 1]] - ), axis=1) - new_list.to_csv(Path(self.dest_ws, "xylist.csv"), index=False, header=False) - self.pf.add_parameters(filenames="xylist.csv", par_type='grid', - par_name_base="xywel", - index_cols={'lay': 0, 'x': 4, 'y': 5}, - use_cols=[3], - pargp=f'xywel', - geostruct=self.grid_gs, - rebuild_pst=True + tag = 'wel' + # test with different array input configurations + list_file_input = [ + Path('wel0.dat'), # sim_ws; just file name as Path instance + 'wel1.dat', # sim_ws; just file name as string + Path(self.sim_ws, 'wel2.dat'), # sim_ws; full path as Path instance + 'external/wel3.dat', # subfolder; relative file path as string + Path('external/wel4.dat'), # subfolder; relative path as Path instance + '../external_files/wel5.dat', # subfolder up one level + ] + par_type = 'constant' + for i, list_file in enumerate(list_file_input): + par_name_base = f'{tag}_{i:d}' + + # create the file + # dest_file is the data file relative to the sim or dest ws + dest_file = Path(list_file) + if self.sim_ws in dest_file.parents: + dest_file = dest_file.relative_to(self.sim_ws) + shutil.copy(self.list_file, Path(self.dest_ws, dest_file)) + + self.pf.add_parameters(filenames=list_file, par_type=par_type, + par_name_base=par_name_base, + index_cols=[0, 1, 2], use_cols=[3], + pargp=f'{tag}_{i}', + comment_char='#', ) - cov = self.pf.build_prior() - x = cov.as_2d[-3:, -3:] - assert np.count_nonzero(x - np.diag(np.diagonal(x))) == 6 - assert np.sum(x < np.diag(x)) == 6 - except Exception as e: - os.chdir(self.original_wd) - raise e - def test_add_array_parameters_pps_grid(self): - """test setting up array parameters with a list of array text - files in a subfolder. - """ - try: - tag = 'hk' - par_styles = ['multiplier', #'direct' - ] - array_files = ['hk_{}_{}.dat', 'external/hk_{}_{}.dat'] - for par_style in par_styles: - mult2model_row = 0 - for j, array_file in enumerate(array_files): - - par_types = {'pilotpoints': 'pp', - 'grid': 'gr'} - for i, (par_type, suffix) in enumerate(par_types.items()): - # (re)create the file - dest_file = array_file.format(mult2model_row, suffix) - shutil.copy(self.array_file, Path(self.dest_ws, dest_file)) - # add the parameters - par_name_base = f'{tag}_{suffix}' - self.pf.add_parameters(filenames=dest_file, par_type=par_type, - zone_array=self.zone_array, - par_name_base=par_name_base, - pargp=f'{tag}_zone', - pp_space=1, geostruct=self.grid_gs, - par_style=par_style - ) - if par_type != 'pilotpoints': - template_file = (self.pf.tpl_d / f'{par_name_base}_inst0_grid.csv.tpl') - assert template_file.exists() - else: - template_file = (self.pf.tpl_d / f'{par_name_base}_inst0pp.dat.tpl') - assert template_file.exists() - - # make the PEST control file - pst = self.pf.build_pst() - rel_tpl = pyemu.utils.pst_from.get_relative_filepath(self.pf.new_d, template_file) - assert rel_tpl in pst.template_files - - # check the mult2model info - df = pd.read_csv(self.dest_ws / 'mult2model_info.csv') - mult_file = Path(df['mlt_file'].values[mult2model_row]) - - # check applying the parameters (in the dest or template ws) - os.chdir(self.dest_ws) - # first delete the model file in the template ws - model_file = df['model_file'].values[mult2model_row] - os.remove(model_file) - # manually apply a multipler - mult = 4 - if par_type != "pilotpoints": - mult_values = np.loadtxt(mult_file) - mult_values[:] = mult - np.savetxt(mult_file, mult_values) - else: - ppdata = pp_file_to_dataframe(df['pp_file'].values[mult2model_row]) - ppdata['parval1'] = mult - write_pp_file(df['pp_file'].values[mult2model_row], ppdata) - # apply the multiplier - pyemu.helpers.apply_list_and_array_pars(arr_par_file='mult2model_info.csv') - # model files should have been remade by apply_list_and_array_pars - for model_file in df['model_file']: - assert os.path.exists(model_file) - result = np.loadtxt(model_file) - # results should be the same with default multipliers of 1 - # assume details of parameterization are handled by other tests - - # not sure why zone 2 is coming back as invalid (1e30) - zone1 = self.zone_array == 1 - assert np.allclose(result[zone1], self.array_data[zone1] * mult) - - # revert to original wd - os.chdir(self.tmp_path) - mult2model_row += 1 - except Exception as e: - os.chdir(self.original_wd) - raise e + assert (self.dest_ws / dest_file).exists() + assert (self.dest_ws / f'org/{dest_file.name}').exists() + # mult file name is par_name_base + `instance` identifier + part_type + mult_filename = f'{par_name_base}_inst0_{par_type}.csv' + assert (self.dest_ws / f'mult/{mult_filename}').exists() + # for now, assume tpl file should be in main folder + template_file = (self.pf.tpl_d / f'{mult_filename}.tpl') + assert template_file.exists() - def test_add_direct_array_parameters(self): + # make the PEST control file + pst = self.pf.build_pst() + rel_tpl = pyemu.utils.pst_from.get_relative_filepath(self.pf.new_d, template_file) + assert rel_tpl in pst.template_files + + # check the mult2model info + df = pd.read_csv(self.dest_ws / 'mult2model_info.csv') + # org data file relative to dest_ws + org_file = Path(df['org_file'].values[i]) + assert org_file == Path(f'org/{dest_file.name}') + # model file relative to dest_ws + model_file = Path(df['model_file'].values[i]) + assert model_file == dest_file + # mult file + mult_file = Path(df['mlt_file'].values[i]) + assert mult_file == Path(f'mult/{mult_filename}') + + # check applying the parameters (in the dest or template ws) + os.chdir(self.dest_ws) + # first delete the model file in the template ws + model_file.unlink() + # manually apply a multipler + mult = 4 + mult_df = pd.read_csv(mult_file) + # no idea why '3' is the column with multipliers and 'parval1_3' isn't + # what is the purpose of 'parval1_3'? + parval_col = '3' + mult_df[parval_col] = mult + mult_df.to_csv(mult_file, index=False) + # apply the multiplier + pyemu.helpers.apply_list_and_array_pars(arr_par_file='mult2model_info.csv') + # model file should have been remade by apply_list_and_array_pars + assert model_file.exists() + result = pd.read_csv(model_file, sep=r'\s+') + # results should be the same with default multipliers of 1 + # assume details of parameterization are handled by other tests + assert np.allclose(result['flux'], self.list_data['flux'] * mult) + + # revert to original wd + os.chdir(self.tmp_path) + new_list = self.list_data.copy() + new_list[['x', 'y']] = new_list[['i', 'j']].apply( + lambda r: pd.Series( + [self.pf._spatial_reference.ycentergrid[r.i - 1, r.j - 1], + self.pf._spatial_reference.xcentergrid[r.i - 1, r.j - 1]] + ), axis=1) + new_list.to_csv(Path(self.dest_ws, "xylist.csv"), index=False, header=False) + self.pf.add_parameters(filenames="xylist.csv", par_type='grid', + par_name_base="xywel", + index_cols={'lay': 0, 'x': 4, 'y': 5}, + use_cols=[3], + pargp=f'xywel', + geostruct=self.grid_gs, + rebuild_pst=True + ) + cov = self.pf.build_prior() + x = cov.as_2d[-3:, -3:] + assert np.count_nonzero(x - np.diag(np.diagonal(x))) == 6 + assert np.sum(x < np.diag(x)) == 6 + + + def test_add_array_parameters_pps_grid(self): """test setting up array parameters with a list of array text files in a subfolder. """ - try: - tag = 'hk' - par_styles = ['direct', #'direct' - ] - array_files = ['hk_{}_{}.dat', 'external/hk_{}_{}.dat'] - for par_style in par_styles: - mult2model_row = 0 - for j, array_file in enumerate(array_files): - - par_types = {#'constant': 'cn', - 'zone': 'zn', - 'grid': 'gr'} - for i, (par_type, suffix) in enumerate(par_types.items()): - # (re)create the file - dest_file = array_file.format(mult2model_row, suffix) - - # make a new input array file with initial values - arr = np.loadtxt(self.array_file) - parval = 8 - arr[:] = parval - np.savetxt(Path(self.dest_ws, dest_file), arr) - - # add the parameters - par_name_base = f'{tag}_{suffix}' - self.pf.add_parameters(filenames=dest_file, par_type=par_type, - zone_array=self.zone_array, - par_name_base=par_name_base, - pargp=f'{tag}_zone', - par_style=par_style - ) - template_file = (self.pf.tpl_d / f'{Path(dest_file).name}.tpl') + tag = 'hk' + par_styles = ['multiplier', #'direct' + ] + array_files = ['hk_{}_{}.dat', 'external/hk_{}_{}.dat'] + for par_style in par_styles: + mult2model_row = 0 + for j, array_file in enumerate(array_files): + + par_types = {'pilotpoints': 'pp', + 'grid': 'gr'} + for i, (par_type, suffix) in enumerate(par_types.items()): + # (re)create the file + dest_file = array_file.format(mult2model_row, suffix) + shutil.copy(self.array_file, Path(self.dest_ws, dest_file)) + # add the parameters + par_name_base = f'{tag}_{suffix}' + self.pf.add_parameters(filenames=dest_file, par_type=par_type, + zone_array=self.zone_array, + par_name_base=par_name_base, + pargp=f'{tag}_zone', + pp_space=1, geostruct=self.grid_gs, + par_style=par_style + ) + if par_type != 'pilotpoints': + template_file = (self.pf.tpl_d / f'{par_name_base}_inst0_grid.csv.tpl') + assert template_file.exists() + else: + template_file = (self.pf.tpl_d / f'{par_name_base}_inst0pp.dat.tpl') assert template_file.exists() - # make the PEST control file - pst = self.pf.build_pst() - rel_tpl = pyemu.utils.pst_from.get_relative_filepath(self.pf.new_d, template_file) - assert rel_tpl in pst.template_files - - # check the mult2model info - df = pd.read_csv(self.dest_ws / 'mult2model_info.csv') - - # check applying the parameters (in the dest or template ws) - os.chdir(self.dest_ws) - - # first delete the model file that was in the template ws - model_file = df['model_file'].values[mult2model_row] - assert Path(model_file) == Path(dest_file), (f"model_file: {model_file} " - f"differs from dest_file {dest_file}") - os.remove(model_file) - - # pretend that PEST created the input files - # values from dest_file above formed basis for parval in PEST control data - # PEST input file is set up as the org/ version - # apply_list_and_array_pars then takes the org/ version and writes model_file - np.savetxt(pst.input_files[mult2model_row], arr) - - pyemu.helpers.apply_list_and_array_pars(arr_par_file='mult2model_info.csv') - # model files should have been remade by apply_list_and_array_pars - for model_file in df['model_file']: - assert os.path.exists(model_file) - result = np.loadtxt(model_file) - # results should be the same with default multipliers of 1 - # assume details of parameterization are handled by other tests - - # not sure why zone 2 is coming back as invalid (1e30) - zone1 = self.zone_array == 1 - assert np.allclose(result[zone1], parval) - - # revert to original wd - os.chdir(self.tmp_path) - mult2model_row += 1 - except Exception as e: - os.chdir(self.original_wd) - raise e + # make the PEST control file + pst = self.pf.build_pst() + rel_tpl = pyemu.utils.pst_from.get_relative_filepath(self.pf.new_d, template_file) + assert rel_tpl in pst.template_files + + # check the mult2model info + df = pd.read_csv(self.dest_ws / 'mult2model_info.csv') + mult_file = Path(df['mlt_file'].values[mult2model_row]) + + # check applying the parameters (in the dest or template ws) + os.chdir(self.dest_ws) + # first delete the model file in the template ws + model_file = df['model_file'].values[mult2model_row] + os.remove(model_file) + # manually apply a multipler + mult = 4 + if par_type != "pilotpoints": + mult_values = np.loadtxt(mult_file) + mult_values[:] = mult + np.savetxt(mult_file, mult_values) + else: + ppdata = pp_file_to_dataframe(df['pp_file'].values[mult2model_row]) + ppdata['parval1'] = mult + write_pp_file(df['pp_file'].values[mult2model_row], ppdata) + # apply the multiplier + pyemu.helpers.apply_list_and_array_pars(arr_par_file='mult2model_info.csv') + # model files should have been remade by apply_list_and_array_pars + for model_file in df['model_file']: + assert os.path.exists(model_file) + result = np.loadtxt(model_file) + # results should be the same with default multipliers of 1 + # assume details of parameterization are handled by other tests + + # not sure why zone 2 is coming back as invalid (1e30) + zone1 = self.zone_array == 1 + assert np.allclose(result[zone1], self.array_data[zone1] * mult) + + # revert to original wd + os.chdir(self.tmp_path) + mult2model_row += 1 - def test_add_array_parameters_to_file_list(self): + + def test_add_direct_array_parameters(self): """test setting up array parameters with a list of array text files in a subfolder. """ - try: - tag = 'r' - array_file_input = ['external/r0.dat', - 'external/r1.dat', - 'external/r2.dat'] - for file in array_file_input: - shutil.copy(self.array_file, Path(self.dest_ws, file)) - - # single 2D zone array applied to each file in filesnames - self.pf.add_parameters(filenames=array_file_input, par_type='zone', - zone_array=self.zone_array, - par_name_base=tag, # basename for parameters that are set up - pargp=f'{tag}_zone', # Parameter group to assign pars to. - ) - # make the PEST control file - pst = self.pf.build_pst() - # check the mult2model info - df = pd.read_csv(self.dest_ws / 'mult2model_info.csv') - mult_file = Path(df['mlt_file'].values[0]) + tag = 'hk' + par_styles = ['direct', #'direct' + ] + array_files = ['hk_{}_{}.dat', 'external/hk_{}_{}.dat'] + for par_style in par_styles: + mult2model_row = 0 + for j, array_file in enumerate(array_files): + + par_types = {#'constant': 'cn', + 'zone': 'zn', + 'grid': 'gr'} + for i, (par_type, suffix) in enumerate(par_types.items()): + # (re)create the file + dest_file = array_file.format(mult2model_row, suffix) + + # make a new input array file with initial values + arr = np.loadtxt(self.array_file) + parval = 8 + arr[:] = parval + np.savetxt(Path(self.dest_ws, dest_file), arr) + + # add the parameters + par_name_base = f'{tag}_{suffix}' + self.pf.add_parameters(filenames=dest_file, par_type=par_type, + zone_array=self.zone_array, + par_name_base=par_name_base, + pargp=f'{tag}_zone', + par_style=par_style + ) + template_file = (self.pf.tpl_d / f'{Path(dest_file).name}.tpl') + assert template_file.exists() + + # make the PEST control file + pst = self.pf.build_pst() + rel_tpl = pyemu.utils.pst_from.get_relative_filepath(self.pf.new_d, template_file) + assert rel_tpl in pst.template_files + + # check the mult2model info + df = pd.read_csv(self.dest_ws / 'mult2model_info.csv') + + # check applying the parameters (in the dest or template ws) + os.chdir(self.dest_ws) + + # first delete the model file that was in the template ws + model_file = df['model_file'].values[mult2model_row] + assert Path(model_file) == Path(dest_file), (f"model_file: {model_file} " + f"differs from dest_file {dest_file}") + os.remove(model_file) + + # pretend that PEST created the input files + # values from dest_file above formed basis for parval in PEST control data + # PEST input file is set up as the org/ version + # apply_list_and_array_pars then takes the org/ version and writes model_file + np.savetxt(pst.input_files[mult2model_row], arr) + + pyemu.helpers.apply_list_and_array_pars(arr_par_file='mult2model_info.csv') + # model files should have been remade by apply_list_and_array_pars + for model_file in df['model_file']: + assert os.path.exists(model_file) + result = np.loadtxt(model_file) + # results should be the same with default multipliers of 1 + # assume details of parameterization are handled by other tests + + # not sure why zone 2 is coming back as invalid (1e30) + zone1 = self.zone_array == 1 + assert np.allclose(result[zone1], parval) + + # revert to original wd + os.chdir(self.tmp_path) + mult2model_row += 1 - # check applying the parameters (in the dest or template ws) - os.chdir(self.dest_ws) - # first delete the model file in the template ws - for model_file in df['model_file']: - os.remove(model_file) - # manually apply a multipler - mult = 4 - mult_values = np.loadtxt(mult_file) - mult_values[:] = mult - np.savetxt(mult_file, mult_values) - # apply the multiplier - pyemu.helpers.apply_list_and_array_pars(arr_par_file='mult2model_info.csv') - # model files should have been remade by apply_list_and_array_pars - for model_file in df['model_file']: - assert os.path.exists(model_file) - result = np.loadtxt(model_file) - # results should be the same with default multipliers of 1 - # assume details of parameterization are handled by other tests - assert np.allclose(result, self.array_data * mult) - # revert to original wd - os.chdir(self.tmp_path) - except Exception as e: - os.chdir(self.original_wd) - raise e + def test_add_array_parameters_to_file_list(self): + """test setting up array parameters with a list of array text + files in a subfolder. + """ + tag = 'r' + array_file_input = ['external/r0.dat', + 'external/r1.dat', + 'external/r2.dat'] + for file in array_file_input: + shutil.copy(self.array_file, Path(self.dest_ws, file)) + + # single 2D zone array applied to each file in filesnames + self.pf.add_parameters(filenames=array_file_input, par_type='zone', + zone_array=self.zone_array, + par_name_base=tag, # basename for parameters that are set up + pargp=f'{tag}_zone', # Parameter group to assign pars to. + ) + # make the PEST control file + pst = self.pf.build_pst() + # check the mult2model info + df = pd.read_csv(self.dest_ws / 'mult2model_info.csv') + mult_file = Path(df['mlt_file'].values[0]) + + # check applying the parameters (in the dest or template ws) + os.chdir(self.dest_ws) + # first delete the model file in the template ws + for model_file in df['model_file']: + os.remove(model_file) + # manually apply a multipler + mult = 4 + mult_values = np.loadtxt(mult_file) + mult_values[:] = mult + np.savetxt(mult_file, mult_values) + # apply the multiplier + pyemu.helpers.apply_list_and_array_pars(arr_par_file='mult2model_info.csv') + # model files should have been remade by apply_list_and_array_pars + for model_file in df['model_file']: + assert os.path.exists(model_file) + result = np.loadtxt(model_file) + # results should be the same with default multipliers of 1 + # assume details of parameterization are handled by other tests + assert np.allclose(result, self.array_data * mult) + + # revert to original wd + os.chdir(self.tmp_path) + def test_add_array_parameters_alt_inst_str_none_m(self): """Given a list of text file arrays, test setting up @@ -3298,31 +3243,28 @@ def test_add_array_parameters_alt_inst_str_none_m(self): TODO: switch to pytest so that we could simply use one function for this with multiple parameters """ - try: - tag = 'r' - array_file_input = ['external/r0.dat', - 'external/r1.dat', - 'external/r2.dat'] - for file in array_file_input: - shutil.copy(self.array_file, Path(self.dest_ws, file)) - for array_file in array_file_input: - self.pf.add_parameters(filenames=array_file, par_type='zone', - par_style="multiplier", - zone_array=self.zone_array, - par_name_base=tag, # basename for parameters that are set up - pargp=f'{tag}_zone', # Parameter group to assign pars to. - alt_inst_str=None - ) - pst = self.pf.build_pst() - # the parameter data section should have - # only 2 parameters, for zones 1 and 2 - parzones = sorted(pst.parameter_data.zone.astype(float).astype(int).tolist()) - assert parzones == [1, 2] - assert len(pst.template_files) == 3 - assert len(self.pf.mult_files) == 3 - except Exception as e: - os.chdir(self.original_wd) - raise e + tag = 'r' + array_file_input = ['external/r0.dat', + 'external/r1.dat', + 'external/r2.dat'] + for file in array_file_input: + shutil.copy(self.array_file, Path(self.dest_ws, file)) + for array_file in array_file_input: + self.pf.add_parameters(filenames=array_file, par_type='zone', + par_style="multiplier", + zone_array=self.zone_array, + par_name_base=tag, # basename for parameters that are set up + pargp=f'{tag}_zone', # Parameter group to assign pars to. + alt_inst_str=None + ) + pst = self.pf.build_pst() + # the parameter data section should have + # only 2 parameters, for zones 1 and 2 + parzones = sorted(pst.parameter_data.zone.astype(float).astype(int).tolist()) + assert parzones == [1, 2] + assert len(pst.template_files) == 3 + assert len(self.pf.mult_files) == 3 + def test_add_array_parameters_alt_inst_str_0_d(self): """Given a list of text file arrays, test setting up @@ -3334,31 +3276,27 @@ def test_add_array_parameters_alt_inst_str_0_d(self): Test alt_inst_str="" and par_style="direct" """ - try: - tag = 'r' - array_file_input = ['external/r0.dat', - 'external/r1.dat', - 'external/r2.dat'] - for file in array_file_input: - shutil.copy(self.array_file, Path(self.dest_ws, file)) - # test both None and "" for alt_inst_str - for array_file in array_file_input: - self.pf.add_parameters(filenames=array_file, par_type='zone', - par_style="direct", - zone_array=self.zone_array, - par_name_base=tag, # basename for parameters that are set up - pargp=f'{tag}_zone', # Parameter group to assign pars to. - alt_inst_str="" - ) - pst = self.pf.build_pst() - # the parameter data section should have - # only 2 parameters, for zones 1 and 2 - parzones = sorted(pst.parameter_data.zone.astype(float).astype(int).tolist()) - assert parzones == [1, 2] - assert len(pst.template_files) == 3 - except Exception as e: - os.chdir(self.original_wd) - raise e + tag = 'r' + array_file_input = ['external/r0.dat', + 'external/r1.dat', + 'external/r2.dat'] + for file in array_file_input: + shutil.copy(self.array_file, Path(self.dest_ws, file)) + # test both None and "" for alt_inst_str + for array_file in array_file_input: + self.pf.add_parameters(filenames=array_file, par_type='zone', + par_style="direct", + zone_array=self.zone_array, + par_name_base=tag, # basename for parameters that are set up + pargp=f'{tag}_zone', # Parameter group to assign pars to. + alt_inst_str="" + ) + pst = self.pf.build_pst() + # the parameter data section should have + # only 2 parameters, for zones 1 and 2 + parzones = sorted(pst.parameter_data.zone.astype(float).astype(int).tolist()) + assert parzones == [1, 2] + assert len(pst.template_files) == 3 def test_get_filepath(): @@ -3517,133 +3455,130 @@ def mf6_freyberg_arr_obs_and_headerless_test(tmp_path): except: return + + org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) - bd = Path.cwd() + os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) - m = sim.get_model("freyberg6") - sim.set_all_data_external(check_data=False) - sim.write_simulation() + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + m = sim.get_model("freyberg6") + sim.set_all_data_external(check_data=False) + sim.write_simulation() - # to by pass the issues with flopy - # shutil.copytree(org_model_ws,tmp_model_ws) - # sim = flopy.mf6.MFSimulation.load(sim_ws=org_model_ws) - # m = sim.get_model("freyberg6") + # to by pass the issues with flopy + # shutil.copytree(org_model_ws,tmp_model_ws) + # sim = flopy.mf6.MFSimulation.load(sim_ws=org_model_ws) + # m = sim.get_model("freyberg6") - # SETUP pest stuff... - os_utils.run("{0} ".format(mf6_exe_path), cwd=tmp_model_ws) + # SETUP pest stuff... + os_utils.run("{0} ".format(mf6_exe_path), cwd=tmp_model_ws) - template_ws = "new_temp" - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - # sr0 = m.sr - # sr = pyemu.helpers.SpatialReference.from_namfile( - # os.path.join(tmp_model_ws, "freyberg6.nam"), - # delr=m.dis.delr.array, delc=m.dis.delc.array) - sr = m.modelgrid - # set up PstFrom object - pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, - remove_existing=True, - longnames=True, spatial_reference=sr, - zero_based=False, start_datetime="1-1-2018") - - list_file = "freyberg6.wel_stress_period_data_1.txt" - df = pd.read_csv(os.path.join(template_ws, list_file), header=None, sep=r'\s+') - df.loc[:,4] = 4 - df.loc[:,5] = 5 - df.to_csv(os.path.join(template_ws,list_file),sep=" ",index=False,header=False) - pf.add_observations(list_file, index_cols=[0, 1, 2], use_cols=[3,5], ofile_skip=0, includes_header=False, - prefix="welobs") - - with open(os.path.join(template_ws,"badlistcall.txt"), "w") as fp: - fp.write("this is actually a header\n") - fp.write("entry 0 10 100.4 2\n") - fp.write("entry 1 10 2.4 5.0") - pf.add_observations("badlistcall.txt", index_cols=[0, 1], use_cols=[3, 4], - ofile_skip=0, includes_header=False, - prefix="badlistcall") + template_ws = "new_temp" + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + # sr0 = m.sr + # sr = pyemu.helpers.SpatialReference.from_namfile( + # os.path.join(tmp_model_ws, "freyberg6.nam"), + # delr=m.dis.delr.array, delc=m.dis.delc.array) + sr = m.modelgrid + # set up PstFrom object + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + remove_existing=True, + longnames=True, spatial_reference=sr, + zero_based=False, start_datetime="1-1-2018") + list_file = "freyberg6.wel_stress_period_data_1.txt" + df = pd.read_csv(os.path.join(template_ws, list_file), header=None, sep=r'\s+') + df.loc[:,4] = 4 + df.loc[:,5] = 5 + df.to_csv(os.path.join(template_ws,list_file),sep=" ",index=False,header=False) + pf.add_observations(list_file, index_cols=[0, 1, 2], use_cols=[3,5], ofile_skip=0, includes_header=False, + prefix="welobs") + + with open(os.path.join(template_ws,"badlistcall.txt"), "w") as fp: + fp.write("this is actually a header\n") + fp.write("entry 0 10 100.4 2\n") + fp.write("entry 1 10 2.4 5.0") + pf.add_observations("badlistcall.txt", index_cols=[0, 1], use_cols=[3, 4], + ofile_skip=0, includes_header=False, + prefix="badlistcall") + + + v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) + gr_gs = pyemu.geostats.GeoStruct(variograms=v) + rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60)) + pf.extra_py_imports.append('flopy') + ib = m.dis.idomain[0].array + tags = {"npf_k_": [0.1, 10.]} + dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") + print(dts) + arr_dict = {} + for tag, bnd in tags.items(): + lb, ub = bnd[0], bnd[1] + arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] + for arr_file in arr_files: + #pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base=arr_file.split('.')[1] + "_gr", + # pargp=arr_file.split('.')[1] + "_gr", zone_array=ib, upper_bound=ub, lower_bound=lb, + # geostruct=gr_gs) + pf.add_parameters(filenames=arr_file, par_type="constant", par_name_base=arr_file.split('.')[1] + "_cn", + pargp=arr_file.split('.')[1] + "_cn", zone_array=ib, upper_bound=ub, lower_bound=lb, + transform="fixed") + pf.add_parameters(filenames=arr_file, par_type="constant", par_name_base=arr_file.split('.')[1] + "_cn", + pargp=arr_file.split('.')[1] + "_cn", zone_array=ib, upper_bound=ub, lower_bound=lb, + transform="log") - v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) - gr_gs = pyemu.geostats.GeoStruct(variograms=v) - rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60)) - pf.extra_py_imports.append('flopy') - ib = m.dis.idomain[0].array - tags = {"npf_k_": [0.1, 10.]} - dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") - print(dts) - arr_dict = {} - for tag, bnd in tags.items(): - lb, ub = bnd[0], bnd[1] - arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] - for arr_file in arr_files: - #pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base=arr_file.split('.')[1] + "_gr", - # pargp=arr_file.split('.')[1] + "_gr", zone_array=ib, upper_bound=ub, lower_bound=lb, - # geostruct=gr_gs) - pf.add_parameters(filenames=arr_file, par_type="constant", par_name_base=arr_file.split('.')[1] + "_cn", - pargp=arr_file.split('.')[1] + "_cn", zone_array=ib, upper_bound=ub, lower_bound=lb, - transform="fixed") - pf.add_parameters(filenames=arr_file, par_type="constant", par_name_base=arr_file.split('.')[1] + "_cn", - pargp=arr_file.split('.')[1] + "_cn", zone_array=ib, upper_bound=ub, lower_bound=lb, - transform="log") - - - pf.add_observations(arr_file,zone_array=ib) - arr_dict[arr_file] = np.loadtxt(pf.new_d / arr_file) - - # add model run command - pf.mod_sys_cmds.append("mf6") - print(pf.mult_files) - print(pf.org_files) - - # build pest - pst = pf.build_pst('freyberg.pst') - pe = pf.draw(100,use_specsim=True) - cov = pf.build_prior() - scnames = set(cov.row_names) - print(pst.npar_adj,pst.npar,pe.shape) - par = pst.parameter_data - fpar = set(par.loc[par.partrans=="fixed","parnme"].tolist()) - spe = set(list(pe.columns)) - assert len(fpar.intersection(spe)) == 0,str(fpar.intersection(spe)) - assert len(fpar.intersection(scnames)) == 0, str(fpar.intersection(scnames)) - pst.try_parse_name_metadata() - obs = pst.observation_data - for fname,arr in arr_dict.items(): - - fobs = obs.loc[obs.obsnme.str.contains(Path(fname).stem), :] - #print(fobs) - fobs = fobs.astype({c: int for c in ['i', 'j']}) - - pval = fobs.loc[fobs.apply(lambda x: x.i==3 and x.j==1,axis=1),"obsval"] - assert len(pval) == 1 - pval = pval.iloc[0] - aval = arr[3,1] - print(fname,pval,aval) - assert pval == aval,"{0},{1},{2}".format(fname,pval,aval) - - df = pd.read_csv(os.path.join(template_ws, list_file), - header=None, sep=r'\s+') - print(df) - wobs = obs.loc[obs.obsnme.str.contains("welobs"),:] - print(wobs) - fvals = df.iloc[:,3] - pvals = wobs.loc[:,"obsval"].iloc[:df.shape[0]] - d = fvals.values - pvals.values - print(d) - assert d.sum() == 0 - fvals = df.iloc[:, 5] - pvals = wobs.loc[:, "obsval"].iloc[df.shape[0]:] - d = fvals.values - pvals.values - print(d) - assert d.sum() == 0 - except Exception as e: - os.chdir(bd) - raise e - os.chdir(bd) + + pf.add_observations(arr_file,zone_array=ib) + arr_dict[arr_file] = np.loadtxt(pf.new_d / arr_file) + + # add model run command + pf.mod_sys_cmds.append("mf6") + print(pf.mult_files) + print(pf.org_files) + + # build pest + pst = pf.build_pst('freyberg.pst') + pe = pf.draw(100,use_specsim=True) + cov = pf.build_prior() + scnames = set(cov.row_names) + print(pst.npar_adj,pst.npar,pe.shape) + par = pst.parameter_data + fpar = set(par.loc[par.partrans=="fixed","parnme"].tolist()) + spe = set(list(pe.columns)) + assert len(fpar.intersection(spe)) == 0,str(fpar.intersection(spe)) + assert len(fpar.intersection(scnames)) == 0, str(fpar.intersection(scnames)) + pst.try_parse_name_metadata() + obs = pst.observation_data + for fname,arr in arr_dict.items(): + + fobs = obs.loc[obs.obsnme.str.contains(Path(fname).stem), :] + #print(fobs) + fobs = fobs.astype({c: int for c in ['i', 'j']}) + + pval = fobs.loc[fobs.apply(lambda x: x.i==3 and x.j==1,axis=1),"obsval"] + assert len(pval) == 1 + pval = pval.iloc[0] + aval = arr[3,1] + print(fname,pval,aval) + assert pval == aval,"{0},{1},{2}".format(fname,pval,aval) + + df = pd.read_csv(os.path.join(template_ws, list_file), + header=None, sep=r'\s+') + print(df) + wobs = obs.loc[obs.obsnme.str.contains("welobs"),:] + print(wobs) + fvals = df.iloc[:,3] + pvals = wobs.loc[:,"obsval"].iloc[:df.shape[0]] + d = fvals.values - pvals.values + print(d) + assert d.sum() == 0 + fvals = df.iloc[:, 5] + pvals = wobs.loc[:, "obsval"].iloc[df.shape[0]:] + d = fvals.values - pvals.values + print(d) + assert d.sum() == 0 def mf6_freyberg_pp_locs_test(tmp_path): @@ -3652,131 +3587,136 @@ def mf6_freyberg_pp_locs_test(tmp_path): pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return - - org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') - tmp_model_ws = setup_tmp(org_model_ws, tmp_path) + #try: + import flopy + #except: + # return - bd = Path.cwd() - os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) - m = sim.get_model("freyberg6") - sim.set_all_data_external(check_data=False) - sim.write_simulation() + # import sys + # sys.path.insert(0,os.path.join("..","..","pypestutils")) + import pypestutils as ppu - # SETUP pest stuff... - os_utils.run("{0} ".format(mf6_exe_path), cwd=tmp_model_ws) + pf, sim = setup_freyberg_mf6(tmp_path) + m = sim.get_model() + template_ws = pf.new_d + # org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') + # tmp_model_ws = setup_tmp(org_model_ws, tmp_path) - template_ws = "new_temp" - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - sr = m.modelgrid + # bd = Path.cwd() + # os.chdir(tmp_path) + # try: + # tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + # sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + # m = sim.get_model("freyberg6") + # sim.set_all_data_external(check_data=False) + # sim.write_simulation() + # + # SETUP pest stuff... + # os_utils.run("{0} ".format(mf6_exe_path), cwd=tmp_model_ws) + # + # template_ws = "new_temp" + # if os.path.exists(template_ws): + # shutil.rmtree(template_ws) + # sr = m.modelgrid # set up PstFrom object - pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, - remove_existing=True, - longnames=True, spatial_reference=sr, - zero_based=False, start_datetime="1-1-2018", - chunk_len=1) + # pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + # remove_existing=True, + # longnames=True, spatial_reference=sr, + # zero_based=False, start_datetime="1-1-2018", + # chunk_len=1) # pf.post_py_cmds.append("generic_function()") - df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) - pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", use_cols=list(df.columns.values)) - v = pyemu.geostats.ExpVario(contribution=1.0, a=5000) - pp_gs = pyemu.geostats.GeoStruct(variograms=v) - pf.extra_py_imports.append('flopy') - ib = m.dis.idomain[0].array - tags = {"npf_k_": [0.1, 10.]}#, "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1], - # "rch_recharge": [.5, 1.5]} - dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") - print(dts) - - xmn = m.modelgrid.xvertices.min() - xmx = m.modelgrid.xvertices.max() - ymn = m.modelgrid.yvertices.min() - ymx = m.modelgrid.yvertices.max() - - numpp = 20 - xvals = np.random.uniform(xmn,xmx,numpp) - yvals = np.random.uniform(ymn, ymx, numpp) - pp_locs = pd.DataFrame({"x":xvals,"y":yvals}) - pp_locs.loc[:,"zone"] = 1 - pp_locs.loc[:,"name"] = ["pp_{0}".format(i) for i in range(numpp)] - pp_locs.loc[:,"parval1"] = 1.0 - - pyemu.pp_utils.write_pp_shapfile(pp_locs,os.path.join(template_ws,"pp_locs.shp")) - df = pyemu.pp_utils.pilot_points_from_shapefile(os.path.join(template_ws,"pp_locs.shp")) - - #pp_locs = pyemu.pp_utils.setup_pilotpoints_grid(sr=sr,prefix_dict={0:"pps_1"}) - #pp_locs = pp_locs.loc[:,["name","x","y","zone","parval1"]] - pp_locs.to_csv(os.path.join(template_ws,"pp.csv")) - pyemu.pp_utils.write_pp_file(os.path.join(template_ws,"pp_file.dat"),pp_locs) - pp_container = ["pp_file.dat","pp.csv","pp_locs.shp"] - - for tag, bnd in tags.items(): - lb, ub = bnd[0], bnd[1] - arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] - if "rch" in tag: - pass - # pf.add_parameters(filenames=arr_files, par_type="grid", par_name_base="rch_gr", - # pargp="rch_gr", zone_array=ib, upper_bound=ub, lower_bound=lb, - # geostruct=gr_gs) - # for arr_file in arr_files: - # kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 - # pf.add_parameters(filenames=arr_file, par_type="constant", par_name_base=arr_file.split('.')[1] + "_cn", - # pargp="rch_const", zone_array=ib, upper_bound=ub, lower_bound=lb, - # geostruct=rch_temporal_gs, - # datetime=dts[kper]) - else: - for i,arr_file in enumerate(arr_files): - if i < len(pp_container): - pp_opt = pp_container[i] - else: - pp_opt = pp_locs - pf.add_parameters(filenames=arr_file, par_type="pilotpoints", - par_name_base=arr_file.split('.')[1] + "_pp", - pargp=arr_file.split('.')[1] + "_pp", zone_array=ib, - upper_bound=ub, lower_bound=lb,pp_space=pp_opt) - - # add model run command - pf.mod_sys_cmds.append("mf6") - print(pf.mult_files) - print(pf.org_files) - - # build pest - pst = pf.build_pst('freyberg.pst') - - num_reals = 10 - pe = pf.draw(num_reals, use_specsim=True) - pe.to_binary(os.path.join(template_ws, "prior.jcb")) - - pst.parameter_data.loc[:,"partrans"] = "fixed" - pst.parameter_data.loc[::10, "partrans"] = "log" - pst.control_data.noptmax = -1 - pst.write(os.path.join(template_ws,"freyberg.pst")) - - #pyemu.os_utils.run("{0} freyberg.pst".format("pestpp-glm"),cwd=template_ws) - m_d = "master_glm" - port = _get_port() - print(f"Running ies on port: {port}") - pyemu.os_utils.start_workers(template_ws,pp_exe_path,"freyberg.pst",num_workers=5, - worker_root=tmp_path, - master_dir=m_d, port=port) - - sen_df = pd.read_csv(os.path.join(m_d,"freyberg.isen"),index_col=0).loc[:,pst.adj_par_names] - print(sen_df.T) - mn = sen_df.values.min() - print(mn) - assert mn > 0.0 - except Exception as e: - os.chdir(bd) - raise e - os.chdir(bd) + # df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) + # pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", use_cols=list(df.columns.values)) + v = pyemu.geostats.ExpVario(contribution=1.0, a=5000) + pp_gs = pyemu.geostats.GeoStruct(variograms=v) + pf.extra_py_imports.append('flopy') + ib = m.dis.idomain[0].array + tags = {"npf_k_": [0.1, 10.]}#, "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1], + # "rch_recharge": [.5, 1.5]} + dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") + print(dts) + + xmn = m.modelgrid.xvertices.min() + xmx = m.modelgrid.xvertices.max() + ymn = m.modelgrid.yvertices.min() + ymx = m.modelgrid.yvertices.max() + + numpp = 20 + xvals = np.random.uniform(xmn,xmx,numpp) + yvals = np.random.uniform(ymn, ymx, numpp) + pp_locs = pd.DataFrame({"x":xvals,"y":yvals}) + pp_locs.loc[:,"zone"] = 1 + pp_locs.loc[:,"name"] = ["pp_{0}".format(i) for i in range(numpp)] + pp_locs.loc[:,"parval1"] = 1.0 + + pyemu.pp_utils.write_pp_shapfile(pp_locs,os.path.join(template_ws,"pp_locs.shp")) + df = pyemu.pp_utils.pilot_points_from_shapefile(os.path.join(template_ws,"pp_locs.shp")) + + #pp_locs = pyemu.pp_utils.setup_pilotpoints_grid(sr=sr,prefix_dict={0:"pps_1"}) + #pp_locs = pp_locs.loc[:,["name","x","y","zone","parval1"]] + pp_locs.to_csv(os.path.join(template_ws,"pp.csv")) + pyemu.pp_utils.write_pp_file(os.path.join(template_ws,"pp_file.dat"),pp_locs) + pp_container = ["pp_file.dat","pp.csv","pp_locs.shp"] + + for tag, bnd in tags.items(): + lb, ub = bnd[0], bnd[1] + arr_files = [f for f in os.listdir(template_ws) if tag in f and f.endswith(".txt")] + if "rch" in tag: + pass + # pf.add_parameters(filenames=arr_files, par_type="grid", par_name_base="rch_gr", + # pargp="rch_gr", zone_array=ib, upper_bound=ub, lower_bound=lb, + # geostruct=gr_gs) + # for arr_file in arr_files: + # kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 + # pf.add_parameters(filenames=arr_file, par_type="constant", par_name_base=arr_file.split('.')[1] + "_cn", + # pargp="rch_const", zone_array=ib, upper_bound=ub, lower_bound=lb, + # geostruct=rch_temporal_gs, + # datetime=dts[kper]) + else: + for i,arr_file in enumerate(arr_files): + if i < len(pp_container): + pp_opt = pp_container[i] + else: + pp_opt = pp_locs + pf.add_parameters(filenames=arr_file, par_type="pilotpoints", + par_name_base=arr_file.split('.')[1] + "_pp", + pargp=arr_file.split('.')[1] + "_pp", zone_array=ib, + upper_bound=ub, lower_bound=lb,pp_space=pp_opt,apply_order=i) + + + # add model run command + pf.mod_sys_cmds.append("mf6") + print(pf.mult_files) + print(pf.org_files) + + # build pest + pst = pf.build_pst('freyberg.pst') + + num_reals = 10 + pe = pf.draw(num_reals, use_specsim=True) + pe.to_binary(os.path.join(template_ws, "prior.jcb")) + pst.parameter_data.loc[:,"partrans"] = "fixed" + pst.parameter_data.loc[::10, "partrans"] = "log" + pst.control_data.noptmax = -1 + pst.write(os.path.join(template_ws,"freyberg.pst")) + + #pyemu.os_utils.run("{0} freyberg.pst".format("pestpp-glm"),cwd=template_ws) + m_d = "master_glm" + port = _get_port() + print(f"Running ies on port: {port}") + print(pp_exe_path) + pyemu.os_utils.start_workers(template_ws,pp_exe_path,"freyberg.pst", + num_workers=5, + worker_root=tmp_path, + master_dir=m_d, port=port) + + sen_df = pd.read_csv(os.path.join(m_d,"freyberg.isen"),index_col=0).loc[:,pst.adj_par_names] + print(sen_df.T) + mn = sen_df.values.min() + print(mn) + assert mn > 0.0 def usg_freyberg_test(tmp_path): @@ -3792,207 +3732,201 @@ def usg_freyberg_test(tmp_path): org_model_ws = os.path.join('..', 'examples', 'freyberg_usg') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) - bd = Path.cwd() os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - # flopy is not liking the rch package in unstruct, so allow it to fail and keep going... - m = flopy.mfusg.MfUsg.load("freyberg.usg.nam", model_ws=tmp_model_ws, - verbose=True, forgive=True, check=False) - - #convert to all open/close - m.external_path = "." - m.write_input() - - nam_file = os.path.join(tmp_model_ws,"freyberg.usg.nam") - - #make sure the model runs in the new dir with all external formats - pyemu.os_utils.run("{0} freyberg.usg.nam".format(usg_exe_path), cwd=tmp_model_ws) - - # for usg, we need to do some trickery to support the unstructured by layers concept - # this is just for array-based parameters, list-based pars are g2g because they have an index - gsf = pyemu.gw_utils.GsfReader(os.path.join(tmp_model_ws,"freyberg.usg.gsf")) - df = gsf.get_node_data() - df.loc[:,"xy"] = df.apply(lambda x: (x.x, x.y),axis=1) - # these need to be zero based since they are with zero-based array indices later... - df.loc[:,"node"] -= 1 - # process each layer - layers = df.layer.unique() - layers.sort() - sr_dict_by_layer = {} - for layer in layers: - df_lay = df.loc[df.layer==layer,:].copy() - df_lay.sort_values(by="node") - #substract off the min node number so that each layers node dict starts at zero - df_lay.loc[:,"node"] = df_lay.node - df_lay.node.min() - print(df_lay) - srd = {n:xy for n,xy in zip(df_lay.node.values,df_lay.xy.values)} - sr_dict_by_layer[layer] = srd - - # gen up a fake zone array - zone_array_k0 = np.ones((1, len(sr_dict_by_layer[1]))) - zone_array_k0[:, 200:420] = 2 - zone_array_k0[:, 600:1000] = 3 - - zone_array_k2 = np.ones((1, len(sr_dict_by_layer[3]))) - zone_array_k2[:, 200:420] = 2 - zone_array_k2[:, 500:1000:3] = 3 - zone_array_k2[:,:100] = 4 - - #gen up some fake pp locs - np.random.seed(pyemu.en.SEED) - num_pp = 20 - data = {"name":[],"x":[],"y":[],"zone":[]} - visited = set() - for i in range(num_pp): - while True: - idx = np.random.randint(0,len(sr_dict_by_layer[1])) - if idx not in visited: - break - x,y = sr_dict_by_layer[1][idx] - data["name"].append("pp_{0}".format(i)) - data["x"].append(x) - data["y"].append(y) - data["zone"].append(zone_array_k2[0,idx]) - visited.add(idx) - # harded coded to get a zone 3 pp - idx = 500 - assert zone_array_k2[0,idx] == 3,zone_array_k2[0,idx] - - x, y = sr_dict_by_layer[1][idx] - data["name"].append("pp_{0}".format(i+1)) + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + # flopy is not liking the rch package in unstruct, so allow it to fail and keep going... + m = flopy.mfusg.MfUsg.load("freyberg.usg.nam", model_ws=tmp_model_ws, + verbose=True, forgive=True, check=False) + + #convert to all open/close + m.external_path = "." + m.write_input() + + nam_file = os.path.join(tmp_model_ws,"freyberg.usg.nam") + + #make sure the model runs in the new dir with all external formats + pyemu.os_utils.run("{0} freyberg.usg.nam".format(usg_exe_path), cwd=tmp_model_ws) + + # for usg, we need to do some trickery to support the unstructured by layers concept + # this is just for array-based parameters, list-based pars are g2g because they have an index + gsf = pyemu.gw_utils.GsfReader(os.path.join(tmp_model_ws,"freyberg.usg.gsf")) + df = gsf.get_node_data() + df.loc[:,"xy"] = df.apply(lambda x: (x.x, x.y),axis=1) + # these need to be zero based since they are with zero-based array indices later... + df.loc[:,"node"] -= 1 + # process each layer + layers = df.layer.unique() + layers.sort() + sr_dict_by_layer = {} + for layer in layers: + df_lay = df.loc[df.layer==layer,:].copy() + df_lay.sort_values(by="node") + #substract off the min node number so that each layers node dict starts at zero + df_lay.loc[:,"node"] = df_lay.node - df_lay.node.min() + print(df_lay) + srd = {n:xy for n,xy in zip(df_lay.node.values,df_lay.xy.values)} + sr_dict_by_layer[layer] = srd + + # gen up a fake zone array + zone_array_k0 = np.ones((1, len(sr_dict_by_layer[1]))) + zone_array_k0[:, 200:420] = 2 + zone_array_k0[:, 600:1000] = 3 + + zone_array_k2 = np.ones((1, len(sr_dict_by_layer[3]))) + zone_array_k2[:, 200:420] = 2 + zone_array_k2[:, 500:1000:3] = 3 + zone_array_k2[:,:100] = 4 + + #gen up some fake pp locs + np.random.seed(pyemu.en.SEED) + num_pp = 20 + data = {"name":[],"x":[],"y":[],"zone":[]} + visited = set() + for i in range(num_pp): + while True: + idx = np.random.randint(0,len(sr_dict_by_layer[1])) + if idx not in visited: + break + x,y = sr_dict_by_layer[1][idx] + data["name"].append("pp_{0}".format(i)) data["x"].append(x) data["y"].append(y) - data["zone"].append(zone_array_k2[0, idx]) - pp_df = pd.DataFrame(data=data,index=data["name"]) - - # a geostruct that describes spatial continuity for properties - # this is used for all props and for both grid and pilot point - # pars cause Im lazy... - v = pyemu.geostats.ExpVario(contribution=1.0,a=500) - gs = pyemu.geostats.GeoStruct(variograms=v) - - # we pass the full listing of node coord info to the constructor for use - # with list-type parameters - template_d = Path(tmp_path, "template") - pf = pyemu.utils.PstFrom(tmp_model_ws,template_d,longnames=True,remove_existing=True, - zero_based=False,spatial_reference=gsf.get_node_coordinates(zero_based=True)) - - pf.add_parameters("hk_Layer_3.ref", par_type="pilotpoints", - par_name_base="hk3_pp", pp_space=pp_df, - geostruct=gs, spatial_reference=sr_dict_by_layer[3], - upper_bound=2.0, lower_bound=0.5, - zone_array=zone_array_k2) - - # we pass layer specific sr dict for each "array" type that is spatially distributed - pf.add_parameters("hk_Layer_1.ref",par_type="grid",par_name_base="hk1_Gr",geostruct=gs, - spatial_reference=sr_dict_by_layer[1], - upper_bound=2.0,lower_bound=0.5) - pf.add_parameters("sy_Layer_1.ref", par_type="zone", par_name_base="sy1_zn",zone_array=zone_array_k0, - upper_bound=1.5,lower_bound=0.5,ult_ubound=0.35) - - - - # add a multiplier par for each well for each stress period - wel_files = [f for f in os.listdir(tmp_model_ws) if f.lower().startswith("wel_") and f.lower().endswith(".dat")] - for wel_file in wel_files: - pf.add_parameters(wel_file,par_type="grid",par_name_base=wel_file.lower().split('.')[0],index_cols=[0],use_cols=[1], - geostruct=gs,lower_bound=0.5,upper_bound=1.5) - - # add pest "observations" for each active node for each stress period - hds_runline, df = pyemu.gw_utils.setup_hds_obs( - os.path.join(pf.new_d, "freyberg.usg.hds"), kperk_pairs=None, - prefix="hds", include_path=False, text="headu", skip=-1.0e+30) - pyemu.gw_utils.apply_hds_obs(os.path.join(pf.new_d, "freyberg.usg.hds"), - precision='single', text='headu') - pf.add_observations_from_ins(os.path.join(pf.new_d, "freyberg.usg.hds.dat.ins"), pst_path=".") - pf.post_py_cmds.append(hds_runline) - - # the command the run the model - pf.mod_sys_cmds.append("{0} freyberg.usg.nam".format(usg_exe_path)) - - #build the control file and draw the prior par ensemble - pf.build_pst() - pst = pf.pst - par = pst.parameter_data - - gr_hk_pars = par.loc[par.parnme.str.contains("hk1_gr"),"parnme"] - pf.pst.parameter_data.loc[gr_hk_pars,"parubnd"] = np.random.random(gr_hk_pars.shape[0]) * 5 - pf.pst.parameter_data.loc[gr_hk_pars, "parlbnd"] = np.random.random(gr_hk_pars.shape[0]) * 0.2 - pe = pf.draw(num_reals=100) - pe.enforce() - pe.to_csv(os.path.join(pf.new_d,"prior.csv")) - cov = pf.build_prior(filename=None) - #make sure the prior cov has off diagonals - cov = pf.build_prior(sigma_range=6) - covx = cov.x.copy() - covx[np.abs(covx)>1.0e-7] = 1.0 - assert covx.sum() > pf.pst.npar_adj + 1 - dcov = pyemu.Cov.from_parameter_data(pf.pst,sigma_range=6) - dcov = dcov.get(cov.row_names) - diag = np.diag(cov.x) - diff = np.abs(diag.flatten() - dcov.x.flatten()) - print(diag) - print(dcov.x) - print(diff) - print(diff.max()) - assert np.isclose(diff.max(), 0), diff.close() - - # test that the arr hds obs process is working - pyemu.gw_utils.apply_hds_obs(os.path.join(pf.new_d, "freyberg.usg.hds"), - precision='single', text='headu') - - # run the full process once using the initial par values in the control file - # since we are using only multipliers, the initial values are all 1's so - # the phi should be pretty close to zero - pf.pst.control_data.noptmax = 0 - pf.pst.write(os.path.join(pf.new_d,"freyberg.usg.pst"),version=2) - pyemu.os_utils.run("{0} freyberg.usg.pst".format(ies_exe_path),cwd=pf.new_d) - pst = pyemu.Pst(os.path.join(pf.new_d,"freyberg.usg.pst")) - assert np.isclose(pst.phi, 0), pst.phi - - #make sure the processed model input arrays are veru similar to the org arrays (again 1s for mults) - for arr_file in ["hk_Layer_1.ref","hk_Layer_3.ref"]: - in_arr = np.loadtxt(os.path.join(pf.new_d,arr_file)) - org_arr = np.loadtxt(os.path.join(pf.new_d,"org",arr_file)) - d = np.abs(in_arr - org_arr) - print(d.sum()) - assert np.isclose(d.sum(), 0), arr_file - - - # now run a random realization from the prior par en and make sure things have changed - pst.parameter_data.loc[pe.columns,"parval1"] = pe.iloc[0,:].values - pst.write(os.path.join(pf.new_d, "freyberg.usg.pst"), version=2) - pyemu.os_utils.run("{0} freyberg.usg.pst".format(ies_exe_path), cwd=pf.new_d) - - pst = pyemu.Pst(os.path.join(pf.new_d, "freyberg.usg.pst")) - assert pst.phi > 1.0e-3, pst.phi - - for arr_file in ["hk_Layer_1.ref", "hk_Layer_3.ref"]: - in_arr = np.loadtxt(os.path.join(pf.new_d, arr_file)) - org_arr = np.loadtxt(os.path.join(pf.new_d, "org", arr_file)) - d = np.abs(in_arr - org_arr) - print(d.sum()) - assert d.sum() > 1.0e-3, arr_file - - # check that the pilot point process is respecting the zone array - par = pst.parameter_data - pp_par = par.loc[par.parnme.str.contains("pp"),:] - pst.parameter_data.loc[pp_par.parnme,"parval1"] = pp_par.zone.apply(np.float64) - pst.control_data.noptmax = 0 - pst.write(os.path.join(pf.new_d,"freyberg.usg.pst"),version=2) - #pst.write_input_files(pf.new_d) - pyemu.os_utils.run("{0} freyberg.usg.pst".format(ies_exe_path), cwd=pf.new_d) - arr = np.loadtxt(os.path.join(pf.new_d,"mult","hk3_pp_inst0_pilotpoints.csv")) - arr[zone_array_k2[0,:]==0] = 0 - d = np.abs(arr - zone_array_k2) - print(d) + data["zone"].append(zone_array_k2[0,idx]) + visited.add(idx) + # harded coded to get a zone 3 pp + idx = 500 + assert zone_array_k2[0,idx] == 3,zone_array_k2[0,idx] + + x, y = sr_dict_by_layer[1][idx] + data["name"].append("pp_{0}".format(i+1)) + data["x"].append(x) + data["y"].append(y) + data["zone"].append(zone_array_k2[0, idx]) + pp_df = pd.DataFrame(data=data,index=data["name"]) + + # a geostruct that describes spatial continuity for properties + # this is used for all props and for both grid and pilot point + # pars cause Im lazy... + v = pyemu.geostats.ExpVario(contribution=1.0,a=500) + gs = pyemu.geostats.GeoStruct(variograms=v) + + # we pass the full listing of node coord info to the constructor for use + # with list-type parameters + template_d = Path(tmp_path, "template") + pf = pyemu.utils.PstFrom(tmp_model_ws,template_d,longnames=True,remove_existing=True, + zero_based=False,spatial_reference=gsf.get_node_coordinates(zero_based=True)) + + pf.add_parameters("hk_Layer_3.ref", par_type="pilotpoints", + par_name_base="hk3_pp", pp_space=pp_df, + geostruct=gs, spatial_reference=sr_dict_by_layer[3], + upper_bound=2.0, lower_bound=0.5, + zone_array=zone_array_k2,apply_order=10) + + # we pass layer specific sr dict for each "array" type that is spatially distributed + pf.add_parameters("hk_Layer_1.ref",par_type="grid",par_name_base="hk1_Gr",geostruct=gs, + spatial_reference=sr_dict_by_layer[1], + upper_bound=2.0,lower_bound=0.5,apply_order=0) + pf.add_parameters("sy_Layer_1.ref", par_type="zone", par_name_base="sy1_zn",zone_array=zone_array_k0, + upper_bound=1.5,lower_bound=0.5,ult_ubound=0.35,apply_order=100) + + + + # add a multiplier par for each well for each stress period + wel_files = [f for f in os.listdir(tmp_model_ws) if f.lower().startswith("wel_") and f.lower().endswith(".dat")] + for wel_file in wel_files: + pf.add_parameters(wel_file,par_type="grid",par_name_base=wel_file.lower().split('.')[0],index_cols=[0],use_cols=[1], + geostruct=gs,lower_bound=0.5,upper_bound=1.5,apply_order=10) + + # add pest "observations" for each active node for each stress period + hds_runline, df = pyemu.gw_utils.setup_hds_obs( + os.path.join(pf.new_d, "freyberg.usg.hds"), kperk_pairs=None, + prefix="hds", include_path=False, text="headu", skip=-1.0e+30) + pyemu.gw_utils.apply_hds_obs(os.path.join(pf.new_d, "freyberg.usg.hds"), + precision='single', text='headu') + pf.add_observations_from_ins(os.path.join(pf.new_d, "freyberg.usg.hds.dat.ins"), pst_path=".") + pf.post_py_cmds.append(hds_runline) + + # the command the run the model + pf.mod_sys_cmds.append("{0} freyberg.usg.nam".format(usg_exe_path)) + + #build the control file and draw the prior par ensemble + pf.build_pst() + pst = pf.pst + par = pst.parameter_data + + gr_hk_pars = par.loc[par.parnme.str.contains("hk1_gr"),"parnme"] + pf.pst.parameter_data.loc[gr_hk_pars,"parubnd"] = np.random.random(gr_hk_pars.shape[0]) * 5 + pf.pst.parameter_data.loc[gr_hk_pars, "parlbnd"] = np.random.random(gr_hk_pars.shape[0]) * 0.2 + pe = pf.draw(num_reals=100) + pe.enforce() + pe.to_csv(os.path.join(pf.new_d,"prior.csv")) + cov = pf.build_prior(filename=None) + #make sure the prior cov has off diagonals + cov = pf.build_prior(sigma_range=6) + covx = cov.x.copy() + covx[np.abs(covx)>1.0e-7] = 1.0 + assert covx.sum() > pf.pst.npar_adj + 1 + dcov = pyemu.Cov.from_parameter_data(pf.pst,sigma_range=6) + dcov = dcov.get(cov.row_names) + diag = np.diag(cov.x) + diff = np.abs(diag.flatten() - dcov.x.flatten()) + print(diag) + print(dcov.x) + print(diff) + print(diff.max()) + assert np.isclose(diff.max(), 0), diff.close() + + # test that the arr hds obs process is working + pyemu.gw_utils.apply_hds_obs(os.path.join(pf.new_d, "freyberg.usg.hds"), + precision='single', text='headu') + + # run the full process once using the initial par values in the control file + # since we are using only multipliers, the initial values are all 1's so + # the phi should be pretty close to zero + pf.pst.control_data.noptmax = 0 + pf.pst.write(os.path.join(pf.new_d,"freyberg.usg.pst"),version=2) + pyemu.os_utils.run("{0} freyberg.usg.pst".format(ies_exe_path),cwd=pf.new_d) + pst = pyemu.Pst(os.path.join(pf.new_d,"freyberg.usg.pst")) + assert np.isclose(pst.phi, 0), pst.phi + + #make sure the processed model input arrays are veru similar to the org arrays (again 1s for mults) + for arr_file in ["hk_Layer_1.ref","hk_Layer_3.ref"]: + in_arr = np.loadtxt(os.path.join(pf.new_d,arr_file)) + org_arr = np.loadtxt(os.path.join(pf.new_d,"org",arr_file)) + d = np.abs(in_arr - org_arr) print(d.sum()) - assert d.sum() == 0.0,d.sum() - except Exception as e: - os.chdir(bd) - raise e - os.chdir(bd) + assert np.isclose(d.sum(), 0), arr_file + + + # now run a random realization from the prior par en and make sure things have changed + pst.parameter_data.loc[pe.columns,"parval1"] = pe.iloc[0,:].values + pst.write(os.path.join(pf.new_d, "freyberg.usg.pst"), version=2) + pyemu.os_utils.run("{0} freyberg.usg.pst".format(ies_exe_path), cwd=pf.new_d) + + pst = pyemu.Pst(os.path.join(pf.new_d, "freyberg.usg.pst")) + assert pst.phi > 1.0e-3, pst.phi + + for arr_file in ["hk_Layer_1.ref", "hk_Layer_3.ref"]: + in_arr = np.loadtxt(os.path.join(pf.new_d, arr_file)) + org_arr = np.loadtxt(os.path.join(pf.new_d, "org", arr_file)) + d = np.abs(in_arr - org_arr) + print(d.sum()) + assert d.sum() > 1.0e-3, arr_file + + # check that the pilot point process is respecting the zone array + par = pst.parameter_data + pp_par = par.loc[par.parnme.str.contains("pp"),:] + pst.parameter_data.loc[pp_par.parnme,"parval1"] = pp_par.zone.apply(np.float64) + pst.control_data.noptmax = 0 + pst.write(os.path.join(pf.new_d,"freyberg.usg.pst"),version=2) + #pst.write_input_files(pf.new_d) + pyemu.os_utils.run("{0} freyberg.usg.pst".format(ies_exe_path), cwd=pf.new_d) + arr = np.loadtxt(os.path.join(pf.new_d,"mult","hk3_pp_inst0_pilotpoints.csv")) + arr[zone_array_k2[0,:]==0] = 0 + d = np.abs(arr - zone_array_k2) + print(d) + print(d.sum()) + assert d.sum() == 0.0,d.sum() def mf6_add_various_obs_test(tmp_path): @@ -4000,63 +3934,57 @@ def mf6_add_various_obs_test(tmp_path): org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) - bd = Path.cwd() os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) - m = sim.get_model("freyberg6") - sim.set_all_data_external(check_data=False) - sim.write_simulation() + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + m = sim.get_model("freyberg6") + sim.set_all_data_external(check_data=False) + sim.write_simulation() - # SETUP pest stuff... - os_utils.run("{0} ".format(mf6_exe_path), cwd=tmp_model_ws) + # SETUP pest stuff... + os_utils.run("{0} ".format(mf6_exe_path), cwd=tmp_model_ws) - template_ws = "new_temp" - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - sr = m.modelgrid - # set up PstFrom object - pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, - remove_existing=True, - longnames=True, spatial_reference=sr, - zero_based=False, start_datetime="1-1-2018", - chunk_len=1) - - # blind obs add - pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols='time', - ofile_sep=',') - pf.add_observations("heads.csv", index_cols=0, obsgp='hds') - pf.add_observations("freyberg6.npf_k_layer1.txt", - obsgp='hk1', zone_array=m.dis.idomain.array[0]) - pf.add_observations("freyberg6.npf_k_layer2.txt", - zone_array=m.dis.idomain.array[0], - prefix='lay2k') - pf.add_observations("freyberg6.npf_k_layer3.txt", - zone_array=m.dis.idomain.array[0]) - - linelen = 10000 - _add_big_obsffile(pf, profile=True, nchar=linelen) - - # TODO more variations on the theme - # add single par so we can run - pf.add_parameters(["freyberg6.npf_k_layer1.txt", - "freyberg6.npf_k_layer2.txt", - "freyberg6.npf_k_layer3.txt"],par_type='constant') - pf.mod_sys_cmds.append("mf6") - pf.add_py_function( - __file__, - f"_add_big_obsffile('.', profile=False, nchar={linelen})", - is_pre_cmd=False) - pst = pf.build_pst('freyberg.pst', version=2) - # pst.write(os.path.join(pf.new_d, "freyberg.usg.pst"), version=2) - pyemu.os_utils.run("{0} {1}".format(ies_exe_path, pst.filename.name), - cwd=pf.new_d) - except Exception as e: - os.chdir(bd) - raise e - os.chdir(bd) + template_ws = "new_temp" + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + sr = m.modelgrid + # set up PstFrom object + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + remove_existing=True, + longnames=True, spatial_reference=sr, + zero_based=False, start_datetime="1-1-2018", + chunk_len=50) + + # blind obs add + pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols='time', + ofile_sep=',') + pf.add_observations("heads.csv", index_cols=0, obsgp='hds') + pf.add_observations("freyberg6.npf_k_layer1.txt", + obsgp='hk1', zone_array=m.dis.idomain.array[0]) + pf.add_observations("freyberg6.npf_k_layer2.txt", + zone_array=m.dis.idomain.array[0], + prefix='lay2k') + pf.add_observations("freyberg6.npf_k_layer3.txt", + zone_array=m.dis.idomain.array[0]) + + linelen = 10000 + _add_big_obsffile(pf, profile=True, nchar=linelen) + + # TODO more variations on the theme + # add single par so we can run + pf.add_parameters(["freyberg6.npf_k_layer1.txt", + "freyberg6.npf_k_layer2.txt", + "freyberg6.npf_k_layer3.txt"],par_type='constant') + pf.mod_sys_cmds.append("mf6") + pf.add_py_function( + __file__, + f"_add_big_obsffile('.', profile=False, nchar={linelen})", + is_pre_cmd=False) + pst = pf.build_pst('freyberg.pst', version=2) + # pst.write(os.path.join(pf.new_d, "freyberg.usg.pst"), version=2) + pyemu.os_utils.run("{0} {1}".format(ies_exe_path, pst.filename.name), + cwd=pf.new_d) def _add_big_obsffile(pf, profile=False, nchar=50000): @@ -4095,304 +4023,301 @@ def mf6_subdir_test(tmp_path): except: return + # import sys + # sys.path.insert(0, os.path.join("..", "..", "pypestutils")) + import pypestutils as ppu + org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') sd = "sub_dir" tmp_model_ws = setup_tmp(org_model_ws, tmp_path, sub=sd) - bd = Path.cwd() - os.chdir(tmp_path) - try: - tmp2_ws = tmp_model_ws.relative_to(tmp_path) - tmp_model_ws = tmp2_ws.parents[tmp2_ws.parts.index(sd) - 1] - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp2_ws)) - m = sim.get_model("freyberg6") - sim.set_all_data_external(check_data=False) - sim.write_simulation() + os.chdir(tmp_path) + # get model dir relative to temp path + tmp2_ws = tmp_model_ws.relative_to(tmp_path) + tmp_model_ws = tmp2_ws.parents[tmp2_ws.parts.index(sd) - 1] + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp2_ws)) + m = sim.get_model("freyberg6") + sim.set_all_data_external(check_data=False) + sim.write_simulation() + + # SETUP pest stuff... + if bin_path == '': + exe = mf6_exe_path # bit of flexibility for local/server run + else: + exe = os.path.join('..', mf6_exe_path) + os_utils.run("{0} ".format(exe), cwd=tmp2_ws) + # call generic once so that the output file exists + df = generic_function(tmp2_ws) + template_ws = "new_temp" + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + # sr0 = m.sr + # sr = pyemu.helpers.SpatialReference.from_namfile( + # os.path.join(tmp_model_ws, "freyberg6.nam"), + # delr=m.dis.delr.array, delc=m.dis.delc.array) + sr = m.modelgrid + # set up PstFrom object + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + remove_existing=True, + longnames=True, spatial_reference=sr, + zero_based=False,start_datetime="1-1-2018", + chunk_len=50) + # obs + # using tabular style model output + # (generated by pyemu.gw_utils.setup_hds_obs()) + # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', + # index_cols='obsnme', use_cols='obsval', prefix='hds') + + # add the values in generic to the ctl file + pf.add_observations( + os.path.join(sd, "generic.csv"), + insfile="generic.csv.ins", + index_cols=["datetime", "index_2"], + use_cols=["simval1", "simval2"] + ) + # add the function call to make generic to the forward run script + pf.add_py_function(__file__, f"generic_function('{sd}')",is_pre_cmd=False) + + # add a function that isnt going to be called directly + pf.add_py_function(__file__, "another_generic_function(some_arg)",is_pre_cmd=None) + + # pf.post_py_cmds.append("generic_function()") + df = pd.read_csv(os.path.join(template_ws, sd, "sfr.csv"), index_col=0) + pf.add_observations(os.path.join(sd, "sfr.csv"), index_cols="time", use_cols=list(df.columns.values)) + pf.add_observations(os.path.join(sd, "freyberg6.npf_k_layer1.txt"), + zone_array=m.dis.idomain.array[0]) + + v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) + gr_gs = pyemu.geostats.GeoStruct(variograms=v) + rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0,a=60)) + pf.extra_py_imports.append('flopy') + ib = m.dis.idomain[0].array + tags = {"npf_k_":[0.1,10.],"npf_k33_":[.1,10],"sto_ss":[.1,10],"sto_sy":[.9,1.1],"rch_recharge":[.5,1.5]} + dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]),unit="d") + print(dts) + for tag,bnd in tags.items(): + lb,ub = bnd[0],bnd[1] + arr_files = [os.path.join(sd, f) for f in os.listdir(os.path.join(tmp_model_ws, sd)) if tag in f and f.endswith(".txt")] + if "rch" in tag: + pf.add_parameters(filenames=arr_files, par_type="grid", par_name_base="rch_gr", + pargp="rch_gr", zone_array=ib, upper_bound=ub, lower_bound=lb, + geostruct=gr_gs) + for arr_file in arr_files: + kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 + pf.add_parameters(filenames=arr_file,par_type="constant",par_name_base=arr_file.split('.')[1]+"_cn", + pargp="rch_const",zone_array=ib,upper_bound=ub,lower_bound=lb,geostruct=rch_temporal_gs, + datetime=dts[kper]) + else: + for arr_file in arr_files: + + # these ult bounds are used later in an assert + # and also are used so that the initial input array files + # are preserved + ult_lb = None + ult_ub = None + if "npf_k_" in arr_file: + ult_ub = 31.0 + ult_lb = -1.3 + pf.add_parameters(filenames=arr_file,par_type="grid",par_name_base=arr_file.split('.')[1]+"_gr", + pargp=arr_file.split('.')[1]+"_gr",zone_array=ib,upper_bound=ub,lower_bound=lb, + geostruct=gr_gs,ult_ubound=None if ult_ub is None else ult_ub + 1, + ult_lbound=None if ult_lb is None else ult_lb + 1) + # use a slightly lower ult bound here + pf.add_parameters(filenames=arr_file, par_type="pilotpoints", par_name_base=arr_file.split('.')[1]+"_pp", + pargp=arr_file.split('.')[1]+"_pp", zone_array=ib,upper_bound=ub,lower_bound=lb, + ult_ubound=None if ult_ub is None else ult_ub - 1, + ult_lbound=None if ult_lb is None else ult_lb - 1) + # + # + # add SP1 spatially constant, but temporally correlated wel flux pars + kper = 0 + list_file = os.path.join( + sd, "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) + ) + pf.add_parameters(filenames=list_file, par_type="constant", + par_name_base="twel_mlt_{0}".format(kper), + pargp="twel_mlt".format(kper), index_cols=[0, 1, 2], + use_cols=[3], upper_bound=1.5, lower_bound=0.5, + datetime=dts[kper], geostruct=rch_temporal_gs, + mfile_skip=1) + # + # # add temporally indep, but spatially correlated wel flux pars + # pf.add_parameters(filenames=list_file, par_type="grid", + # par_name_base="wel_grid_{0}".format(kper), + # pargp="wel_{0}".format(kper), index_cols=[0, 1, 2], + # use_cols=[3], upper_bound=1.5, lower_bound=0.5, + # geostruct=gr_gs, mfile_skip=1) + # kper = 1 + # list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) + # pf.add_parameters(filenames=list_file, par_type="constant", + # par_name_base="twel_mlt_{0}".format(kper), + # pargp="twel_mlt".format(kper), index_cols=[0, 1, 2], + # use_cols=[3], upper_bound=1.5, lower_bound=0.5, + # datetime=dts[kper], geostruct=rch_temporal_gs, + # mfile_skip='#') + # # add temporally indep, but spatially correlated wel flux pars + # pf.add_parameters(filenames=list_file, par_type="grid", + # par_name_base="wel_grid_{0}".format(kper), + # pargp="wel_{0}".format(kper), index_cols=[0, 1, 2], + # use_cols=[3], upper_bound=1.5, lower_bound=0.5, + # geostruct=gr_gs, mfile_skip='#') + # kper = 2 + # list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) + # pf.add_parameters(filenames=list_file, par_type="constant", + # par_name_base="twel_mlt_{0}".format(kper), + # pargp="twel_mlt".format(kper), index_cols=['#k', 'i', 'j'], + # use_cols=['flux'], upper_bound=1.5, lower_bound=0.5, + # datetime=dts[kper], geostruct=rch_temporal_gs) + # # add temporally indep, but spatially correlated wel flux pars + # pf.add_parameters(filenames=list_file, par_type="grid", + # par_name_base="wel_grid_{0}".format(kper), + # pargp="wel_{0}".format(kper), index_cols=['#k', 'i', 'j'], + # use_cols=['flux'], upper_bound=1.5, lower_bound=0.5, + # geostruct=gr_gs) + # kper = 3 + # list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) + # pf.add_parameters(filenames=list_file, par_type="constant", + # par_name_base="twel_mlt_{0}".format(kper), + # pargp="twel_mlt".format(kper), index_cols=['#k', 'i', 'j'], + # use_cols=['flux'], upper_bound=1.5, lower_bound=0.5, + # datetime=dts[kper], geostruct=rch_temporal_gs, + # mfile_skip=1) + # # add temporally indep, but spatially correlated wel flux pars + # pf.add_parameters(filenames=list_file, par_type="grid", + # par_name_base="wel_grid_{0}".format(kper), + # pargp="wel_{0}".format(kper), index_cols=['#k', 'i', 'j'], + # use_cols=['flux'], upper_bound=1.5, lower_bound=0.5, + # geostruct=gr_gs, mfile_skip=1) + # list_files = ["freyberg6.wel_stress_period_data_{0}.txt".format(t) + # for t in range(5, m.nper+1)] + # for list_file in list_files: + # kper = int(list_file.split(".")[1].split('_')[-1]) - 1 + # # add spatially constant, but temporally correlated wel flux pars + # pf.add_parameters(filenames=list_file,par_type="constant",par_name_base="twel_mlt_{0}".format(kper), + # pargp="twel_mlt".format(kper),index_cols=[0,1,2],use_cols=[3], + # upper_bound=1.5,lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs) + # + # # add temporally indep, but spatially correlated wel flux pars + # pf.add_parameters(filenames=list_file, par_type="grid", par_name_base="wel_grid_{0}".format(kper), + # pargp="wel_{0}".format(kper), index_cols=[0, 1, 2], use_cols=[3], + # upper_bound=1.5, lower_bound=0.5, geostruct=gr_gs) + # + # # test non spatial idx in list like + # pf.add_parameters(filenames="freyberg6.sfr_packagedata_test.txt", par_name_base="sfr_rhk", + # pargp="sfr_rhk", index_cols=['#rno'], use_cols=['rhk'], upper_bound=10., + # lower_bound=0.1, + # par_type="grid") + # + # # add model run command + pf.pre_py_cmds.append(f"os.chdir('{sd}')") + pf.mod_sys_cmds.append("mf6") + pf.post_py_cmds.insert(0, "os.chdir('..')") + print(pf.mult_files) + print(pf.org_files) + + # build pest + pst = pf.build_pst('freyberg.pst') - # SETUP pest stuff... - if bin_path == '': - exe = mf6_exe_path # bit of flexibility for local/server run - else: - exe = os.path.join('..', mf6_exe_path) - os_utils.run("{0} ".format(exe), cwd=tmp2_ws) - # call generic once so that the output file exists - df = generic_function(tmp2_ws) - template_ws = "new_temp" - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - # sr0 = m.sr - # sr = pyemu.helpers.SpatialReference.from_namfile( - # os.path.join(tmp_model_ws, "freyberg6.nam"), - # delr=m.dis.delr.array, delc=m.dis.delc.array) - sr = m.modelgrid - # set up PstFrom object - pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, - remove_existing=True, - longnames=True, spatial_reference=sr, - zero_based=False,start_datetime="1-1-2018", - chunk_len=1) - # obs - # using tabular style model output - # (generated by pyemu.gw_utils.setup_hds_obs()) - # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', - # index_cols='obsnme', use_cols='obsval', prefix='hds') - - # add the values in generic to the ctl file - pf.add_observations( - os.path.join(sd, "generic.csv"), - insfile="generic.csv.ins", - index_cols=["datetime", "index_2"], - use_cols=["simval1", "simval2"] - ) - # add the function call to make generic to the forward run script - pf.add_py_function(__file__, f"generic_function('{sd}')",is_pre_cmd=False) + # # quick check of write and apply method + # pars = pst.parameter_data + # # set reach 1 hk to 100 + # sfr_pars = pars.loc[pars.parnme.str.startswith('sfr')].index + # pars.loc[sfr_pars, 'parval1'] = np.random.random(len(sfr_pars)) * 10 + # + # sfr_pars = pars.loc[sfr_pars].copy() + # sfr_pars[['inst', 'usecol', '#rno']] = sfr_pars.parnme.apply( + # lambda x: pd.DataFrame([s.split(':') for s in x.split('_') + # if ':' in s]).set_index(0)[1]) + # + # sfr_pars['#rno'] = sfr_pars['#rno'] .astype(int) + # os.chdir(pf.new_d) + # pst.write_input_files() + # try: + # pyemu.helpers.apply_list_and_array_pars() + # except Exception as e: + # os.chdir('..') + # raise e + # os.chdir('..') + # # verify apply + # df = pd.read_csv(os.path.join( + # pf.new_d, "freyberg6.sfr_packagedata_test.txt"), + # delim_whitespace=True, index_col=0) + # df.index = df.index - 1 + # print(df.rhk) + # print((sfr_pkgdf.set_index('rno').loc[df.index, 'rhk'] * + # sfr_pars.set_index('#rno').loc[df.index, 'parval1'])) + # assert np.isclose( + # df.rhk, (sfr_pkgdf.set_index('rno').loc[df.index, 'rhk'] * + # sfr_pars.set_index('#rno').loc[df.index, 'parval1'])).all() + # pars.loc[sfr_pars.index, 'parval1'] = 1.0 + # + # # add more: + # pf.add_parameters(filenames="freyberg6.sfr_packagedata.txt", par_name_base="sfr_rhk", + # pargp="sfr_rhk", index_cols={'k': 1, 'i': 2, 'j': 3}, use_cols=[9], upper_bound=10., + # lower_bound=0.1, + # par_type="grid", rebuild_pst=True) + # + # df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0) + pf.add_observations(os.path.join(sd, "heads.csv"), + insfile=os.path.join(sd, "heads.csv.ins"), + index_cols="time", + prefix="hds", + rebuild_pst=True) + # + # # test par mults are working + check_apply(pf, 10) - # add a function that isnt going to be called directly - pf.add_py_function(__file__, "another_generic_function(some_arg)",is_pre_cmd=None) + # + # cov = pf.build_prior(fmt="none").to_dataframe() + # twel_pars = [p for p in pst.par_names if "twel_mlt" in p] + # twcov = cov.loc[twel_pars,twel_pars] + # dsum = np.diag(twcov.values).sum() + # assert twcov.sum().sum() > dsum + # + # rch_cn = [p for p in pst.par_names if "_cn" in p] + # print(rch_cn) + # rcov = cov.loc[rch_cn,rch_cn] + # dsum = np.diag(rcov.values).sum() + # assert rcov.sum().sum() > dsum + # + # num_reals = 100 + # pe = pf.draw(num_reals, use_specsim=True) + # pe.to_binary(os.path.join(template_ws, "prior.jcb")) + # assert pe.shape[1] == pst.npar_adj, "{0} vs {1}".format(pe.shape[0], pst.npar_adj) + # assert pe.shape[0] == num_reals + # + # + pst.control_data.noptmax = 0 + pst.pestpp_options["additional_ins_delimiters"] = "," + # + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + # + res_file = os.path.join(pf.new_d, "freyberg.base.rei") + assert os.path.exists(res_file), res_file + pst.set_res(res_file) + # print(pst.phi) + assert np.isclose(pst.phi, 0), pst.phi + # + # check mult files are in pst input files + csv = os.path.join(template_ws, "mult2model_info.csv") + df = pd.read_csv(csv, index_col=0) + pst_input_files = {str(f) for f in pst.input_files} + mults_not_linked_to_pst = ((set(df.mlt_file.unique()) - + pst_input_files) - + set(df.loc[df.pp_file.notna()].mlt_file)) + assert len(mults_not_linked_to_pst) == 0, print(mults_not_linked_to_pst) - # pf.post_py_cmds.append("generic_function()") - df = pd.read_csv(os.path.join(template_ws, sd, "sfr.csv"), index_col=0) - pf.add_observations(os.path.join(sd, "sfr.csv"), index_cols="time", use_cols=list(df.columns.values)) - pf.add_observations(os.path.join(sd, "freyberg6.npf_k_layer1.txt"), - zone_array=m.dis.idomain.array[0]) + # make sure the appropriate ult bounds have made it thru + # df = pd.read_csv(os.path.join(template_ws,"mult2model_info.csv")) + # print(df.columns) + # df = df.loc[df.model_file.apply(lambda x: "npf_k_" in x),:] + # print(df) + # print(df.upper_bound) + # print(df.lower_bound) + # assert np.abs(float(df.upper_bound.min()) - 30.) < 1.0e-6,df.upper_bound.min() + # assert np.abs(float(df.lower_bound.max()) - -0.3) < 1.0e-6,df.lower_bound.max() - v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) - gr_gs = pyemu.geostats.GeoStruct(variograms=v) - rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0,a=60)) - pf.extra_py_imports.append('flopy') - ib = m.dis.idomain[0].array - tags = {"npf_k_":[0.1,10.],"npf_k33_":[.1,10],"sto_ss":[.1,10],"sto_sy":[.9,1.1],"rch_recharge":[.5,1.5]} - dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]),unit="d") - print(dts) - for tag,bnd in tags.items(): - lb,ub = bnd[0],bnd[1] - arr_files = [os.path.join(sd, f) for f in os.listdir(os.path.join(tmp_model_ws, sd)) if tag in f and f.endswith(".txt")] - if "rch" in tag: - pf.add_parameters(filenames=arr_files, par_type="grid", par_name_base="rch_gr", - pargp="rch_gr", zone_array=ib, upper_bound=ub, lower_bound=lb, - geostruct=gr_gs) - for arr_file in arr_files: - kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 - pf.add_parameters(filenames=arr_file,par_type="constant",par_name_base=arr_file.split('.')[1]+"_cn", - pargp="rch_const",zone_array=ib,upper_bound=ub,lower_bound=lb,geostruct=rch_temporal_gs, - datetime=dts[kper]) - else: - for arr_file in arr_files: - - # these ult bounds are used later in an assert - # and also are used so that the initial input array files - # are preserved - ult_lb = None - ult_ub = None - if "npf_k_" in arr_file: - ult_ub = 31.0 - ult_lb = -1.3 - pf.add_parameters(filenames=arr_file,par_type="grid",par_name_base=arr_file.split('.')[1]+"_gr", - pargp=arr_file.split('.')[1]+"_gr",zone_array=ib,upper_bound=ub,lower_bound=lb, - geostruct=gr_gs,ult_ubound=None if ult_ub is None else ult_ub + 1, - ult_lbound=None if ult_lb is None else ult_lb + 1) - # use a slightly lower ult bound here - pf.add_parameters(filenames=arr_file, par_type="pilotpoints", par_name_base=arr_file.split('.')[1]+"_pp", - pargp=arr_file.split('.')[1]+"_pp", zone_array=ib,upper_bound=ub,lower_bound=lb, - ult_ubound=None if ult_ub is None else ult_ub - 1, - ult_lbound=None if ult_lb is None else ult_lb - 1) - # - # - # add SP1 spatially constant, but temporally correlated wel flux pars - kper = 0 - list_file = os.path.join( - sd, "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) - ) - pf.add_parameters(filenames=list_file, par_type="constant", - par_name_base="twel_mlt_{0}".format(kper), - pargp="twel_mlt".format(kper), index_cols=[0, 1, 2], - use_cols=[3], upper_bound=1.5, lower_bound=0.5, - datetime=dts[kper], geostruct=rch_temporal_gs, - mfile_skip=1) - # - # # add temporally indep, but spatially correlated wel flux pars - # pf.add_parameters(filenames=list_file, par_type="grid", - # par_name_base="wel_grid_{0}".format(kper), - # pargp="wel_{0}".format(kper), index_cols=[0, 1, 2], - # use_cols=[3], upper_bound=1.5, lower_bound=0.5, - # geostruct=gr_gs, mfile_skip=1) - # kper = 1 - # list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) - # pf.add_parameters(filenames=list_file, par_type="constant", - # par_name_base="twel_mlt_{0}".format(kper), - # pargp="twel_mlt".format(kper), index_cols=[0, 1, 2], - # use_cols=[3], upper_bound=1.5, lower_bound=0.5, - # datetime=dts[kper], geostruct=rch_temporal_gs, - # mfile_skip='#') - # # add temporally indep, but spatially correlated wel flux pars - # pf.add_parameters(filenames=list_file, par_type="grid", - # par_name_base="wel_grid_{0}".format(kper), - # pargp="wel_{0}".format(kper), index_cols=[0, 1, 2], - # use_cols=[3], upper_bound=1.5, lower_bound=0.5, - # geostruct=gr_gs, mfile_skip='#') - # kper = 2 - # list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) - # pf.add_parameters(filenames=list_file, par_type="constant", - # par_name_base="twel_mlt_{0}".format(kper), - # pargp="twel_mlt".format(kper), index_cols=['#k', 'i', 'j'], - # use_cols=['flux'], upper_bound=1.5, lower_bound=0.5, - # datetime=dts[kper], geostruct=rch_temporal_gs) - # # add temporally indep, but spatially correlated wel flux pars - # pf.add_parameters(filenames=list_file, par_type="grid", - # par_name_base="wel_grid_{0}".format(kper), - # pargp="wel_{0}".format(kper), index_cols=['#k', 'i', 'j'], - # use_cols=['flux'], upper_bound=1.5, lower_bound=0.5, - # geostruct=gr_gs) - # kper = 3 - # list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) - # pf.add_parameters(filenames=list_file, par_type="constant", - # par_name_base="twel_mlt_{0}".format(kper), - # pargp="twel_mlt".format(kper), index_cols=['#k', 'i', 'j'], - # use_cols=['flux'], upper_bound=1.5, lower_bound=0.5, - # datetime=dts[kper], geostruct=rch_temporal_gs, - # mfile_skip=1) - # # add temporally indep, but spatially correlated wel flux pars - # pf.add_parameters(filenames=list_file, par_type="grid", - # par_name_base="wel_grid_{0}".format(kper), - # pargp="wel_{0}".format(kper), index_cols=['#k', 'i', 'j'], - # use_cols=['flux'], upper_bound=1.5, lower_bound=0.5, - # geostruct=gr_gs, mfile_skip=1) - # list_files = ["freyberg6.wel_stress_period_data_{0}.txt".format(t) - # for t in range(5, m.nper+1)] - # for list_file in list_files: - # kper = int(list_file.split(".")[1].split('_')[-1]) - 1 - # # add spatially constant, but temporally correlated wel flux pars - # pf.add_parameters(filenames=list_file,par_type="constant",par_name_base="twel_mlt_{0}".format(kper), - # pargp="twel_mlt".format(kper),index_cols=[0,1,2],use_cols=[3], - # upper_bound=1.5,lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs) - # - # # add temporally indep, but spatially correlated wel flux pars - # pf.add_parameters(filenames=list_file, par_type="grid", par_name_base="wel_grid_{0}".format(kper), - # pargp="wel_{0}".format(kper), index_cols=[0, 1, 2], use_cols=[3], - # upper_bound=1.5, lower_bound=0.5, geostruct=gr_gs) - # - # # test non spatial idx in list like - # pf.add_parameters(filenames="freyberg6.sfr_packagedata_test.txt", par_name_base="sfr_rhk", - # pargp="sfr_rhk", index_cols=['#rno'], use_cols=['rhk'], upper_bound=10., - # lower_bound=0.1, - # par_type="grid") - # - # # add model run command - pf.pre_py_cmds.append(f"os.chdir('{sd}')") - pf.mod_sys_cmds.append("mf6") - pf.post_py_cmds.insert(0, "os.chdir('..')") - print(pf.mult_files) - print(pf.org_files) - - # build pest - pst = pf.build_pst('freyberg.pst') - - # # quick check of write and apply method - # pars = pst.parameter_data - # # set reach 1 hk to 100 - # sfr_pars = pars.loc[pars.parnme.str.startswith('sfr')].index - # pars.loc[sfr_pars, 'parval1'] = np.random.random(len(sfr_pars)) * 10 - # - # sfr_pars = pars.loc[sfr_pars].copy() - # sfr_pars[['inst', 'usecol', '#rno']] = sfr_pars.parnme.apply( - # lambda x: pd.DataFrame([s.split(':') for s in x.split('_') - # if ':' in s]).set_index(0)[1]) - # - # sfr_pars['#rno'] = sfr_pars['#rno'] .astype(int) - # os.chdir(pf.new_d) - # pst.write_input_files() - # try: - # pyemu.helpers.apply_list_and_array_pars() - # except Exception as e: - # os.chdir('..') - # raise e - # os.chdir('..') - # # verify apply - # df = pd.read_csv(os.path.join( - # pf.new_d, "freyberg6.sfr_packagedata_test.txt"), - # delim_whitespace=True, index_col=0) - # df.index = df.index - 1 - # print(df.rhk) - # print((sfr_pkgdf.set_index('rno').loc[df.index, 'rhk'] * - # sfr_pars.set_index('#rno').loc[df.index, 'parval1'])) - # assert np.isclose( - # df.rhk, (sfr_pkgdf.set_index('rno').loc[df.index, 'rhk'] * - # sfr_pars.set_index('#rno').loc[df.index, 'parval1'])).all() - # pars.loc[sfr_pars.index, 'parval1'] = 1.0 - # - # # add more: - # pf.add_parameters(filenames="freyberg6.sfr_packagedata.txt", par_name_base="sfr_rhk", - # pargp="sfr_rhk", index_cols={'k': 1, 'i': 2, 'j': 3}, use_cols=[9], upper_bound=10., - # lower_bound=0.1, - # par_type="grid", rebuild_pst=True) - # - # df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0) - pf.add_observations(os.path.join(sd, "heads.csv"), - insfile=os.path.join(sd, "heads.csv.ins"), - index_cols="time", - prefix="hds", - rebuild_pst=True) - # - # # test par mults are working - os.chdir(pf.new_d) - pst.write_input_files() - pyemu.helpers.apply_list_and_array_pars( - arr_par_file="mult2model_info.csv",chunk_len=1) - os.chdir(tmp_path) - # - # cov = pf.build_prior(fmt="none").to_dataframe() - # twel_pars = [p for p in pst.par_names if "twel_mlt" in p] - # twcov = cov.loc[twel_pars,twel_pars] - # dsum = np.diag(twcov.values).sum() - # assert twcov.sum().sum() > dsum - # - # rch_cn = [p for p in pst.par_names if "_cn" in p] - # print(rch_cn) - # rcov = cov.loc[rch_cn,rch_cn] - # dsum = np.diag(rcov.values).sum() - # assert rcov.sum().sum() > dsum - # - # num_reals = 100 - # pe = pf.draw(num_reals, use_specsim=True) - # pe.to_binary(os.path.join(template_ws, "prior.jcb")) - # assert pe.shape[1] == pst.npar_adj, "{0} vs {1}".format(pe.shape[0], pst.npar_adj) - # assert pe.shape[0] == num_reals - # - # - pst.control_data.noptmax = 0 - pst.pestpp_options["additional_ins_delimiters"] = "," - # - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - # - res_file = os.path.join(pf.new_d, "freyberg.base.rei") - assert os.path.exists(res_file), res_file - pst.set_res(res_file) - # print(pst.phi) - assert np.isclose(pst.phi, 0), pst.phi - # - # check mult files are in pst input files - csv = os.path.join(template_ws, "mult2model_info.csv") - df = pd.read_csv(csv, index_col=0) - pst_input_files = {str(f) for f in pst.input_files} - mults_not_linked_to_pst = ((set(df.mlt_file.unique()) - - pst_input_files) - - set(df.loc[df.pp_file.notna()].mlt_file)) - assert len(mults_not_linked_to_pst) == 0, print(mults_not_linked_to_pst) - - # make sure the appropriate ult bounds have made it thru - # df = pd.read_csv(os.path.join(template_ws,"mult2model_info.csv")) - # print(df.columns) - # df = df.loc[df.model_file.apply(lambda x: "npf_k_" in x),:] - # print(df) - # print(df.upper_bound) - # print(df.lower_bound) - # assert np.abs(float(df.upper_bound.min()) - 30.) < 1.0e-6,df.upper_bound.min() - # assert np.abs(float(df.lower_bound.max()) - -0.3) < 1.0e-6,df.lower_bound.max() - except Exception as e: - os.chdir(bd) - raise Exception(str(e)) - os.chdir(bd) def shortname_conversion_test(tmp_path): @@ -4413,169 +4338,162 @@ def shortname_conversion_test(tmp_path): os.mkdir(tmp_model_ws) bd = Path.cwd() os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - dims = (20,60) - np.savetxt(os.path.join(tmp_model_ws,"parfile1"), np.ones(dims)) - np.savetxt(os.path.join(tmp_model_ws, "parfile2"), np.ones(dims)) - np.savetxt(os.path.join(tmp_model_ws, "parfile3"), np.ones(dims)) - np.savetxt(os.path.join(tmp_model_ws, "parfile4"), np.ones(dims)) - np.savetxt(os.path.join(tmp_model_ws, "parfile5"), np.ones(dims)) - np.savetxt(os.path.join(tmp_model_ws, "parfile6"), np.ones(dims)) - np.savetxt(os.path.join(tmp_model_ws, "obsfile1"), np.ones(dims)) - np.savetxt(os.path.join(tmp_model_ws, "obsfile2"), np.ones(dims)) - - np.savetxt(os.path.join(tmp_model_ws, "parfile7"), np.ones(dims)) - np.savetxt(os.path.join(tmp_model_ws, "obsfile3"), np.ones(dims)) - # SETUP pest stuff... + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + dims = (20,60) + np.savetxt(os.path.join(tmp_model_ws,"parfile1"), np.ones(dims)) + np.savetxt(os.path.join(tmp_model_ws, "parfile2"), np.ones(dims)) + np.savetxt(os.path.join(tmp_model_ws, "parfile3"), np.ones(dims)) + np.savetxt(os.path.join(tmp_model_ws, "parfile4"), np.ones(dims)) + np.savetxt(os.path.join(tmp_model_ws, "parfile5"), np.ones(dims)) + np.savetxt(os.path.join(tmp_model_ws, "parfile6"), np.ones(dims)) + np.savetxt(os.path.join(tmp_model_ws, "obsfile1"), np.ones(dims)) + np.savetxt(os.path.join(tmp_model_ws, "obsfile2"), np.ones(dims)) + + np.savetxt(os.path.join(tmp_model_ws, "parfile7"), np.ones(dims)) + np.savetxt(os.path.join(tmp_model_ws, "obsfile3"), np.ones(dims)) + # SETUP pest stuff... - template_ws = "new_temp" - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - # set up PstFrom object - # obs - # using tabular style model output - # (generated by pyemu.gw_utils.setup_hds_obs()) - # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', - # index_cols='obsnme', use_cols='obsval', prefix='hds') - sr = pyemu.helpers.SpatialReference(delr=[10]*dims[1], - delc=[10]*dims[0], - rotation=0, - epsg=3070, - xul=0., - yul=0., - units='meters', # gis units of meters? - lenuni=2 # model units of meters - ) - pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, - remove_existing=True, - longnames=False, - zero_based=False, - spatial_reference=sr, - pp_solve_num_threads=1) + template_ws = "new_temp" + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + # set up PstFrom object + # obs + # using tabular style model output + # (generated by pyemu.gw_utils.setup_hds_obs()) + # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2', + # index_cols='obsnme', use_cols='obsval', prefix='hds') + sr = pyemu.helpers.SpatialReference(delr=[10]*dims[1], + delc=[10]*dims[0], + rotation=0, + epsg=3070, + xul=0., + yul=0., + units='meters', # gis units of meters? + lenuni=2 # model units of meters + ) + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + remove_existing=True, + longnames=False, + zero_based=False, + spatial_reference=sr, + pp_solve_num_threads=1) - v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) - gr_gs = pyemu.geostats.GeoStruct(variograms=v) - c = 0 - parfiles = [f.name for f in Path(template_ws).glob("parfile*")][0:3] - for f in parfiles: - c += 1 - pf.add_parameters(filenames=f, par_type="grid", - par_name_base=f"par{c}", - pargp=f"pargp{c}", upper_bound=0.1, lower_bound=10.0, - geostruct=gr_gs) - pf.add_parameters(filenames=parfiles, - par_type="constant", par_name_base="cpar", - pargp="cpargp", upper_bound=0.1, lower_bound=10.0, - geostruct=gr_gs) - pf.add_parameters(filenames=parfiles, - par_type="pilotpoints", par_name_base="pppar", - pargp="pppargp", upper_bound=0.1, lower_bound=10.0, + v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) + gr_gs = pyemu.geostats.GeoStruct(variograms=v) + c = 0 + parfiles = [f.name for f in Path(template_ws).glob("parfile*")][0:3] + for f in parfiles: + c += 1 + pf.add_parameters(filenames=f, par_type="grid", + par_name_base=f"par{c}", + pargp=f"pargp{c}", upper_bound=0.1, lower_bound=10.0, geostruct=gr_gs) + pf.add_parameters(filenames=parfiles, + par_type="constant", par_name_base="cpar", + pargp="cpargp", upper_bound=0.1, lower_bound=10.0, + geostruct=gr_gs) + pf.add_parameters(filenames=parfiles, + par_type="pilotpoints", par_name_base="pppar", + pargp="pppargp", upper_bound=0.1, lower_bound=10.0, + geostruct=gr_gs) - pp_locs = pyemu.pp_utils.setup_pilotpoints_grid(sr=sr,prefix_dict={0:"pps_1"}) - pp_locs = pp_locs.loc[:,["name","x","y","zone","parval1"]] - pf.add_parameters(filenames=parfiles, - par_type="pilotpoints", par_name_base="pppar2", - pargp="pppargp2", upper_bound=0.1, lower_bound=10.0, - geostruct=gr_gs,pp_space=pp_locs) + pp_locs = pyemu.pp_utils.setup_pilotpoints_grid(sr=sr,prefix_dict={0:"pps_1"}) + pp_locs = pp_locs.loc[:,["name","x","y","zone","parval1"]] + pf.add_parameters(filenames=parfiles, + par_type="pilotpoints", par_name_base="pppar2", + pargp="pppargp2", upper_bound=0.1, lower_bound=10.0, + geostruct=gr_gs,pp_space=pp_locs) - pf.add_observations( - "obsfile1", - prefix="longobservationname", - rebuild_pst=False, - obsgp="longobservationgroup", - includes_header=False - ) - pf.add_observations( - "obsfile2", - prefix="longobservationname2", - rebuild_pst=False, - obsgp="longobservationgroup2", - includes_header=False - ) - pf.add_observations( - "obsfile3", - prefix="longobservationname-lt", - rebuild_pst=False, - obsgp="less_longobservationgroup", - includes_header=False, - insfile="lt_obsfile3.ins" - ) - pf.add_observations( - "obsfile3", - prefix="longobservationname-gt", - rebuild_pst=False, - obsgp="greater_longobservationgroup", - includes_header=False, - insfile="gt_obsfile3.ins" - ) - pst = pf.build_pst() - obs = set(pst.observation_data.obsnme) - trie = pyemu.helpers.Trie() - [trie.add(ob) for ob in obs] - rex = re.compile(trie.pattern()) - for ins in pst.instruction_files: - with open(os.path.join(pf.new_d, ins), "rt") as f: - obsin = set(rex.findall(f.read())) - obs = obs - obsin - assert len(obs) == 0, f"{len(obs)} obs not found in insfiles: {obs[:100]}..." - - par = set(pst.parameter_data.parnme) - trie = pyemu.helpers.Trie() - [trie.add(p) for p in par] - rex = re.compile(trie.pattern()) - for tpl in pst.template_files: - with open(os.path.join(pf.new_d, tpl), "rt") as f: - parin = set(rex.findall(f.read())) - par = par - parin - assert len(par) == 0, f"{len(par)} pars not found in tplfiles: {par}..." - # test update/rebuild - pf.add_observations( - "obsfile3", - prefix="longobservationname3", - rebuild_pst=True, - obsgp="longobservationgroup3", - includes_header=False - ) - pf.add_parameters(filenames="parfile7", - par_type="grid", par_name_base="par7", - pargp="par7", upper_bound=0.1, lower_bound=10.0, - geostruct=gr_gs, - rebuild_pst=True) - - obs = set(pst.observation_data.obsnme) - trie = pyemu.helpers.Trie() - [trie.add(ob) for ob in obs] - rex = re.compile(trie.pattern()) - for ins in pst.instruction_files: - with open(os.path.join(pf.new_d, ins), "rt") as f: - obsin = set(rex.findall(f.read())) - obs = obs - obsin - assert len(obs) == 0, f"{len(obs)} obs not found in insfiles: {obs[:100]}..." - - par = set(pst.parameter_data.parnme) - parin = set() - trie = pyemu.helpers.Trie() - [trie.add(p) for p in par] - rex = re.compile(trie.pattern()) - for tpl in pst.template_files: - with open(os.path.join(pf.new_d, tpl), "rt") as f: - parin = set(rex.findall(f.read())) - par = par - parin - assert len(par) == 0, f"{len(par)} pars not found in tplfiles: {par[:100]}..." - except Exception as e: - os.chdir(bd) - raise Exception(str(e)) + pf.add_observations( + "obsfile1", + prefix="longobservationname", + rebuild_pst=False, + obsgp="longobservationgroup", + includes_header=False + ) + pf.add_observations( + "obsfile2", + prefix="longobservationname2", + rebuild_pst=False, + obsgp="longobservationgroup2", + includes_header=False + ) + pf.add_observations( + "obsfile3", + prefix="longobservationname-lt", + rebuild_pst=False, + obsgp="less_longobservationgroup", + includes_header=False, + insfile="lt_obsfile3.ins" + ) + pf.add_observations( + "obsfile3", + prefix="longobservationname-gt", + rebuild_pst=False, + obsgp="greater_longobservationgroup", + includes_header=False, + insfile="gt_obsfile3.ins" + ) + pst = pf.build_pst() + obs = set(pst.observation_data.obsnme) + trie = pyemu.helpers.Trie() + [trie.add(ob) for ob in obs] + rex = re.compile(trie.pattern()) + for ins in pst.instruction_files: + with open(os.path.join(pf.new_d, ins), "rt") as f: + obsin = set(rex.findall(f.read())) + obs = obs - obsin + assert len(obs) == 0, f"{len(obs)} obs not found in insfiles: {obs[:100]}..." - os.chdir(bd) + par = set(pst.parameter_data.parnme) + trie = pyemu.helpers.Trie() + [trie.add(p) for p in par] + rex = re.compile(trie.pattern()) + for tpl in pst.template_files: + with open(os.path.join(pf.new_d, tpl), "rt") as f: + parin = set(rex.findall(f.read())) + par = par - parin + assert len(par) == 0, f"{len(par)} pars not found in tplfiles: {par}..." + # test update/rebuild + pf.add_observations( + "obsfile3", + prefix="longobservationname3", + rebuild_pst=True, + obsgp="longobservationgroup3", + includes_header=False + ) + pf.add_parameters(filenames="parfile7", + par_type="grid", par_name_base="par7", + pargp="par7", upper_bound=0.1, lower_bound=10.0, + geostruct=gr_gs, + rebuild_pst=True) + + obs = set(pst.observation_data.obsnme) + trie = pyemu.helpers.Trie() + [trie.add(ob) for ob in obs] + rex = re.compile(trie.pattern()) + for ins in pst.instruction_files: + with open(os.path.join(pf.new_d, ins), "rt") as f: + obsin = set(rex.findall(f.read())) + obs = obs - obsin + assert len(obs) == 0, f"{len(obs)} obs not found in insfiles: {obs[:100]}..." + + par = set(pst.parameter_data.parnme) + parin = set() + trie = pyemu.helpers.Trie() + [trie.add(p) for p in par] + rex = re.compile(trie.pattern()) + for tpl in pst.template_files: + with open(os.path.join(pf.new_d, tpl), "rt") as f: + parin = set(rex.findall(f.read())) + par = par - parin + assert len(par) == 0, f"{len(par)} pars not found in tplfiles: {par[:100]}..." -@pytest.mark.parametrize("setup_freyberg_mf6", ['freyberg_quadtree'], indirect=True) -def vertex_grid_test(setup_freyberg_mf6): - pf, sim = setup_freyberg_mf6 +def vertex_grid_test(tmp_path): + pf, sim = setup_freyberg_mf6(tmp_path, "freyberg_quadtree") m = sim.get_model() mg = m.modelgrid # the model grid is vertex type @@ -4720,37 +4638,32 @@ def test_defaults(tmp_path): org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) - bd = Path.cwd() + os.chdir(tmp_path) - try: - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) - m = sim.get_model() - sim.set_all_data_external(check_data=False) - sim.write_simulation() + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + m = sim.get_model() + sim.set_all_data_external(check_data=False) + sim.write_simulation() - # SETUP pest stuff... - os_utils.run("{0} ".format(mf6_exe_path), cwd=tmp_model_ws) - template_ws = "new_temp" - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - # sr0 = m.sr - # sr = pyemu.helpers.SpatialReference.from_namfile( - # os.path.join(tmp_model_ws, "freyberg6.nam"), - # delr=m.dis.delr.array, delc=m.dis.delc.array) - sr = m.modelgrid - # set up PstFrom object - pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws) - kper = 0 - list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper + 1) - pf.add_parameters(list_file, par_type="grid", index_cols=[0, 1, 2], - use_cols=[3]) - pf.add_observations("heads.csv", index_cols="time") - pst = pf.build_pst() - except Exception as e: - os.chdir(bd) - raise Exception(str(e)) - os.chdir(bd) + # SETUP pest stuff... + os_utils.run("{0} ".format(mf6_exe_path), cwd=tmp_model_ws) + template_ws = "new_temp" + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + # sr0 = m.sr + # sr = pyemu.helpers.SpatialReference.from_namfile( + # os.path.join(tmp_model_ws, "freyberg6.nam"), + # delr=m.dis.delr.array, delc=m.dis.delc.array) + sr = m.modelgrid + # set up PstFrom object + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws) + kper = 0 + list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper + 1) + pf.add_parameters(list_file, par_type="grid", index_cols=[0, 1, 2], + use_cols=[3]) + pf.add_observations("heads.csv", index_cols="time") + pst = pf.build_pst() def list_float_int_index_test(tmp_path): @@ -4815,15 +4728,8 @@ def list_float_int_index_test(tmp_path): bpar = par.parnme.str.contains("ghbcondN") bparval1 = np.linspace(0.1, 10, sum(bpar)) par.loc[bpar, "parval1"] = bparval1 - pst.write_input_files(pf.new_d) - bd = os.getcwd() - os.chdir(pf.new_d) - try: - pyemu.helpers.apply_list_and_array_pars(chunk_len=1000) - except Exception as e: - os.chdir(bd) - raise e - os.chdir(bd) + check_apply(pf, 1000) + faultdf_n = pd.read_csv(os.path.join(pf.new_d, "ppoints.faults.csv")) idxcheck = faultdf_n.set_index(faultidx).index.difference(faultdf_o.set_index(faultidx).index) assert len(idxcheck) == 0, idxcheck @@ -4838,312 +4744,325 @@ def list_float_int_index_test(tmp_path): assert np.isclose(diff,bparval1).all(), diff.loc[~np.isclose(diff,bparval1)] -def mf6_freyberg_thresh_test(tmp_path): +def mf6_freyberg_thresh_test(tmp_path): + + import numpy as np + import pandas as pd + # pd.set_option('display.max_rows', 500) + # pd.set_option('display.max_columns', 500) + # pd.set_option('display.width', 1000) + # try: + import flopy + # except: + # return + # import sys + # sys.path.insert(0,os.path.join("..","..","pypestutils")) + import pypestutils as ppu + + + org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') + tmp_model_ws = setup_tmp(org_model_ws, tmp_path) + + + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + + os.chdir(tmp_path) + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + m = sim.get_model("freyberg6") + sim.set_all_data_external() + sim.write_simulation() + + + + # SETUP pest stuff... + os_utils.run("{0} ".format("mf6"), cwd=tmp_model_ws) + + template_ws = Path(tmp_path, "new_temp_thresh") + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + sr = m.modelgrid + # set up PstFrom object + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + remove_existing=True, + longnames=True, spatial_reference=sr, + zero_based=False, start_datetime="1-1-2018") + + df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0) + pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", + use_cols=list(df.columns.values), + prefix="hds", rebuild_pst=True) + + # Add stream flow observation + # df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) + pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", + use_cols=["GAGE_1", "HEADWATER", "TAILWATER"], ofile_sep=",") + + # Setup geostruct for spatial pars + gr_v = pyemu.geostats.ExpVario(contribution=1.0, a=500) + gr_gs = pyemu.geostats.GeoStruct(variograms=gr_v, transform="log") + pp_v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) + pp_gs = pyemu.geostats.GeoStruct(variograms=pp_v, transform="log") + rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60)) + pf.extra_py_imports.append('flopy') + ib = m.dis.idomain[0].array + #tags = {"npf_k_": [0.1, 10.], "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1], + # "rch_recharge": [.5, 1.5]} + tags = {"npf_k_": [0.1, 10.],"rch_recharge": [.5, 1.5]} + dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") + print(dts) + # ib = m.dis.idomain.array[0,:,:] + # setup from array style pars + num_cat_arrays = 0 + for tag, bnd in tags.items(): + lb, ub = bnd[0], bnd[1] + arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] + if "rch" in tag: + pass + # for arr_file in arr_files: + # # indy direct grid pars for each array type file + # recharge_files = ["recharge_1.txt", "recharge_2.txt", "recharge_3.txt"] + # pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base="rch_gr", + # pargp="rch_gr", zone_array=ib, upper_bound=1.0e-3, lower_bound=1.0e-7, + # par_style="direct") + # # additional constant mults + # kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 + # pf.add_parameters(filenames=arr_file, par_type="constant", + # par_name_base=arr_file.split('.')[1] + "_cn", + # pargp="rch_const", zone_array=ib, upper_bound=ub, lower_bound=lb, + # geostruct=rch_temporal_gs, + # datetime=dts[kper]) + else: + for arr_file in arr_files: + print(arr_file) + k = int(arr_file.split(".")[1][-1]) - 1 + if k == 1: + arr_file = arr_file.replace("_k_","_k33_") + pth_arr_file = os.path.join(pf.new_d,arr_file) + arr = np.loadtxt(pth_arr_file) + cat_dict = {1:[0.4,arr.mean()],2:[0.6,arr.mean()]} + thresharr,threshcsv = pyemu.helpers.setup_threshold_pars(pth_arr_file,cat_dict=cat_dict, + testing_workspace=pf.new_d,inact_arr=ib) + + pf.pre_py_cmds.append("pyemu.helpers.apply_threshold_pars('{0}')".format(os.path.split(threshcsv)[1])) + prefix = arr_file.split('.')[1].replace("_","-") + pf.add_parameters(filenames=os.path.split(thresharr)[1],par_type="grid",transform="none", + par_name_base=prefix+"-threshgr_k:{0}".format(k), + pargp=prefix + "-threshgr_k:{0}".format(k), + lower_bound=0.0,upper_bound=1.0,geostruct=gr_gs,par_style="d") + + + pf.add_parameters(filenames=os.path.split(thresharr)[1],par_type="pilotpoints",transform="none", + par_name_base=prefix+"-threshpp_k:{0}".format(k), + pargp=prefix + "-threshpp_k:{0}".format(k), + lower_bound=0.0,upper_bound=2.0,geostruct=pp_gs,par_style="m", + pp_space=3,pp_options={"try_use_ppu":True} + ) + + + pf.add_parameters(filenames=os.path.split(threshcsv)[1], par_type="grid",index_cols=["threshcat"], + use_cols=["threshproportion","threshfill"], + par_name_base=[prefix+"threshproportion_k:{0}".format(k),prefix+"threshfill_k:{0}".format(k)], + pargp=[prefix+"threshproportion_k:{0}".format(k),prefix+"threshfill_k:{0}".format(k)], + lower_bound=[0.1,0.1],upper_bound=[10.0,10.0],transform="none",par_style='d') + + pf.add_observations(arr_file,prefix="hkarr-"+prefix+"_k:{0}".format(k), + obsgp="hkarr-"+prefix+"_k:{0}".format(k),zone_array=ib) + + pf.add_observations(arr_file+".threshcat.dat", prefix="tcatarr-" + prefix+"_k:{0}".format(k), + obsgp="tcatarr-" + prefix+"_k:{0}".format(k),zone_array=ib) + + pf.add_observations(arr_file + ".thresharr.dat", + prefix="tarr-" +prefix+"_k:{0}".format(k), + obsgp="tarr-" + prefix + "_k:{0}".format(k), zone_array=ib) + + df = pd.read_csv(threshcsv.replace(".csv","_results.csv"),index_col=0) + + pf.add_observations(os.path.split(threshcsv)[1].replace(".csv","_results.csv"),index_cols="threshcat",use_cols=df.columns.tolist(),prefix=prefix+"-results_k:{0}".format(k), + obsgp=prefix+"-results_k:{0}".format(k),ofile_sep=",") + num_cat_arrays += 1 + + # add model run command + pf.mod_sys_cmds.append("mf6") + print(pf.mult_files) + print(pf.org_files) + + # build pest + pst = pf.build_pst('freyberg.pst') + #cov = pf.build_prior(fmt="none") + #cov.to_coo(os.path.join(template_ws, "prior.jcb")) + pst.try_parse_name_metadata() - import numpy as np - import pandas as pd - # pd.set_option('display.max_rows', 500) - # pd.set_option('display.max_columns', 500) - # pd.set_option('display.width', 1000) - # try: - import flopy - # except: - # return + pst.control_data.noptmax = 0 + pst.pestpp_options["additional_ins_delimiters"] = "," - org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') - tmp_model_ws = setup_tmp(org_model_ws, tmp_path) + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + res_file = os.path.join(pf.new_d, "freyberg.base.rei") + assert os.path.exists(res_file), res_file + pst.set_res(res_file) + print(pst.phi) + assert pst.phi < 0.1, pst.phi - tmp_model_ws = tmp_model_ws.relative_to(tmp_path) - bd = os.getcwd() - os.chdir(tmp_path) - try: - sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) - m = sim.get_model("freyberg6") - sim.set_all_data_external() - sim.write_simulation() + #set the initial and bounds for the fill values + par = pst.parameter_data + cat1par = par.loc[par.apply(lambda x: x.threshcat=="0" and x.usecol=="threshfill",axis=1),"parnme"] + cat2par = par.loc[par.apply(lambda x: x.threshcat == "1" and x.usecol == "threshfill", axis=1), "parnme"] + print(cat1par,cat2par) + assert cat1par.shape[0] == num_cat_arrays + assert cat2par.shape[0] == num_cat_arrays + + cat1parhk = [p for p in cat1par if "k:1" not in p] + cat2parhk = [p for p in cat2par if "k:1" not in p] + cat1parvk = [p for p in cat1par if "k:1" in p] + cat2parvk = [p for p in cat2par if "k:1" in p] + for lst in [cat2parvk,cat2parhk,cat1parhk,cat1parvk]: + assert len(lst) > 0 + par.loc[cat1parhk,"parval1"] = 0.1 + par.loc[cat1parhk, "parubnd"] = 1.0 + par.loc[cat1parhk, "parlbnd"] = 0.01 + par.loc[cat1parhk, "partrans"] = "log" + par.loc[cat2parhk, "parval1"] = 10 + par.loc[cat2parhk, "parubnd"] = 100 + par.loc[cat2parhk, "parlbnd"] = 1 + par.loc[cat2parhk, "partrans"] = "log" + + par.loc[cat1parvk, "parval1"] = 0.0001 + par.loc[cat1parvk, "parubnd"] = 0.01 + par.loc[cat1parvk, "parlbnd"] = 0.00001 + par.loc[cat1parvk, "partrans"] = "log" + par.loc[cat2parvk, "parval1"] = 0.1 + par.loc[cat2parvk, "parubnd"] = 1 + par.loc[cat2parvk, "parlbnd"] = 0.01 + par.loc[cat2parvk, "partrans"] = "log" + + + cat1par = par.loc[par.apply(lambda x: x.threshcat == "0" and x.usecol == "threshproportion", axis=1), "parnme"] + cat2par = par.loc[par.apply(lambda x: x.threshcat == "1" and x.usecol == "threshproportion", axis=1), "parnme"] + + print(cat1par, cat2par) + assert cat1par.shape[0] == num_cat_arrays + assert cat2par.shape[0] == num_cat_arrays + + par.loc[cat1par, "parval1"] = 0.5 + par.loc[cat1par, "parubnd"] = 1.0 + par.loc[cat1par, "parlbnd"] = 0.0001 + par.loc[cat1par,"partrans"] = "none" + + # since the apply method only looks that first proportion, we can just fix this one + par.loc[cat2par, "parval1"] = 1 + par.loc[cat2par, "parubnd"] = 1 + par.loc[cat2par, "parlbnd"] = 1 + par.loc[cat2par,"partrans"] = "fixed" + + assert par.loc[par.parnme.str.contains("threshgr"),:].shape[0] > 0 + #par.loc[par.parnme.str.contains("threshgr"),"parval1"] = 0.5 + par.loc[par.parnme.str.contains("threshgr"),"partrans"] = "fixed" + + print(pst.adj_par_names) + print(pst.npar,pst.npar_adj) + + org_par = par.copy() + num_reals = 100 + np.random.seed() + pe = pf.draw(num_reals, use_specsim=False) + pe.enforce() + print(pe.shape) + assert pe.shape[1] == pst.npar_adj, "{0} vs {1}".format(pe.shape[1], pst.npar_adj) + assert pe.shape[0] == num_reals + # cat1par = cat1par[0] + # pe = pe.loc[pe.loc[:,cat1par].values>0.35,:] + # pe = pe.loc[pe.loc[:, cat1par].values < 0.5, :] + # cat2par = par.loc[par.apply(lambda x: x.threshcat == "1" and x.usecol == "threshfill", axis=1), "parnme"] + # cat2par = cat2par[0] + # pe = pe.loc[pe.loc[:, cat2par].values > 10, :] + # pe = pe.loc[pe.loc[:, cat2par].values < 50, :] + + print(pe.shape) + assert pe.shape[0] > 0 + #print(pe.loc[:,cat1par].describe()) + #print(pe.loc[:, cat2par].describe()) + #return + truth_idx = pe.index[0] + pe = pe.loc[pe.index.map(lambda x: x != truth_idx),:] + pe.to_dense(os.path.join(template_ws, "prior.jcb")) + + + # just use a real as the truth... + pst.parameter_data.loc[pst.adj_par_names,"parval1"] = pe.loc[pe.index[1],pst.adj_par_names].values + pst.control_data.noptmax = 0 + pst.write(os.path.join(pf.new_d,"truth.pst"),version=2) + pyemu.os_utils.run("{0} truth.pst".format(ies_exe_path),cwd=pf.new_d) - # SETUP pest stuff... - os_utils.run("{0} ".format("mf6"), cwd=tmp_model_ws) + pst = pyemu.Pst(os.path.join(pf.new_d,"truth.pst")) - template_ws = Path(tmp_path, "new_temp_thresh") - if os.path.exists(template_ws): - shutil.rmtree(template_ws) - sr = m.modelgrid - # set up PstFrom object - pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, - remove_existing=True, - longnames=True, spatial_reference=sr, - zero_based=False, start_datetime="1-1-2018") + obs = pst.observation_data + obs["weight"] = 1.0 + obs["obsval"] = pst.res.loc[pst.obs_names,"modelled"].values + pst.write(os.path.join(pf.new_d, "truth2.pst"), version=2) + pyemu.os_utils.run("{0} truth2.pst".format(ies_exe_path), cwd=pf.new_d) + pst2 = pyemu.Pst(os.path.join(pf.new_d, "truth2.pst")) + print(pst2.phi) + assert pst2.phi < 0.1 + + obs.loc[:,"weight"] = 0.0 + obs.loc[:,"standard_deviation"] = np.nan + onames = obs.loc[obs.obsnme.apply(lambda x: ("trgw" in x or "gage" in x) and ("hdstd" not in x and "sfrtd" not in x)),"obsnme"].values + #obs.loc[obs.oname=="hds","weight"] = 1.0 + #obs.loc[obs.oname == "hds", "standard_deviation"] = 0.001 + snames = [o for o in onames if "gage" in o] + obs.loc[onames,"weight"] = 1.0 + obs.loc[snames,"weight"] = 1./(obs.loc[snames,"obsval"] * 0.2).values + #obs.loc[onames,"obsval"] = truth.values + #obs.loc[onames,"obsval"] *= np.random.normal(1.0,0.01,onames.shape[0]) + + pst.write(os.path.join(pf.new_d, "freyberg.pst"),version=2) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + pst = pyemu.Pst(os.path.join(pf.new_d,"freyberg.pst")) + assert pst.phi < 0.01,str(pst.phi) + + # reset away from the truth... + pst.parameter_data.loc[:,"parval1"] = org_par.parval1.values.copy() + + pst.control_data.noptmax = 2 + pst.pestpp_options["ies_par_en"] = "prior.jcb" + pst.pestpp_options["ies_num_reals"] = 30 + pst.pestpp_options["ies_subset_size"] = -10 + pst.pestpp_options["ies_no_noise"] = True + #pst.pestpp_options["ies_bad_phi_sigma"] = 2.0 + pst.pestpp_options["overdue_giveup_fac"] = 100.0 + #pst.pestpp_options["panther_agent_freeze_on_fail"] = True + + #pst.write(os.path.join(pf.new_d, "freyberg.pst")) + #pyemu.os_utils.start_workers(pf.new_d,ies_exe_path,"freyberg.pst",worker_root=".",master_dir="master_thresh",num_workers=15) + + #num_reals = 100 + #pe = pf.draw(num_reals, use_specsim=False) + #pe.enforce() + #pe.to_dense(os.path.join(template_ws, "prior.jcb")) + #pst.pestpp_options["ies_par_en"] = "prior.jcb" + + pst.write(os.path.join(pf.new_d, "freyberg.pst"), version=2) + m_d = "master_thresh" + port = _get_port() + pyemu.os_utils.start_workers(pf.new_d, ies_exe_path, "freyberg.pst", + worker_root=".", master_dir=m_d, num_workers=10, + port=port) + phidf = pd.read_csv(os.path.join(m_d,"freyberg.phi.actual.csv")) + print(phidf["mean"]) - df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0) - pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", - use_cols=list(df.columns.values), - prefix="hds", rebuild_pst=True) + assert phidf["mean"].min() < phidf["mean"].max() - # Add stream flow observation - # df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) - pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", - use_cols=["GAGE_1", "HEADWATER", "TAILWATER"], ofile_sep=",") - - # Setup geostruct for spatial pars - gr_v = pyemu.geostats.ExpVario(contribution=1.0, a=500) - gr_gs = pyemu.geostats.GeoStruct(variograms=gr_v, transform="log") - pp_v = pyemu.geostats.ExpVario(contribution=1.0, a=1000) - pp_gs = pyemu.geostats.GeoStruct(variograms=pp_v, transform="log") - rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60)) - pf.extra_py_imports.append('flopy') - ib = m.dis.idomain[0].array - #tags = {"npf_k_": [0.1, 10.], "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1], - # "rch_recharge": [.5, 1.5]} - tags = {"npf_k_": [0.1, 10.],"rch_recharge": [.5, 1.5]} - dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") - print(dts) - # ib = m.dis.idomain.array[0,:,:] - # setup from array style pars - num_cat_arrays = 0 - for tag, bnd in tags.items(): - lb, ub = bnd[0], bnd[1] - arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] - if "rch" in tag: - for arr_file in arr_files: - # indy direct grid pars for each array type file - recharge_files = ["recharge_1.txt", "recharge_2.txt", "recharge_3.txt"] - pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base="rch_gr", - pargp="rch_gr", zone_array=ib, upper_bound=1.0e-3, lower_bound=1.0e-7, - par_style="direct") - # additional constant mults - kper = int(arr_file.split('.')[1].split('_')[-1]) - 1 - pf.add_parameters(filenames=arr_file, par_type="constant", - par_name_base=arr_file.split('.')[1] + "_cn", - pargp="rch_const", zone_array=ib, upper_bound=ub, lower_bound=lb, - geostruct=rch_temporal_gs, - datetime=dts[kper]) - else: - for arr_file in arr_files: - print(arr_file) - k = int(arr_file.split(".")[1][-1]) - 1 - if k == 1: - arr_file = arr_file.replace("_k_","_k33_") - pth_arr_file = os.path.join(pf.new_d,arr_file) - arr = np.loadtxt(pth_arr_file) - cat_dict = {1:[0.4,arr.mean()],2:[0.6,arr.mean()]} - thresharr,threshcsv = pyemu.helpers.setup_threshold_pars(pth_arr_file,cat_dict=cat_dict, - testing_workspace=pf.new_d,inact_arr=ib) - - pf.pre_py_cmds.append("pyemu.helpers.apply_threshold_pars('{0}')".format(os.path.split(threshcsv)[1])) - prefix = arr_file.split('.')[1].replace("_","-") - pf.add_parameters(filenames=os.path.split(thresharr)[1],par_type="grid",transform="none", - par_name_base=prefix+"-threshgr_k:{0}".format(k), - pargp=prefix + "-threshgr_k:{0}".format(k), - lower_bound=0.0,upper_bound=1.0,geostruct=gr_gs,par_style="d") - - - pf.add_parameters(filenames=os.path.split(thresharr)[1],par_type="pilotpoints",transform="none", - par_name_base=prefix+"-threshpp_k:{0}".format(k), - pargp=prefix + "-threshpp_k:{0}".format(k), - lower_bound=0.0,upper_bound=2.0,geostruct=pp_gs,par_style="m", - pp_space=3 - ) - - - pf.add_parameters(filenames=os.path.split(threshcsv)[1], par_type="grid",index_cols=["threshcat"], - use_cols=["threshproportion","threshfill"], - par_name_base=[prefix+"threshproportion_k:{0}".format(k),prefix+"threshfill_k:{0}".format(k)], - pargp=[prefix+"threshproportion_k:{0}".format(k),prefix+"threshfill_k:{0}".format(k)], - lower_bound=[0.1,0.1],upper_bound=[10.0,10.0],transform="none",par_style='d') - - pf.add_observations(arr_file,prefix="hkarr-"+prefix+"_k:{0}".format(k), - obsgp="hkarr-"+prefix+"_k:{0}".format(k),zone_array=ib) - - pf.add_observations(arr_file+".threshcat.dat", prefix="tcatarr-" + prefix+"_k:{0}".format(k), - obsgp="tcatarr-" + prefix+"_k:{0}".format(k),zone_array=ib) - - pf.add_observations(arr_file + ".thresharr.dat", - prefix="tarr-" +prefix+"_k:{0}".format(k), - obsgp="tarr-" + prefix + "_k:{0}".format(k), zone_array=ib) - - df = pd.read_csv(threshcsv.replace(".csv","_results.csv"),index_col=0) - pf.add_observations(os.path.split(threshcsv)[1].replace(".csv","_results.csv"),index_cols="threshcat",use_cols=df.columns.tolist(),prefix=prefix+"-results_k:{0}".format(k), - obsgp=prefix+"-results_k:{0}".format(k),ofile_sep=",") - num_cat_arrays += 1 - - # add model run command - pf.mod_sys_cmds.append("mf6") - print(pf.mult_files) - print(pf.org_files) - - # build pest - pst = pf.build_pst('freyberg.pst') - #cov = pf.build_prior(fmt="none") - #cov.to_coo(os.path.join(template_ws, "prior.jcb")) - pst.try_parse_name_metadata() - - pst.control_data.noptmax = 0 - pst.pestpp_options["additional_ins_delimiters"] = "," - - pst.write(os.path.join(pf.new_d, "freyberg.pst")) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - - res_file = os.path.join(pf.new_d, "freyberg.base.rei") - assert os.path.exists(res_file), res_file - pst.set_res(res_file) - print(pst.phi) - assert pst.phi < 0.1, pst.phi - - #set the initial and bounds for the fill values - par = pst.parameter_data - cat1par = par.loc[par.apply(lambda x: x.threshcat=="0" and x.usecol=="threshfill",axis=1),"parnme"] - cat2par = par.loc[par.apply(lambda x: x.threshcat == "1" and x.usecol == "threshfill", axis=1), "parnme"] - print(cat1par,cat2par) - assert cat1par.shape[0] == num_cat_arrays - assert cat2par.shape[0] == num_cat_arrays - - cat1parhk = [p for p in cat1par if "k:1" not in p] - cat2parhk = [p for p in cat2par if "k:1" not in p] - cat1parvk = [p for p in cat1par if "k:1" in p] - cat2parvk = [p for p in cat2par if "k:1" in p] - for lst in [cat2parvk,cat2parhk,cat1parhk,cat1parvk]: - assert len(lst) > 0 - par.loc[cat1parhk,"parval1"] = 0.1 - par.loc[cat1parhk, "parubnd"] = 1.0 - par.loc[cat1parhk, "parlbnd"] = 0.01 - par.loc[cat1parhk, "partrans"] = "log" - par.loc[cat2parhk, "parval1"] = 10 - par.loc[cat2parhk, "parubnd"] = 100 - par.loc[cat2parhk, "parlbnd"] = 1 - par.loc[cat2parhk, "partrans"] = "log" - - par.loc[cat1parvk, "parval1"] = 0.0001 - par.loc[cat1parvk, "parubnd"] = 0.01 - par.loc[cat1parvk, "parlbnd"] = 0.00001 - par.loc[cat1parvk, "partrans"] = "log" - par.loc[cat2parvk, "parval1"] = 0.1 - par.loc[cat2parvk, "parubnd"] = 1 - par.loc[cat2parvk, "parlbnd"] = 0.01 - par.loc[cat2parvk, "partrans"] = "log" - - - cat1par = par.loc[par.apply(lambda x: x.threshcat == "0" and x.usecol == "threshproportion", axis=1), "parnme"] - cat2par = par.loc[par.apply(lambda x: x.threshcat == "1" and x.usecol == "threshproportion", axis=1), "parnme"] - - print(cat1par, cat2par) - assert cat1par.shape[0] == num_cat_arrays - assert cat2par.shape[0] == num_cat_arrays - - par.loc[cat1par, "parval1"] = 0.5 - par.loc[cat1par, "parubnd"] = 1.0 - par.loc[cat1par, "parlbnd"] = 0.0 - par.loc[cat1par,"partrans"] = "none" - - # since the apply method only looks that first proportion, we can just fix this one - par.loc[cat2par, "parval1"] = 1 - par.loc[cat2par, "parubnd"] = 1 - par.loc[cat2par, "parlbnd"] = 1 - par.loc[cat2par,"partrans"] = "fixed" - - assert par.loc[par.parnme.str.contains("threshgr"),:].shape[0] > 0 - #par.loc[par.parnme.str.contains("threshgr"),"parval1"] = 0.5 - par.loc[par.parnme.str.contains("threshgr"),"partrans"] = "fixed" - - print(pst.adj_par_names) - print(pst.npar,pst.npar_adj) - - org_par = par.copy() - num_reals = 100 - pe = pf.draw(num_reals, use_specsim=False) - pe.enforce() - print(pe.shape) - assert pe.shape[1] == pst.npar_adj, "{0} vs {1}".format(pe.shape[1], pst.npar_adj) - assert pe.shape[0] == num_reals - - # cat1par = cat1par[0] - # pe = pe.loc[pe.loc[:,cat1par].values>0.35,:] - # pe = pe.loc[pe.loc[:, cat1par].values < 0.5, :] - # cat2par = par.loc[par.apply(lambda x: x.threshcat == "1" and x.usecol == "threshfill", axis=1), "parnme"] - # cat2par = cat2par[0] - # pe = pe.loc[pe.loc[:, cat2par].values > 10, :] - # pe = pe.loc[pe.loc[:, cat2par].values < 50, :] - - print(pe.shape) - assert pe.shape[0] > 0 - #print(pe.loc[:,cat1par].describe()) - #print(pe.loc[:, cat2par].describe()) - #return - truth_idx = pe.index[0] - pe = pe.loc[pe.index.map(lambda x: x != truth_idx),:] - pe.to_dense(os.path.join(template_ws, "prior.jcb")) - - - # just use a real as the truth... - pst.parameter_data.loc[pst.adj_par_names,"parval1"] = pe.loc[pe.index[0],pst.adj_par_names].values - pst.control_data.noptmax = 0 - pst.write(os.path.join(pf.new_d,"truth.pst"),version=2) - pyemu.os_utils.run("{0} truth.pst".format(ies_exe_path),cwd=pf.new_d) - - pst = pyemu.Pst(os.path.join(pf.new_d,"truth.pst")) - - obs = pst.observation_data - obs.loc[:,"obsval"] = pst.res.loc[pst.obs_names,"modelled"].values - obs.loc[:,"weight"] = 0.0 - obs.loc[:,"standard_deviation"] = np.nan - onames = obs.loc[obs.obsnme.apply(lambda x: ("trgw" in x or "gage" in x) and ("hdstd" not in x and "sfrtd" not in x)),"obsnme"].values - #obs.loc[obs.oname=="hds","weight"] = 1.0 - #obs.loc[obs.oname == "hds", "standard_deviation"] = 0.001 - snames = [o for o in onames if "gage" in o] - obs.loc[onames,"weight"] = 1.0 - obs.loc[snames,"weight"] = 1./(obs.loc[snames,"obsval"] * 0.2).values - #obs.loc[onames,"obsval"] = truth.values - #obs.loc[onames,"obsval"] *= np.random.normal(1.0,0.01,onames.shape[0]) - - pst.write(os.path.join(pf.new_d, "freyberg.pst"),version=2) - pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) - pst = pyemu.Pst(os.path.join(pf.new_d,"freyberg.pst")) - assert pst.phi < 0.01,str(pst.phi) - - # reset away from the truth... - pst.parameter_data.loc[:,"parval1"] = org_par.parval1.values.copy() - - pst.control_data.noptmax = 2 - pst.pestpp_options["ies_par_en"] = "prior.jcb" - pst.pestpp_options["ies_num_reals"] = 30 - pst.pestpp_options["ies_subset_size"] = -10 - pst.pestpp_options["ies_no_noise"] = True - #pst.pestpp_options["ies_bad_phi_sigma"] = 2.0 - pst.pestpp_options["overdue_giveup_fac"] = 100.0 - #pst.pestpp_options["panther_agent_freeze_on_fail"] = True - - #pst.write(os.path.join(pf.new_d, "freyberg.pst")) - #pyemu.os_utils.start_workers(pf.new_d,ies_exe_path,"freyberg.pst",worker_root=".",master_dir="master_thresh",num_workers=15) - - #num_reals = 100 - #pe = pf.draw(num_reals, use_specsim=False) - #pe.enforce() - #pe.to_dense(os.path.join(template_ws, "prior.jcb")) - #pst.pestpp_options["ies_par_en"] = "prior.jcb" - - pst.write(os.path.join(pf.new_d, "freyberg.pst"), version=2) - m_d = "master_thresh" - pyemu.os_utils.start_workers(pf.new_d, ies_exe_path, "freyberg.pst", worker_root=".", master_dir=m_d, - num_workers=10) - phidf = pd.read_csv(os.path.join(m_d,"freyberg.phi.actual.csv")) - print(phidf["mean"]) + #pst.pestpp_options["ies_multimodal_alpha"] = 0.99 - assert phidf["mean"].min() < phidf["mean"].max() + #pst.pestpp_options["ies_num_threads"] = 6 + #pst.write(os.path.join(pf.new_d, "freyberg.pst"),version=2) - #pst.pestpp_options["ies_multimodal_alpha"] = 0.99 - - #pst.pestpp_options["ies_num_threads"] = 6 - #pst.write(os.path.join(pf.new_d, "freyberg.pst"),version=2) + #pyemu.os_utils.start_workers(pf.new_d, ies_exe_path, "freyberg.pst", worker_root=".", master_dir="master_thresh_mm", + # num_workers=40) - #pyemu.os_utils.start_workers(pf.new_d, ies_exe_path, "freyberg.pst", worker_root=".", master_dir="master_thresh_mm", - # num_workers=40) - except Exception as e: - os.chdir(bd) - raise Exception(e) - os.chdir(bd) def plot_thresh(m_d): import flopy @@ -5237,11 +5156,10 @@ def plot_thresh(m_d): print(mn, mx) - import matplotlib.pyplot as plt from matplotlib.backends.backend_pdf import PdfPages - with PdfPages(os.path.join(m_d,"results_{0}_hk_layer_{1}.pdf".format(iiter,k+1))) as pdf: + with PdfPages(os.path.join(m_d,"results_{0}_layer_{1}.pdf".format(iiter,k+1))) as pdf: ireal = 0 #for real in pr_oe.index: for real in reals_to_plot: @@ -5277,7 +5195,7 @@ def plot_thresh(m_d): ptarr[ib == 0] = np.nan cb = axes[1,0].imshow(prarr,vmin=cmn,vmax=cmx,cmap="plasma") plt.colorbar(cb, ax=axes[1,0]) - cb = axes[1,1].imshow(prarr,vmin=cmn,vmax=cmx,cmap="plasma") + cb = axes[1,1].imshow(ptarr,vmin=cmn,vmax=cmx,cmap="plasma") plt.colorbar(cb,ax=axes[1,1]) cb = axes[1,2].imshow(tcarray, vmin=cmn,vmax=cmx,cmap="plasma") plt.colorbar(cb,ax=axes[1,2]) @@ -5413,10 +5331,781 @@ def test_array_fmt_pst_from(tmp_path): arr3 = np.loadtxt(Path(tmp_path, "weird_tmp", "ar3.arr")) +def hyperpars_test(tmp_path): + tdir = os.getcwd() + pf, sim = setup_freyberg_mf6(tmp_path) + shutil.copy(Path(tdir, 'utils', 'ppuref.txt'), tmp_path) + m = sim.get_model() + template_ws = pf.new_d + + v = pyemu.geostats.ExpVario(contribution=1.0, a=5000) + pp_gs = pyemu.geostats.GeoStruct(variograms=v) + pf.extra_py_imports.append('flopy') + ib = m.dis.idomain[0].array + + xmn = m.modelgrid.xvertices.min() + xmx = m.modelgrid.xvertices.max() + ymn = m.modelgrid.yvertices.min() + ymx = m.modelgrid.yvertices.max() + + numpp = 20 + xvals = np.random.uniform(xmn, xmx, numpp) + yvals = np.random.uniform(ymn, ymx, numpp) + pp_locs = pd.DataFrame({"x": xvals, "y": yvals}) + pp_locs.loc[:, "zone"] = 1 + pp_locs.loc[:, "name"] = ["pp_{0}".format(i) for i in range(numpp)] + pp_locs.loc[:, "parval1"] = 1.0 + + # pp_locs = pyemu.pp_utils.setup_pilotpoints_grid(sr=sr,prefix_dict={0:"pps_1"}) + # pp_locs = pp_locs.loc[:,["name","x","y","zone","parval1"]] + # pp_locs.to_csv(os.path.join(template_ws, "pp.csv")) + value_v = pyemu.geostats.ExpVario(contribution=1, a=3000, anisotropy=5, bearing=0.0) + value_gs = pyemu.geostats.GeoStruct(variograms=value_v) + bearing_v = pyemu.geostats.ExpVario(contribution=1, a=10000, anisotropy=3, bearing=90.0) + bearing_gs = pyemu.geostats.GeoStruct(variograms=bearing_v) + aniso_v = pyemu.geostats.ExpVario(contribution=1, a=10000, anisotropy=3, bearing=45.0) + aniso_gs = pyemu.geostats.GeoStruct(variograms=aniso_v) + tag = "npf_k_" + bnd = [0.1, 10.] + lb, ub = bnd[0], bnd[1] + arr_files = [f for f in os.listdir(template_ws) if tag in f and f.endswith(".txt")] + for i, arr_file in enumerate(arr_files): + if 'layer1' not in arr_file: + continue + ppu_file = arr_file + ".ppu" + shutil.copy(os.path.join(template_ws,arr_file), os.path.join(template_ws, ppu_file)) + tag = arr_file.split('.')[1].replace("_", "-") + "_pp" + tagppu = tag+"-ppu" + ppdf = pf.add_parameters(filenames=arr_file, par_type="pilotpoints", + par_name_base=tag, + pargp=tag, zone_array=ib, + upper_bound=ub, lower_bound=lb, pp_space=4, + pp_options={"try_use_ppu": False, "prep_hyperpars": True}, + apply_order=2, geostruct=value_gs) + + pf.add_parameters(filenames=ppu_file, par_type="pilotpoints", + par_name_base=tagppu, + pargp=tagppu, zone_array=ib, + upper_bound=ub, lower_bound=lb, pp_space=4, + pp_options={"try_use_ppu": True, "prep_hyperpars": True}, + apply_order=2, geostruct=value_gs) + + tag = arr_file.split('.')[1].replace("_", "-") + tagppu = tag+"-ppu" + pf.add_observations(arr_file, prefix=tag + "input", obsgp=tag + "input") + pf.add_observations(ppu_file, prefix=tagppu + "input", obsgp=tagppu + "input") + + tfiles = [f for f in os.listdir(pf.new_d) if tag in f] + afile = [f for f in tfiles if "aniso" in f][0] + # pf.add_parameters(afile,par_type="pilotpoints",par_name_base=tag+"aniso", + # pargp=tag+"aniso",pp_space=5,lower_bound=-1.0,upper_bound=1.0, + # pp_options={"try_use_ppu":True},apply_order=1,geostruct=aniso_gs, + # par_style="a",transform="none",initial_value=0.0) + pf.add_parameters(afile, par_type="constant", par_name_base=tag + "aniso", + pargp=tag + "aniso", lower_bound=-1.0, upper_bound=1.0, + apply_order=1, + par_style="a", transform="none", initial_value=0.0) + pf.add_observations(afile, prefix=tag + "aniso", obsgp=tag + "aniso") + # one set of pilot point hyperpars across multiple bearing arrays + bfiles = [f for f in tfiles if "bearing" in f] + pf.add_parameters(bfiles, par_type="pilotpoints", par_name_base=tag + "bearing", + pargp=tag + "bearing", pp_space=5, lower_bound=-45, upper_bound=45, + par_style="a", transform="none", zone_array=ib, + pp_options={"try_use_ppu": True, "use_pp_zones":True}, + apply_order=1, geostruct=bearing_gs) + for f in bfiles: + if 'ppu' in f: + tagf = tagppu + else: + tagf = tag + pf.add_observations(f, prefix=tagf + "bearing", obsgp=tagf + "bearing") + + # build pest + pst = pf.build_pst('freyberg.pst') + + par = pst.parameter_data + apar = par.loc[par.pname.str.contains("aniso"), :] + bpar = par.loc[par.pname.str.contains("bearing"), :] + basepar = par.loc[par.pargp == "npf-k-layer1_pp", :] + testpar = par.loc[par.pargp == "npf-k-layer1_pp-ppu", :] + assert len(basepar)==len(testpar) + bpar[['i','j']] = bpar[['i','j']].astype(int) + basepar[['i', 'j']] = basepar[['i', 'j']].astype(int) + testpar[['i', 'j']] = testpar[['i', 'j']].astype(int) + + mid = ib.shape[1]//2 + mx = 100 + mn = 0.01 + c = mid / np.sqrt(-2 * np.log(mn / mx)) + # bearing as a function of column + par.loc[bpar.parnme,'parval1'] = -45+(90*(bpar.i-bpar.i.min())/(bpar.i.max()-bpar.i.min())).values + # pp mult as a function of column + par.loc[testpar.parnme, 'parval1'] = mx * np.exp(-((testpar.j - mid) ** 2) / (2 * c ** 2)) + par.loc[basepar.parnme, 'parval1'] = mx * np.exp(-((basepar.j - mid) ** 2) / (2 * c ** 2)) + + check_apply(pf) + + # testing + # start by checking that arrays are equiv -- they should be as the bearing hyperpar files were set up with the same + # pars + ppuk = np.loadtxt(os.path.join(template_ws, "freyberg6.npf_k_layer1.txt.ppu")) + assert np.isclose( + ppuk, + np.loadtxt(os.path.join(template_ws, "freyberg6.npf_k_layer1.txt")) + ).all() + # check against reference file + # org file = ppu + refk = np.loadtxt(Path(tmp_path, 'ppuref.txt')) + assert np.isclose( + ppuk, + refk).all() + + bearef = np.loadtxt(Path(template_ws, "npf-k-layer1_pp-ppu.bearing.dat")) + + # switch bearing + par.loc[bpar.parnme,'parval1'] = 45-(90*(bpar.i-bpar.i.min())/(bpar.i.max()-bpar.i.min())).values + check_apply(pf) + + bear2 = np.loadtxt(Path(template_ws, "npf-k-layer1_pp-ppu.bearing.dat")) + assert (bearef == bear2*-1).all() + + ppuk2 = np.loadtxt(os.path.join(template_ws, "freyberg6.npf_k_layer1.txt.ppu")) + assert (ppuk != ppuk2*-1).all() + + # mg = m.modelgrid + # from matplotlib import use + # import matplotlib.pyplot as plt + # use('qt5agg') + # fig, axs = plt.subplots(1,3, figsize=(10,5), layout='constrained') + # for ax, f in zip(axs, ["npf-k-layer1_pp-ppu.aniso.dat", "npf-k-layer1_pp-ppu.bearing.dat", + # "freyberg6.npf_k_layer1.txt"]): + # if f == "freyberg6.npf_k_layer1.txt": + # from matplotlib.colors import LogNorm + # norm = LogNorm() + # ppf = "npf-k-layer1_pp-ppu_inst0pp.dat" + # else: + # norm = None + # ppf = "npf-k-layer1bearing_inst0pp.dat" + # ppdf = pd.read_csv(Path(template_ws, ppf), + # names=['ppname', 'x', 'y', 'zone', 'val'], + # sep=r'\s+').set_index('ppname') + # ppdf[['i', 'j']] = pd.DataFrame(ppdf.apply(lambda x: mg.intersect(x.x, x.y), axis=1).to_list(), index=ppdf.index) + # im = ax.imshow(np.loadtxt(Path(template_ws, f)), norm=norm) + # plt.colorbar(im, ax=ax) + # ax.scatter(ppdf.j, ppdf.i, marker='o', fc='none', ec='r') + # ax.set_title(Path(f).stem) + + +def mf6_freyberg_ppu_hyperpars_invest(tmp_path): + import numpy as np + import pandas as pd + import os + import matplotlib.pyplot as plt + + import pyemu + + # sys.path.insert(0,os.path.join("..","..","pypestutils")) + import pypestutils as ppu + + pd.set_option('display.max_rows', 500) + pd.set_option('display.max_columns', 500) + pd.set_option('display.width', 1000) + try: + import flopy + except: + return + + org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') + tmp_model_ws = setup_tmp(org_model_ws, tmp_path) + + os.chdir(tmp_path) + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + m = sim.get_model("freyberg6") + sim.set_all_data_external(check_data=False) + sim.write_simulation() + + # SETUP pest stuff... + os_utils.run("{0} ".format(mf6_exe_path), cwd=tmp_model_ws) + + template_ws = "new_temp" + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + sr = m.modelgrid + # set up PstFrom object + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + remove_existing=True, + longnames=True, spatial_reference=sr, + zero_based=False, start_datetime="1-1-2018", + chunk_len=10) + + wfiles = [f for f in os.listdir(pf.new_d) if ".wel_stress_period_data_" in f and f.endswith(".txt")] + pf.add_parameters(wfiles,par_type='grid',index_cols=[0,1,2],use_cols=[3],pargp="welgrid",par_name_base="welgrid", + upper_bound=1.2,lower_bound=0.8) + + + # pf.post_py_cmds.append("generic_function()") + df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) + pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", use_cols=list(df.columns.values)) + v = pyemu.geostats.ExpVario(contribution=1.0, a=5000) + pp_gs = pyemu.geostats.GeoStruct(variograms=v) + pf.extra_py_imports.append('flopy') + ib = m.dis.idomain[0].array + tags = {"npf_k_": [0.1, 10.]}#, "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1], + # "rch_recharge": [.5, 1.5]} + dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") + print(dts) + + xmn = m.modelgrid.xvertices.min() + xmx = m.modelgrid.xvertices.max() + ymn = m.modelgrid.yvertices.min() + ymx = m.modelgrid.yvertices.max() + + numpp = 20 + xvals = np.random.uniform(xmn,xmx,numpp) + yvals = np.random.uniform(ymn, ymx, numpp) + pp_locs = pd.DataFrame({"x":xvals,"y":yvals}) + pp_locs.loc[:,"zone"] = 1 + pp_locs.loc[:,"name"] = ["pp_{0}".format(i) for i in range(numpp)] + pp_locs.loc[:,"parval1"] = 1.0 + + pyemu.pp_utils.write_pp_shapfile(pp_locs,os.path.join(template_ws,"pp_locs.shp")) + df = pyemu.pp_utils.pilot_points_from_shapefile(os.path.join(template_ws,"pp_locs.shp")) + + #pp_locs = pyemu.pp_utils.setup_pilotpoints_grid(sr=sr,prefix_dict={0:"pps_1"}) + #pp_locs = pp_locs.loc[:,["name","x","y","zone","parval1"]] + pp_locs.to_csv(os.path.join(template_ws,"pp.csv")) + pyemu.pp_utils.write_pp_file(os.path.join(template_ws,"pp_file.dat"),pp_locs) + pp_container = ["pp_file.dat","pp.csv","pp_locs.shp"] + value_v = pyemu.geostats.ExpVario(contribution=1, a=5000, anisotropy=5, bearing=0.0) + value_gs = pyemu.geostats.GeoStruct(variograms=value_v) + bearing_v = pyemu.geostats.ExpVario(contribution=1,a=10000,anisotropy=3,bearing=90.0) + bearing_gs = pyemu.geostats.GeoStruct(variograms=bearing_v) + aniso_v = pyemu.geostats.ExpVario(contribution=1, a=10000, anisotropy=3, bearing=45.0) + aniso_gs = pyemu.geostats.GeoStruct(variograms=aniso_v) + for tag, bnd in tags.items(): + lb, ub = bnd[0], bnd[1] + arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] + if "rch" in tag: + pass + else: + for i,arr_file in enumerate(arr_files): + if i < len(pp_container): + pp_opt = pp_container[i] + else: + pp_opt = pp_locs + + pth_arr_file = os.path.join(pf.new_d,arr_file) + arr = np.loadtxt(pth_arr_file) + + tag = arr_file.split('.')[1].replace("_","-") + "_pp" + + pf.add_parameters(filenames=arr_file, par_type="pilotpoints", + par_name_base=tag, + pargp=tag, zone_array=ib, + upper_bound=ub, lower_bound=lb,pp_space=5, + pp_options={"try_use_ppu":False,"prep_hyperpars":True}, + apply_order=2,geostruct=value_gs) + + + + tag = arr_file.split('.')[1].replace("_","-") + pf.add_observations(arr_file,prefix=tag+"input",obsgp=tag+"input") + tfiles = [f for f in os.listdir(pf.new_d) if tag in f] + afile = [f for f in tfiles if "aniso" in f][0] + # pf.add_parameters(afile,par_type="pilotpoints",par_name_base=tag+"aniso", + # pargp=tag+"aniso",pp_space=5,lower_bound=-1.0,upper_bound=1.0, + # pp_options={"try_use_ppu":True},apply_order=1,geostruct=aniso_gs, + # par_style="a",transform="none",initial_value=0.0) + pf.add_parameters(afile,par_type="constant",par_name_base=tag+"aniso", + pargp=tag+"aniso",lower_bound=-1.0,upper_bound=1.0, + apply_order=1, + par_style="a",transform="none",initial_value=0.0) + pf.add_observations(afile, prefix=tag+"aniso", obsgp=tag+"aniso") + bfile = [f for f in tfiles if "bearing" in f][0] + pf.add_parameters(bfile, par_type="pilotpoints", par_name_base=tag + "bearing", + pargp=tag + "bearing", pp_space=6,lower_bound=-45,upper_bound=45, + par_style="a",transform="none", + pp_options={"try_use_ppu":True}, + apply_order=1,geostruct=bearing_gs) + pf.add_observations(bfile, prefix=tag + "bearing", obsgp=tag + "bearing") + + + + # this is just for local prelim testing + pf.pre_py_cmds.insert(0,"import sys") + pf.pre_py_cmds.insert(1,"sys.path.append(os.path.join('..','..', '..', 'pypestutils'))") + + + # add model run command + pf.mod_sys_cmds.append("mf6") + print(pf.mult_files) + print(pf.org_files) + + # build pest + pst = pf.build_pst('freyberg.pst') + + pst.control_data.noptmax = 0 + pst.write(os.path.join(template_ws, "freyberg.pst"),version=2) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path),cwd=template_ws) + pst = pyemu.Pst(os.path.join(template_ws, "freyberg.pst")) + + + par = pst.parameter_data + apar = par.loc[par.pname.str.contains("aniso"),:] + bpar = par.loc[par.pname.str.contains("bearing"), :] + par.loc[apar.parnme,"parval1"] = 0 + par.loc[apar.parnme,"parlbnd"] = -2 + par.loc[apar.parnme,"parubnd"] = 2 + + par.loc[bpar.parnme,"parval1"] = 90 + par.loc[bpar.parnme,"parlbnd"] = -75 + par.loc[bpar.parnme,"parubnd"] = 75 + + + num_reals = 10 + pe = pf.draw(num_reals, use_specsim=True) + pe.to_binary(os.path.join(template_ws, "prior.jcb")) + pst.parameter_data["parval1"] = pe._df.loc[pe.index[0],pst.par_names] + + pst.control_data.noptmax = 0 + pst.write(os.path.join(template_ws, "freyberg.pst"),version=2) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=template_ws) + pst = pyemu.Pst(os.path.join(template_ws, "freyberg.pst")) + + obs = pst.observation_data + aobs = obs.loc[obs.otype=="arr",:].copy() + aobs["i"] = aobs.i.astype(int) + aobs["j"] = aobs.j.astype(int) + hk0 = aobs.loc[aobs.oname=="npf-k-layer1input",:].copy() + bearing0 = aobs.loc[aobs.oname=="npf-k-layer1bearing",:].copy() + aniso0 = aobs.loc[aobs.oname == "npf-k-layer1aniso", :].copy() + fig,axes = plt.subplots(1,3,figsize=(10,5)) + nrow = hk0.i.max()+1 + ncol = hk0.j.max()+1 + for ax,name,df in zip(axes,["aniso","bearing","input"],[aniso0,bearing0,hk0]): + arr = np.zeros((nrow,ncol)) + arr[df.i,df.j] = pst.res.loc[df.obsnme,"modelled"].values + if name == "input": + arr = np.log10(arr) + cb = ax.imshow(arr) + plt.colorbar(cb,ax=ax) + ax.set_title(name,loc="left") + plt.tight_layout() + plt.savefig("hyper.pdf") + plt.close(fig) + + pst.pestpp_options["ies_par_en"] = "prior.jcb" + pst.control_data.noptmax = -1 + pst.write(os.path.join(template_ws,"freyberg.pst")) + + #pyemu.os_utils.run("{0} freyberg.pst".format("pestpp-glm"),cwd=template_ws) + m_d = "master_ies" + port = _get_port() + print(f"Running ies on port: {port}") + pyemu.os_utils.start_workers(template_ws,ies_exe_path,"freyberg.pst",num_workers=5, + worker_root=tmp_path, + master_dir=m_d, port=port) + + +def mf6_freyberg_ppu_hyperpars_thresh_invest(tmp_path): + + import numpy as np + import pandas as pd + # pd.set_option('display.max_rows', 500) + # pd.set_option('display.max_columns', 500) + # pd.set_option('display.width', 1000) + # try: + import flopy + # except: + # return + # import sys + # sys.path.insert(0,os.path.join("..","..","pypestutils")) + + import pypestutils as ppu + + org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') + tmp_model_ws = setup_tmp(org_model_ws, tmp_path) + + + tmp_model_ws = tmp_model_ws.relative_to(tmp_path) + + os.chdir(tmp_path) + sim = flopy.mf6.MFSimulation.load(sim_ws=str(tmp_model_ws)) + m = sim.get_model("freyberg6") + sim.set_all_data_external() + sim.write_simulation() + + # SETUP pest stuff... + os_utils.run("{0} ".format("mf6"), cwd=tmp_model_ws) + + template_ws = Path(tmp_path, "new_temp_thresh") + if os.path.exists(template_ws): + shutil.rmtree(template_ws) + sr = m.modelgrid + # set up PstFrom object + pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws, + remove_existing=True, + longnames=True, spatial_reference=sr, + zero_based=False, start_datetime="1-1-2018") + + pf.pre_py_cmds.insert(0,"import sys") + pf.pre_py_cmds.insert(1,"sys.path.append(os.path.join('..','..', '..', 'pypestutils'))") + + + df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0) + pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", + use_cols=list(df.columns.values), + prefix="hds", rebuild_pst=True) + + # Add stream flow observation + # df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0) + pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", + use_cols=["GAGE_1", "HEADWATER", "TAILWATER"], ofile_sep=",") + + # Setup geostruct for spatial pars + pf.extra_py_imports.append('flopy') + ib = m.dis.idomain[0].array + #tags = {"npf_k_": [0.1, 10.], "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1], + # "rch_recharge": [.5, 1.5]} + tags = {"npf_k_": [0.1, 10.]} + dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") + print(dts) + # ib = m.dis.idomain.array[0,:,:] + # setup from array style pars + pf.extra_py_imports.append('flopy') + ib = m.dis.idomain[0].array + tags = {"npf_k_": [0.1, 10.]}#, "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1], + # "rch_recharge": [.5, 1.5]} + dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d") + print(dts) + + xmn = m.modelgrid.xvertices.min() + xmx = m.modelgrid.xvertices.max() + ymn = m.modelgrid.yvertices.min() + ymx = m.modelgrid.yvertices.max() + + numpp = 20 + xvals = np.random.uniform(xmn,xmx,numpp) + yvals = np.random.uniform(ymn, ymx, numpp) + pp_locs = pd.DataFrame({"x":xvals,"y":yvals}) + pp_locs.loc[:,"zone"] = 1 + pp_locs.loc[:,"name"] = ["pp_{0}".format(i) for i in range(numpp)] + pp_locs.loc[:,"parval1"] = 1.0 + + pyemu.pp_utils.write_pp_shapfile(pp_locs,os.path.join(template_ws,"pp_locs.shp")) + df = pyemu.pp_utils.pilot_points_from_shapefile(os.path.join(template_ws,"pp_locs.shp")) + + #pp_locs = pyemu.pp_utils.setup_pilotpoints_grid(sr=sr,prefix_dict={0:"pps_1"}) + #pp_locs = pp_locs.loc[:,["name","x","y","zone","parval1"]] + pp_locs.to_csv(os.path.join(template_ws,"pp.csv")) + pyemu.pp_utils.write_pp_file(os.path.join(template_ws,"pp_file.dat"),pp_locs) + pp_container = ["pp_file.dat","pp.csv","pp_locs.shp"] + value_v = pyemu.geostats.ExpVario(contribution=1, a=5000, anisotropy=5, bearing=0.0) + value_gs = pyemu.geostats.GeoStruct(variograms=value_v) + bearing_v = pyemu.geostats.ExpVario(contribution=1,a=5000,anisotropy=3,bearing=0.0) + bearing_gs = pyemu.geostats.GeoStruct(variograms=bearing_v) + aniso_v = pyemu.geostats.ExpVario(contribution=1, a=10000, anisotropy=3, bearing=45.0) + aniso_gs = pyemu.geostats.GeoStruct(variograms=aniso_v) + num_cat_arrays = 0 + for tag, bnd in tags.items(): + lb, ub = bnd[0], bnd[1] + arr_files = [f for f in os.listdir(tmp_model_ws) if tag in f and f.endswith(".txt")] + if "rch" in tag: + pass + else: + for i,arr_file in enumerate(arr_files): + print(arr_file) + k = int(arr_file.split(".")[1][-1]) - 1 + if k == 1: + arr_file = arr_file.replace("_k_","_k33_") + pth_arr_file = os.path.join(pf.new_d,arr_file) + arr = np.loadtxt(pth_arr_file) + cat_dict = {1:[0.4,arr.mean()],2:[0.6,arr.mean()]} + thresharr,threshcsv = pyemu.helpers.setup_threshold_pars(pth_arr_file,cat_dict=cat_dict, + testing_workspace=pf.new_d,inact_arr=ib) + + pf.pre_py_cmds.append("pyemu.helpers.apply_threshold_pars('{0}')".format(os.path.split(threshcsv)[1])) + prefix = arr_file.split('.')[1].replace("_","-") + + + if i < len(pp_container): + pp_opt = pp_container[i] + else: + pp_opt = pp_locs + + pth_arr_file = os.path.join(pf.new_d,arr_file) + arr = np.loadtxt(pth_arr_file) + + tag = arr_file.split('.')[1].replace("_","-") + "_pp" + prefix = arr_file.split('.')[1].replace("_","-") + + pf.add_parameters(filenames=os.path.split(thresharr)[1],par_type="pilotpoints",transform="none", + par_name_base=tag+"-threshpp_k:{0}".format(k), + pargp=tag + "-threshpp_k:{0}".format(k), + lower_bound=0.0,upper_bound=2.0,par_style="m", + pp_space=5,pp_options={"try_use_ppu":False,"prep_hyperpars":True}, + apply_order=2,geostruct=value_gs + ) + + tag = arr_file.split('.')[1].replace("_","-") + + tfiles = [f for f in os.listdir(pf.new_d) if tag in f] + afile = [f for f in tfiles if "aniso" in f][0] + # pf.add_parameters(afile,par_type="pilotpoints",par_name_base=tag+"aniso", + # pargp=tag+"aniso",pp_space=5,lower_bound=-1.0,upper_bound=1.0, + # pp_options={"try_use_ppu":True},apply_order=1,geostruct=aniso_gs, + # par_style="a",transform="none",initial_value=0.0) + pf.add_parameters(afile,par_type="constant",par_name_base=tag+"aniso", + pargp=tag+"aniso",lower_bound=-1.0,upper_bound=1.0, + apply_order=1, + par_style="a",transform="none",initial_value=0.0) + pf.add_observations(afile, prefix=tag+"aniso", obsgp=tag+"aniso") + bfile = [f for f in tfiles if "bearing" in f][0] + pf.add_parameters(bfile, par_type="pilotpoints", par_name_base=tag + "bearing", + pargp=tag + "bearing", pp_space=6,lower_bound=-45,upper_bound=45, + par_style="a",transform="none", + pp_options={"try_use_ppu":True}, + apply_order=1,geostruct=bearing_gs) + pf.add_observations(bfile, prefix=tag + "bearing", obsgp=tag + "bearing") + + + pf.add_parameters(filenames=os.path.split(threshcsv)[1], par_type="grid",index_cols=["threshcat"], + use_cols=["threshproportion","threshfill"], + par_name_base=[prefix+"threshproportion_k:{0}".format(k),prefix+"threshfill_k:{0}".format(k)], + pargp=[prefix+"threshproportion_k:{0}".format(k),prefix+"threshfill_k:{0}".format(k)], + lower_bound=[0.1,0.1],upper_bound=[10.0,10.0],transform="none",par_style='d') + + pf.add_observations(arr_file,prefix="hkarr-"+prefix+"_k:{0}".format(k), + obsgp="hkarr-"+prefix+"_k:{0}".format(k),zone_array=ib) + + pf.add_observations(arr_file+".threshcat.dat", prefix="tcatarr-" + prefix+"_k:{0}".format(k), + obsgp="tcatarr-" + prefix+"_k:{0}".format(k),zone_array=ib) + + pf.add_observations(arr_file + ".thresharr.dat", + prefix="tarr-" +prefix+"_k:{0}".format(k), + obsgp="tarr-" + prefix + "_k:{0}".format(k), zone_array=ib) + + df = pd.read_csv(threshcsv.replace(".csv","_results.csv"),index_col=0) + pf.add_observations(os.path.split(threshcsv)[1].replace(".csv","_results.csv"),index_cols="threshcat",use_cols=df.columns.tolist(),prefix=prefix+"-results_k:{0}".format(k), + obsgp=prefix+"-results_k:{0}".format(k),ofile_sep=",") + num_cat_arrays += 1 + + # add model run command + pf.mod_sys_cmds.append("mf6") + print(pf.mult_files) + print(pf.org_files) + + # build pest + pst = pf.build_pst('freyberg.pst') + #cov = pf.build_prior(fmt="none") + #cov.to_coo(os.path.join(template_ws, "prior.jcb")) + pst.try_parse_name_metadata() + + pst.control_data.noptmax = 0 + pst.pestpp_options["additional_ins_delimiters"] = "," + + pst.write(os.path.join(pf.new_d, "freyberg.pst")) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + + res_file = os.path.join(pf.new_d, "freyberg.base.rei") + assert os.path.exists(res_file), res_file + pst.set_res(res_file) + print(pst.phi) + #assert pst.phi < 0.1, pst.phi + + #set the initial and bounds for the fill values + par = pst.parameter_data + + apar = par.loc[par.pname.str.contains("aniso"),:] + bpar = par.loc[par.pname.str.contains("bearing"), :] + par.loc[apar.parnme,"parval1"] = 5 + par.loc[apar.parnme,"parlbnd"] = 4 + par.loc[apar.parnme,"parubnd"] = 6 + + par.loc[bpar.parnme,"parval1"] = 0 + par.loc[bpar.parnme,"parlbnd"] = -90 + par.loc[bpar.parnme,"parubnd"] = 90 + + cat1par = par.loc[par.apply(lambda x: x.threshcat=="0" and x.usecol=="threshfill",axis=1),"parnme"] + cat2par = par.loc[par.apply(lambda x: x.threshcat == "1" and x.usecol == "threshfill", axis=1), "parnme"] + print(cat1par,cat2par) + assert cat1par.shape[0] == num_cat_arrays + assert cat2par.shape[0] == num_cat_arrays + + cat1parhk = [p for p in cat1par if "k:1" not in p] + cat2parhk = [p for p in cat2par if "k:1" not in p] + cat1parvk = [p for p in cat1par if "k:1" in p] + cat2parvk = [p for p in cat2par if "k:1" in p] + for lst in [cat2parvk,cat2parhk,cat1parhk,cat1parvk]: + assert len(lst) > 0 + par.loc[cat1parhk,"parval1"] = 0.1 + par.loc[cat1parhk, "parubnd"] = 1.0 + par.loc[cat1parhk, "parlbnd"] = 0.01 + par.loc[cat1parhk, "partrans"] = "log" + par.loc[cat2parhk, "parval1"] = 10 + par.loc[cat2parhk, "parubnd"] = 100 + par.loc[cat2parhk, "parlbnd"] = 1 + par.loc[cat2parhk, "partrans"] = "log" + + par.loc[cat1parvk, "parval1"] = 0.0001 + par.loc[cat1parvk, "parubnd"] = 0.01 + par.loc[cat1parvk, "parlbnd"] = 0.000001 + par.loc[cat1parvk, "partrans"] = "log" + par.loc[cat2parvk, "parval1"] = 0.1 + par.loc[cat2parvk, "parubnd"] = 1 + par.loc[cat2parvk, "parlbnd"] = 0.01 + par.loc[cat2parvk, "partrans"] = "log" + + + cat1par = par.loc[par.apply(lambda x: x.threshcat == "0" and x.usecol == "threshproportion", axis=1), "parnme"] + cat2par = par.loc[par.apply(lambda x: x.threshcat == "1" and x.usecol == "threshproportion", axis=1), "parnme"] + + print(cat1par, cat2par) + assert cat1par.shape[0] == num_cat_arrays + assert cat2par.shape[0] == num_cat_arrays + + par.loc[cat1par, "parval1"] = 0.8 + par.loc[cat1par, "parubnd"] = 1.0 + par.loc[cat1par, "parlbnd"] = 0.6 + par.loc[cat1par,"partrans"] = "none" + + # since the apply method only looks that first proportion, we can just fix this one + par.loc[cat2par, "parval1"] = 1 + par.loc[cat2par, "parubnd"] = 1 + par.loc[cat2par, "parlbnd"] = 1 + par.loc[cat2par,"partrans"] = "fixed" + + #assert par.loc[par.parnme.str.contains("threshgr"),:].shape[0] > 0 + #par.loc[par.parnme.str.contains("threshgr"),"parval1"] = 0.5 + #par.loc[par.parnme.str.contains("threshgr"),"partrans"] = "fixed" + + print(pst.adj_par_names) + print(pst.npar,pst.npar_adj) + + org_par = par.copy() + num_reals = 100 + pe = pf.draw(num_reals, use_specsim=False) + pe.enforce() + print(pe.shape) + assert pe.shape[1] == pst.npar_adj, "{0} vs {1}".format(pe.shape[1], pst.npar_adj) + assert pe.shape[0] == num_reals + + # cat1par = cat1par[0] + # pe = pe.loc[pe.loc[:,cat1par].values>0.35,:] + # pe = pe.loc[pe.loc[:, cat1par].values < 0.5, :] + # cat2par = par.loc[par.apply(lambda x: x.threshcat == "1" and x.usecol == "threshfill", axis=1), "parnme"] + # cat2par = cat2par[0] + # pe = pe.loc[pe.loc[:, cat2par].values > 10, :] + # pe = pe.loc[pe.loc[:, cat2par].values < 50, :] + + print(pe.shape) + assert pe.shape[0] > 0 + #print(pe.loc[:,cat1par].describe()) + #print(pe.loc[:, cat2par].describe()) + #return + #pe._df.index = np.arange(pe.shape[0]) + + truth_idx = 1 + + + #pe = pe._df + + # just use a real as the truth... + pst.parameter_data.loc[pst.adj_par_names,"parval1"] = pe.loc[truth_idx,pst.adj_par_names].values + pe = pe.loc[pe.index.map(lambda x: x != truth_idx), :] + pe.to_dense(os.path.join(template_ws, "prior.jcb")) + pst.control_data.noptmax = 0 + pst.write(os.path.join(pf.new_d,"truth.pst"),version=2) + pyemu.os_utils.run("{0} truth.pst".format(ies_exe_path),cwd=pf.new_d) + + pst = pyemu.Pst(os.path.join(pf.new_d,"truth.pst")) + + obs = pst.observation_data + obs.loc[:,"obsval"] = pst.res.loc[pst.obs_names,"modelled"].values + obs.loc[:,"weight"] = 0.0 + obs.loc[:,"standard_deviation"] = np.nan + onames = obs.loc[obs.obsnme.apply(lambda x: ("trgw" in x or "gage" in x) and ("hdstd" not in x and "sfrtd" not in x)),"obsnme"].values + #obs.loc[obs.oname=="hds","weight"] = 1.0 + #obs.loc[obs.oname == "hds", "standard_deviation"] = 0.001 + snames = [o for o in onames if "gage" in o] + obs.loc[onames,"weight"] = 1.0 + obs.loc[onames, "standard_deviation"] = 0.5 + + obs.loc[snames,"weight"] = 1./(obs.loc[snames,"obsval"] * 0.2).values + obs.loc[snames, "standard_deviation"] = (obs.loc[snames, "obsval"] * 0.2).values + + #obs.loc[onames,"obsval"] = truth.values + #obs.loc[onames,"obsval"] *= np.random.normal(1.0,0.01,onames.shape[0]) + + pst.write(os.path.join(pf.new_d, "freyberg.pst"),version=2) + pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) + pst = pyemu.Pst(os.path.join(pf.new_d,"freyberg.pst")) + assert pst.phi < 0.01,str(pst.phi) + + # reset away from the truth... + pst.parameter_data.loc[:,"parval1"] = org_par.parval1.values.copy() + + pst.control_data.noptmax = 1 + pst.pestpp_options["ies_par_en"] = "prior.jcb" + pst.pestpp_options["ies_num_reals"] = 30 + pst.pestpp_options["ies_subset_size"] = -10 + pst.pestpp_options["ies_no_noise"] = True + #pst.pestpp_options["ies_bad_phi_sigma"] = 2.0 + pst.pestpp_options["overdue_giveup_fac"] = 100.0 + pst.pestpp_options["ies_multimodal_alpha"] = 0.99 + #pst.pestpp_options["panther_agent_freeze_on_fail"] = True + + #pst.write(os.path.join(pf.new_d, "freyberg.pst")) + #pyemu.os_utils.start_workers(pf.new_d,ies_exe_path,"freyberg.pst",worker_root=".",master_dir="master_thresh",num_workers=15) + + #num_reals = 100 + #pe = pf.draw(num_reals, use_specsim=False) + #pe.enforce() + #pe.to_dense(os.path.join(template_ws, "prior.jcb")) + #pst.pestpp_options["ies_par_en"] = "prior.jcb" + + pst.write(os.path.join(pf.new_d, "freyberg.pst"), version=2) + m_d = "master_thresh_nonstat" + port = _get_port() + pyemu.os_utils.start_workers(pf.new_d, ies_exe_path, "freyberg.pst", + worker_root=".", master_dir=m_d, + num_workers=15, port=port) + phidf = pd.read_csv(os.path.join(m_d,"freyberg.phi.actual.csv")) + print(phidf["mean"]) + + assert phidf["mean"].min() < phidf["mean"].max() + + + + # pst.pestpp_options["ies_n_iter_mean"] = 3 + # + # #pst.pestpp_options["ies_num_threads"] = 6 + # pst.write(os.path.join(pf.new_d, "freyberg.pst"),version=2) + # + # m_d = "master_thresh_nonstat_nim" + # pyemu.os_utils.start_workers(pf.new_d, ies_exe_path, "freyberg.pst", worker_root=".", master_dir=m_d, + # num_workers=15) + # phidf = pd.read_csv(os.path.join(m_d, "freyberg.phi.actual.csv")) + # print(phidf["mean"]) + # + # assert phidf["mean"].min() < phidf["mean"].max() + + if __name__ == "__main__": - shortname_conversion_test('.') - #mf6_freyberg_pp_locs_test() + #mf6_freyberg_pp_locs_test('.') + #mf6_subdir_test(".") + mf6_freyberg_ppu_hyperpars_invest(".") + # mf6_freyberg_ppu_hyperpars_thresh_invest(".") + #while True: + # mf6_freyberg_thresh_test(".") + plot_thresh("master_thresh_nonstat") + #mf6_freyberg_thresh_test(".") + plot_thresh("master_thresh_nonstat") + #plot_thresh("master_thresh_nonstat_nim") + # invest() + # test_add_array_parameters_pps_grid() #freyberg_test(os.path.abspath(".")) # freyberg_prior_build_test() #mf6_freyberg_test(os.path.abspath(".")) @@ -5424,9 +6113,10 @@ def test_array_fmt_pst_from(tmp_path): #shortname_conversion_test() #mf6_freyberg_shortnames_test() #mf6_freyberg_direct_test() - + #freyberg_test() #mf6_freyberg_thresh_test(".") - + #mf6_freyberg_test() + #test_defaults(".") #plot_thresh("master_thresh") #plot_thresh("master_thresh_mm") #mf6_freyberg_varying_idomain() @@ -5434,12 +6124,12 @@ def test_array_fmt_pst_from(tmp_path): # mf6_freyberg_short_direct_test() # mf6_add_various_obs_test() # mf6_subdir_test() - # tpf = TestPstFrom() - # tpf.setup() + #tpf = TestPstFrom() + #tpf.setup() #tpf.test_add_array_parameters_to_file_list() #tpf.test_add_array_parameters_alt_inst_str_none_m() #tpf.test_add_array_parameters_alt_inst_str_0_d() - # tpf.test_add_array_parameters_pps_grid() + #tpf.test_add_array_parameters_pps_grid() # tpf.test_add_list_parameters() # # pstfrom_profile() # mf6_freyberg_arr_obs_and_headerless_test() diff --git a/autotest/utils/ppuref.txt b/autotest/utils/ppuref.txt new file mode 100644 index 00000000..7791b870 --- /dev/null +++ b/autotest/utils/ppuref.txt @@ -0,0 +1,40 @@ +1.107487509999999897e+00 2.550984639999999803e+00 8.386606260000000646e+00 1.994052569999999847e+01 3.748867829999999657e+01 5.994425280000000100e+01 8.401623779999999897e+01 1.182541506000000027e+02 1.649234331000000111e+02 1.227448394999999977e+02 7.371864419999999996e+01 5.710688549999999708e+01 5.146821539999999828e+01 2.054877092999999988e+01 6.149409930000000024e+00 2.890124319999999969e+00 1.594617100000000010e+00 1.563269360000000052e+00 2.075287030000000144e+00 2.713501299999999894e+00 +1.231858349999999991e+00 9.797188500000000300e-01 3.544896989999999803e+00 1.189278417000000054e+01 2.770735769999999931e+01 5.082825629999999961e+01 7.680062519999999893e+01 1.012246142999999989e+02 1.476524024999999938e+02 2.199106461000000081e+02 1.230768831000000034e+02 6.661432890000000384e+01 5.344702920000000290e+01 5.820296609999999760e+01 1.619411129999999943e+01 5.145764820000000128e+00 2.224519030000000175e+00 1.165974870000000108e+00 1.420115110000000014e+00 2.030848460000000077e+00 +1.427373409999999954e+00 1.469238330000000037e+00 8.262686999999999671e-01 5.375326410000000443e+00 1.713476070000000107e+01 3.874522890000000075e+01 6.872603999999999758e+01 9.697395509999999774e+01 1.174736759999999975e+02 1.638728576999999973e+02 3.000000000000000000e+02 1.120726331999999985e+02 5.515900740000000013e+01 4.326853469999999646e+01 6.872603999999999758e+01 1.110971301000000011e+01 3.909305550000000018e+00 1.797106359999999903e+00 8.262686999999999671e-01 1.375301110000000104e+00 +1.113946110000000100e+00 1.648161180000000003e+00 1.933734880000000045e+00 2.525847189999999909e+00 1.111107185999999913e+01 2.833677392999999967e+01 5.638500030000000152e+01 9.960853470000000698e+01 1.276578947999999940e+02 1.300695363000000100e+02 1.631717337000000043e+02 2.058824549999999931e+02 8.846409149999999499e+01 4.122849719999999962e+01 3.011996969999999862e+01 2.220288767999999990e+01 5.459246010000000204e+00 2.463426880000000097e+00 1.359589120000000095e+00 9.658675500000000191e-01 +9.627601599999999760e-01 1.167213870000000098e+00 2.020697909999999986e+00 2.914622200000000163e+00 7.707454470000000057e+00 2.694227018999999856e+01 5.051846520000000140e+01 8.370591389999999876e+01 1.470353028000000108e+02 1.648063299000000086e+02 1.380910433999999896e+02 1.484497599000000037e+02 1.415161268999999891e+02 5.982330540000000241e+01 2.701107557999999997e+01 1.858893272999999979e+01 7.084547820000000051e+00 2.328978250000000028e+00 1.546572190000000013e+00 1.111213840000000008e+00 +1.138250629999999930e+00 8.929058699999999904e-01 1.452767329999999912e+00 2.837446209999999969e+00 5.483022570000000151e+00 2.344242699000000130e+01 5.842947569999999757e+01 8.485669679999999460e+01 1.212048989999999975e+02 2.124950961000000120e+02 1.866490124999999978e+02 1.338249356999999975e+02 1.182588972000000069e+02 9.691924349999999322e+01 3.384406500000000051e+01 1.557009732000000035e+01 9.446490510000000285e+00 2.320083050000000036e+00 1.286207139999999916e+00 1.169326220000000083e+00 +1.093964500000000006e+00 1.134433889999999945e+00 8.262686999999999671e-01 2.270561199999999946e+00 4.806112650000000208e+00 1.349091143999999964e+01 6.872603999999999758e+01 1.007830766999999952e+02 1.282958678999999904e+02 1.646042073000000130e+02 3.000000000000000000e+02 1.723125750000000096e+02 1.132148006999999978e+02 8.168332859999999584e+01 6.872603999999999758e+01 1.567359858000000017e+01 8.022608939999999578e+00 3.864652439999999967e+00 8.262686999999999671e-01 1.061917510000000009e+00 +1.034022320000000050e+00 1.112194539999999954e+00 1.202669739999999932e+00 1.708320840000000063e+00 4.134142620000000434e+00 1.034467193999999957e+01 3.747415680000000293e+01 1.026186881999999940e+02 1.523239232999999899e+02 1.702724244000000056e+02 2.059661886000000095e+02 2.135288390999999990e+02 1.389947868000000142e+02 8.211596699999999771e+01 4.750212510000000066e+01 2.212991348999999985e+01 6.520053479999999624e+00 3.676692570000000160e+00 1.594971659999999902e+00 8.909076200000000112e-01 +1.087838620000000089e+00 9.452440800000000420e-01 1.349390969999999967e+00 1.574763799999999936e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 8.745209880000000169e+01 1.502937641999999983e+02 2.097334839000000102e+02 1.968027126000000067e+02 2.086740881999999999e+02 1.463595449999999971e+02 1.011086432999999971e+02 4.398080070000000319e+01 2.270793261000000030e+01 7.496462519999999685e+00 2.577922399999999836e+00 1.699025039999999986e+00 1.004252509999999932e+00 +1.107873179999999902e+00 9.569348300000000140e-01 1.120959999999999956e+00 1.838007769999999930e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.463767421999999954e+02 2.137628765999999985e+02 2.303897948999999983e+02 1.991643497999999965e+02 1.512176546999999971e+02 1.027031147999999945e+02 5.166605460000000249e+01 2.026561767000000103e+01 7.960838129999999957e+00 2.574733929999999837e+00 1.310608250000000030e+00 1.104087990000000019e+00 +1.054446269999999908e+00 1.032426489999999975e+00 8.262686999999999671e-01 1.781902010000000036e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.429220417999999881e+02 2.058877332000000138e+02 3.000000000000000000e+02 2.052380417999999906e+02 1.585156575000000032e+02 1.003200401999999940e+02 6.872603999999999758e+01 2.064387516000000034e+01 7.379913509999999732e+00 2.807441949999999853e+00 8.262686999999999671e-01 1.041702029999999946e+00 +1.089921600000000046e+00 9.435886399999999785e-01 1.068588639999999979e+00 1.715893190000000068e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.345619844000000001e+02 2.118777992999999924e+02 2.544721728000000098e+02 2.190212616000000025e+02 1.599371213999999952e+02 1.058946377999999982e+02 5.738571329999999904e+01 2.180429519999999854e+01 7.224343980000000442e+00 2.514908020000000022e+00 1.124962919999999977e+00 9.207932100000000286e-01 +1.120980920000000047e+00 9.074172999999999822e-01 1.111572699999999969e+00 1.791341110000000070e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.393529934000000026e+02 2.205101442000000134e+02 2.523716714999999908e+02 2.256955470000000048e+02 1.658050062000000082e+02 1.132154960999999957e+02 5.706147570000000258e+01 2.247813057000000114e+01 7.192402590000000373e+00 2.307209939999999904e+00 1.141232369999999996e+00 8.868301800000000235e-01 +1.206350339999999965e+00 9.266813699999999487e-01 9.875572000000000239e-01 1.934326360000000022e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.364074502999999936e+02 2.196890190000000018e+02 2.718350348999999824e+02 2.259979026000000033e+02 1.665450932999999907e+02 1.119678344999999950e+02 6.217714500000000299e+01 2.292450653999999943e+01 7.041606749999999693e+00 2.290785889999999991e+00 9.941355200000000503e-01 9.177876299999999654e-01 +1.360903299999999927e+00 9.561493899999999879e-01 8.262686999999999671e-01 2.049232410000000115e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.320969042000000115e+02 2.195275589999999966e+02 3.000000000000000000e+02 2.311908294000000126e+02 1.636846668000000022e+02 1.070259951000000029e+02 6.872603999999999758e+01 2.307927765000000164e+01 7.078787010000000102e+00 2.226076429999999995e+00 8.262686999999999671e-01 9.463110200000000027e-01 +1.580131649999999999e+00 1.001359950000000110e+00 9.145146499999999845e-01 2.017541899999999888e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.385671499999999980e+02 2.230608225000000004e+02 2.831623961999999892e+02 2.307962793000000090e+02 1.613732730000000117e+02 1.067524542000000025e+02 6.538762470000000349e+01 2.393115345000000005e+01 6.993671609999999816e+00 2.042176099999999828e+00 9.168682999999999694e-01 9.854190000000000449e-01 +1.652741690000000041e+00 1.035575609999999980e+00 9.132334199999999624e-01 1.972874660000000002e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.397320580999999891e+02 2.210734059000000116e+02 2.813562870000000089e+02 2.335307055000000105e+02 1.632439458000000059e+02 1.095469547999999946e+02 6.542326020000000142e+01 2.461203360000000018e+01 6.843939090000000114e+00 2.026337690000000080e+00 9.137814600000000453e-01 1.019665870000000085e+00 +2.027271789999999907e+00 1.121283440000000020e+00 8.768958399999999820e-01 1.951659770000000016e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.339590636000000075e+02 1.989747021000000018e+02 2.889551324999999906e+02 2.283447357000000011e+02 1.569259935000000041e+02 1.079802018000000032e+02 6.685642140000000211e+01 2.537087543999999895e+01 6.914429550000000368e+00 2.023576610000000109e+00 8.763514399999999815e-01 1.117185700000000059e+00 +2.310617310000000035e+00 1.195847340000000036e+00 8.262686999999999671e-01 2.004443080000000155e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.065489041999999955e+02 1.900082123999999908e+02 3.000000000000000000e+02 2.195519007000000045e+02 1.423840745999999911e+02 1.015256511000000046e+02 6.872603999999999758e+01 2.474388542999999885e+01 7.106137979999999743e+00 2.049193409999999993e+00 8.262686999999999671e-01 1.176400260000000086e+00 +2.493133090000000163e+00 1.245619849999999973e+00 8.624167799999999939e-01 1.916547640000000108e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.151146944000000047e+02 1.971884102999999868e+02 2.924870528999999806e+02 2.187444270000000017e+02 1.426114838999999961e+02 1.023620451000000031e+02 6.750544050000000595e+01 2.468039927999999961e+01 7.220097540000000258e+00 2.043502489999999838e+00 8.625247499999999512e-01 1.220278640000000081e+00 +2.504015330000000095e+00 1.248590239999999962e+00 8.716692600000000013e-01 1.861640119999999898e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 1.254857568000000043e+02 2.042563686000000018e+02 2.901501425999999810e+02 2.216246003999999914e+02 1.465581372000000044e+02 1.077603615000000019e+02 6.710595179999999971e+01 2.473948083000000153e+01 7.246391520000000419e+00 2.066030050000000173e+00 8.758368100000000211e-01 1.231998589999999894e+00 +2.277209050000000179e+00 1.259812759999999976e+00 8.633287499999999781e-01 1.806217170000000038e+00 3.000000000000000000e+00 3.000000000000000000e+00 5.906417909999999694e+01 9.205124969999999962e+01 1.260744701999999933e+02 2.058897212999999908e+02 2.921272068000000104e+02 2.160391332000000091e+02 1.415981619000000080e+02 1.039983126000000055e+02 6.733045559999999341e+01 2.465653452000000101e+01 7.084219830000000329e+00 2.017194939999999992e+00 8.663384100000000032e-01 1.172165169999999979e+00 +2.254378089999999890e+00 1.206150159999999971e+00 8.262686999999999671e-01 1.702433819999999987e+00 5.260442040000000041e+00 3.000000000000000000e+00 6.872603999999999758e+01 9.380806259999999952e+01 1.381889748000000111e+02 2.159912885999999901e+02 3.000000000000000000e+02 2.205076098000000115e+02 1.468724142000000086e+02 1.023999929999999949e+02 6.872603999999999758e+01 2.618462832000000162e+01 6.765032760000000422e+00 1.970134880000000033e+00 8.262686999999999671e-01 1.107080899999999923e+00 +1.929175330000000077e+00 1.135562289999999974e+00 8.790182100000000220e-01 1.640128849999999971e+00 5.166546270000000440e+00 1.997754704999999831e+01 6.712286249999999654e+01 1.013317461000000037e+02 1.500723347999999930e+02 2.247004257000000109e+02 2.875446251999999845e+02 2.260390496999999925e+02 1.514678360999999995e+02 1.034865422999999964e+02 6.613360409999999945e+01 2.561590710000000115e+01 7.267241639999999947e+00 2.061189399999999949e+00 8.908050800000000269e-01 1.042595970000000039e+00 +1.679177579999999947e+00 1.080897989999999975e+00 9.139318499999999901e-01 1.615021290000000054e+00 5.405552280000000209e+00 2.198061792000000025e+01 6.600747599999999693e+01 1.060505720999999966e+02 1.606343033999999932e+02 2.318404778999999962e+02 2.776949627999999848e+02 2.297101383000000112e+02 1.500829722000000004e+02 1.076273915999999957e+02 6.367142189999999857e+01 2.470403819999999939e+01 7.092331680000000027e+00 2.062203709999999912e+00 9.456390799999999652e-01 1.026076070000000007e+00 +1.517706949999999999e+00 1.060695919999999903e+00 9.103959999999999830e-01 1.666828069999999995e+00 6.013200570000000411e+00 2.668323489000000137e+01 6.676053810000000510e+01 1.081862277000000034e+02 1.567392594000000088e+02 2.304213201000000026e+02 2.784741189000000077e+02 2.292816870000000051e+02 1.630801950000000033e+02 1.081639784999999989e+02 6.578222610000000259e+01 2.325696794999999995e+01 6.980921969999999810e+00 2.122479220000000222e+00 9.466466199999999942e-01 9.642788799999999494e-01 +1.477740199999999948e+00 1.048366520000000079e+00 8.262686999999999671e-01 1.793495030000000101e+00 7.419056580000000345e+00 3.081651060000000086e+01 6.872603999999999758e+01 1.048373279999999994e+02 1.594921472999999992e+02 2.241109170000000006e+02 3.000000000000000000e+02 2.237245379999999955e+02 1.650313946999999928e+02 1.081541298000000069e+02 6.872603999999999758e+01 2.339320632000000089e+01 7.051304909999999815e+00 2.319984040000000025e+00 8.262686999999999671e-01 9.518528499999999726e-01 +1.348509629999999904e+00 1.013185180000000019e+00 9.969646300000000183e-01 2.039207349999999863e+00 9.760457069999999291e+00 3.413712540000000217e+01 6.894377400000000478e+01 1.057326465000000013e+02 1.571701784999999916e+02 2.200004414999999938e+02 2.629378980000000183e+02 2.219532968999999980e+02 1.587809943000000032e+02 1.089098915999999946e+02 5.870094420000000213e+01 2.233858005000000091e+01 7.086142080000000121e+00 2.396927990000000008e+00 1.057752760000000070e+00 9.271659199999999768e-01 +1.271323030000000021e+00 9.712395499999999515e-01 1.156535059999999948e+00 2.452294109999999971e+00 1.283683337999999985e+01 3.754937009999999731e+01 6.917787930000000074e+01 1.045227267000000069e+02 1.519073018999999931e+02 2.189003361000000041e+02 2.410450098000000025e+02 2.237916912000000025e+02 1.575314946000000020e+02 1.073539884000000058e+02 5.215261379999999747e+01 2.259222786000000127e+01 7.260346769999999950e+00 2.329390899999999931e+00 1.258961320000000050e+00 8.988120400000000343e-01 +1.193762860000000092e+00 1.062186910000000095e+00 1.176040659999999960e+00 3.603099209999999886e+00 1.703323427999999851e+01 4.310893140000000301e+01 7.036777530000000525e+01 1.035005763000000059e+02 1.525699319999999943e+02 2.017488729000000092e+02 2.399078441999999995e+02 2.153269947000000002e+02 1.554290952000000061e+02 1.008297182999999961e+02 5.666890169999999927e+01 2.176190688000000151e+01 7.150512329999999750e+00 2.686818660000000136e+00 1.219023399999999979e+00 9.724501199999999734e-01 +1.203594770000000036e+00 1.217792699999999950e+00 8.262686999999999671e-01 6.421768740000000086e+00 2.334547901999999908e+01 4.755322739999999726e+01 6.872603999999999758e+01 1.060230047999999954e+02 1.475154575999999906e+02 1.873590120000000070e+02 3.000000000000000000e+02 1.898348067000000015e+02 1.510521045000000129e+02 9.394853849999999795e+01 6.872603999999999758e+01 1.954526381999999884e+01 7.445815890000000437e+00 2.997872740000000036e+00 8.262686999999999671e-01 1.157158310000000023e+00 +1.376840030000000104e+00 1.124733360000000015e+00 2.519892689999999824e+00 1.194498266999999991e+01 3.029209649999999954e+01 5.047497090000000242e+01 7.486827089999999885e+01 1.059318714000000057e+02 1.363272533999999894e+02 2.046150282000000118e+02 2.152305977999999982e+02 1.804614335999999923e+02 1.427447151000000076e+02 1.001767518000000052e+02 4.927839869999999678e+01 1.907643798000000146e+01 8.392868970000000317e+00 2.544878619999999980e+00 1.454220209999999902e+00 1.258309499999999970e+00 +3.000000000000000000e+00 1.645104020000000000e+00 6.755784659999999775e+00 1.954777839000000128e+01 3.688973759999999658e+01 5.534391810000000334e+01 7.926567149999999629e+01 9.857315069999999935e+01 1.422664388999999971e+02 1.969204418999999859e+02 1.779089405999999940e+02 1.976877981000000091e+02 1.442888748000000021e+02 9.450526859999999374e+01 4.122509699999999810e+01 2.267073969000000133e+01 7.454008049999999663e+00 2.688291029999999804e+00 2.034246270000000134e+00 1.136669589999999896e+00 +3.000000000000000000e+00 3.000000000000000000e+00 1.297888017000000005e+01 2.606707557000000008e+01 4.296647310000000175e+01 6.156554400000000271e+01 7.618541220000000180e+01 9.920147939999999664e+01 1.472237679000000128e+02 1.542458736000000101e+02 1.888420472999999902e+02 2.144875485000000026e+02 1.282931810999999982e+02 7.384932179999999846e+01 4.644826859999999868e+01 2.232821775000000031e+01 6.390994169999999919e+00 4.027279949999999609e+00 1.831187169999999975e+00 1.142539170000000048e+00 +3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.288721079999999830e+01 4.938550260000000236e+01 6.190669859999999858e+01 6.872603999999999758e+01 1.074476504999999946e+02 1.235757009000000011e+02 1.553445611999999869e+02 3.000000000000000000e+02 1.608423663000000090e+02 9.961779180000000622e+01 7.840712550000000647e+01 6.872603999999999758e+01 1.555172774999999952e+01 8.104468020000000550e+00 4.041138390000000413e+00 8.262686999999999671e-01 2.137373960000000128e+00 +3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 5.175492719999999736e+01 5.946211230000000114e+01 8.077442820000000268e+01 9.901055189999999584e+01 1.239209699999999970e+02 2.115059601000000100e+02 1.728202172999999959e+02 1.214097383999999948e+02 1.105559171999999961e+02 9.853831590000000062e+01 3.373539900000000102e+01 1.548863298000000022e+01 8.680177079999999989e+00 2.466370159999999867e+00 2.311277290000000040e+00 3.000000000000000000e+00 +3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 5.201451360000000079e+01 6.501478260000000375e+01 8.122905269999999689e+01 1.012074374999999975e+02 1.459429737000000102e+02 1.523059787999999912e+02 1.302218898000000138e+02 1.407950588999999866e+02 1.421175680999999997e+02 5.992233180000000203e+01 2.627389188000000075e+01 1.593389276999999993e+01 7.694550210000000057e+00 4.888040339999999873e+00 4.837210319999999619e+00 3.000000000000000000e+00 +3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 6.833074080000000095e+01 8.304244979999999998e+01 1.000786868999999939e+02 1.217485712999999947e+02 1.310458037999999874e+02 1.650409589999999866e+02 2.047106946000000107e+02 9.081187710000000379e+01 4.048704299999999989e+01 2.759646509999999964e+01 2.324359877999999924e+01 1.023117668999999985e+01 9.766423050000000217e+00 3.000000000000000000e+00 3.000000000000000000e+00 +3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 6.662142570000000319e+01 6.872603999999999758e+01 1.065115746000000030e+02 1.283030583000000036e+02 1.771988006999999925e+02 3.000000000000000000e+02 1.200086619000000070e+02 5.975044350000000293e+01 4.402333260000000337e+01 6.872603999999999758e+01 1.680235554000000064e+01 9.894171780000000638e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 +3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 6.193170810000000159e+01 9.028236959999999556e+01 1.201122048000000007e+02 1.613197236000000032e+02 2.396523567000000128e+02 1.445496615000000133e+02 7.832756700000000194e+01 6.042564899999999994e+01 6.474845039999999585e+01 2.407054806000000013e+01 1.178702919000000016e+01 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 3.000000000000000000e+00 diff --git a/autotest/utils_tests.py b/autotest/utils_tests.py index 4d34ff5e..39d89438 100644 --- a/autotest/utils_tests.py +++ b/autotest/utils_tests.py @@ -5,6 +5,7 @@ # os.mkdir("temp") from pathlib import Path import pyemu +from pst_from_tests import _get_port def add_pi_obj_func_test(tmp_path): import os @@ -391,6 +392,7 @@ def pp_to_shapefile_test(tmp_path): shutil.copy(o_pp_file, pp_file) shp_file = os.path.join(tmp_path, "points1.dat.shp") pyemu.pp_utils.write_pp_shapfile(pp_file, shp_file) + df = pyemu.pp_utils.pilot_points_from_shapefile(shp_file) def write_tpl_test(tmp_path): @@ -441,7 +443,7 @@ def master_and_workers(tmp_path): # not run?!? os.mkdir(master_dir) assert os.path.exists(worker_dir) pyemu.helpers.start_workers(worker_dir,"pestpp","pest.pst",1, - worker_root=tmp_path,master_dir=master_dir, port=4009) + worker_root=tmp_path,master_dir=master_dir, port=_get_port()) #now try it from within the master dir base_cwd = os.getcwd() @@ -449,7 +451,7 @@ def master_and_workers(tmp_path): # not run?!? worker_dir = Path(worker_dir).relative_to(master_dir) pyemu.helpers.start_workers(worker_dir, "pestpp","pest.pst",3, - master_dir='.', port=4009) + master_dir='.', port=_get_port()) os.chdir(base_cwd) @@ -2062,31 +2064,6 @@ def run_sp_success_test(): pyemu.os_utils.run("ls", use_sp=True, shell=True) assert True -def test_fake_frun_sp(setup_freyberg_mf6): - from pst_from_tests import ies_exe_path - pf, sim = setup_freyberg_mf6 - v = pyemu.geostats.ExpVario(contribution=1.0, a=500) - gs = pyemu.geostats.GeoStruct(variograms=v, transform='log') - pf.add_parameters( - "freyberg6.npf_k_layer1.txt", - par_type="grid", - geostruct=gs, - pargp=f"hk_k:{0}" - ) - pf.add_observations( - "heads.csv", - index_cols=['time'], - obsgp="head" - ) - pst = pf.build_pst() - pst = pyemu.utils.setup_fake_forward_run(pst, "fake.pst", pf.new_d, - new_cwd=pf.new_d) - pyemu.os_utils.run(f"{ies_exe_path} fake.pst", - use_sp=True, cwd=pf.new_d) - bd = Path.cwd() - os.chdir(pf.new_d) - pyemu.utils.calc_array_par_summary_stats("mult2model_info.csv") - def run_sp_failure_test(): with pytest.raises(Exception): pyemu.os_utils.run("junk_command", use_sp=True, @@ -2471,9 +2448,9 @@ def ac_draw_test(tmp_path): # plt.show() -def test_fake_frun(setup_freyberg_mf6): - from pst_from_tests import ies_exe_path - pf, sim = setup_freyberg_mf6 +def test_fake_frun(tmp_path): + from pst_from_tests import ies_exe_path, setup_freyberg_mf6 + pf, sim = setup_freyberg_mf6(tmp_path) v = pyemu.geostats.ExpVario(contribution=1.0, a=500) gs = pyemu.geostats.GeoStruct(variograms=v, transform='log') pf.add_parameters( @@ -2494,6 +2471,10 @@ def test_fake_frun(setup_freyberg_mf6): bd = Path.cwd() os.chdir(pf.new_d) pyemu.utils.calc_array_par_summary_stats("mult2model_info.csv") + os.chdir(bd) + pyemu.os_utils.run(f"{ies_exe_path} fake.pst", cwd=pf.new_d, use_sp=True) + os.chdir(pf.new_d) + pyemu.utils.calc_array_par_summary_stats("mult2model_info.csv") def obs_ensemble_quantile_test(): @@ -2522,6 +2503,9 @@ def thresh_pars_test(): arr = np.ones((dim,dim)) gs = pyemu.geostats.GeoStruct(variograms=[pyemu.geostats.ExpVario(1.0,30.0)]) ss = pyemu.geostats.SpecSim2d(np.ones(dim),np.ones(dim),gs) + #seed = np.random.randint(100000) + np.random.seed(9371) + #print("seed",seed) arr = 10**(ss.draw_arrays()[0]) print(arr) @@ -2543,13 +2527,14 @@ def thresh_pars_test(): print(np.unique(newarr)) tarr = np.zeros_like(newarr) - tarr[np.isclose(newarr,cat_dict[1][1])] = 1.0 + tarr[np.isclose(newarr,cat_dict[1][1],rtol=1e-5,atol=1e-5)] = 1.0 #tarr[inact_arr==0] = np.nan tot = inact_arr.sum() prop = np.nansum(tarr) / tot print(prop,cat_dict[1]) print(np.nansum(tarr),tot) - assert np.isclose(prop,cat_dict[1][0],0.01),"cat_dict 1,{0} vs {1}, tot:{2}, prop:{3}".format(prop,cat_dict[1],tot,np.nansum(tarr)) + if not np.isclose(prop,cat_dict[1][0],0.01): + print("cat_dict 1,{0} vs {1}, tot:{2}, prop:{3}".format(prop,cat_dict[1],tot,np.nansum(tarr))) tarr = np.zeros_like(newarr) tarr[np.isclose(newarr, cat_dict[2][1])] = 1.0 @@ -2570,9 +2555,75 @@ def thresh_pars_test(): # plt.show() +def test_ppu_import(): + import pypestutils as ppu + + + +def ppu_geostats_test(tmp_path): + import sys + import os + import numpy as np + import matplotlib.pyplot as plt + import pyemu + + import flopy + + sys.path.insert(0,os.path.join("..","..","pypestutils")) + + import pypestutils as ppu + + o_model_ws = os.path.join("..","examples","Freyberg","extra_crispy") + model_ws = os.path.join(tmp_path, "extra_crispy") + if os.path.exists(model_ws): + shutil.rmtree(model_ws) + shutil.copytree(o_model_ws, model_ws) + ml = flopy.modflow.Modflow.load("freyberg.nam",model_ws=model_ws,check=False) + pp_dir = os.path.join(tmp_path) + #ml.export(os.path.join("temp","test_unrot_grid.shp")) + sr = pyemu.helpers.SpatialReference().from_namfile( + os.path.join(ml.model_ws, ml.namefile), + delc=ml.dis.delc, delr=ml.dis.delr) + sr.rotation = 0. + par_info_unrot = pyemu.pp_utils.setup_pilotpoints_grid(sr=sr, prefix_dict={0: "hk1",1:"hk2"}, + every_n_cell=6, pp_dir=pp_dir, tpl_dir=pp_dir, + shapename=os.path.join(tmp_path, "test_unrot.shp"), + ) + #print(par_info_unrot.parnme.value_counts()) + par_info_unrot.loc[:,"parval1"] = np.random.uniform(10,100,par_info_unrot.shape[0]) + gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(a=1000,contribution=1.0,anisotropy=3.0,bearing=45)) + ok = pyemu.geostats.OrdinaryKrige(gs,par_info_unrot) + ppu_factor_filename = os.path.join("utils","ppu_factors.dat") + pyemu_factor_filename = os.path.join("utils", "pyemu_factors.dat") + + ok.calc_factors_grid(sr, try_use_ppu=False) + ok.to_grid_factors_file(pyemu_factor_filename) + ok.calc_factors_grid(sr,try_use_ppu=True,ppu_factor_filename=ppu_factor_filename) + out_file = os.path.join("utils","pyemu_array.dat") + pyemu.geostats.fac2real(par_info_unrot,pyemu_factor_filename,out_file=out_file) + out_file_ppu = os.path.join("utils", "ppu_array.dat") + pyemu.geostats.fac2real(par_info_unrot, ppu_factor_filename, out_file=out_file_ppu) + arr_ppu = np.loadtxt(out_file_ppu) + arr = np.loadtxt(out_file) + diff = 100 * np.abs(arr - arr_ppu) / np.abs(arr) + assert diff.max() < 1.0 + # fig,axes = plt.subplots(1,3,figsize=(10,10)) + # cb = axes[0].imshow(arr) + # plt.colorbar(cb, ax=axes[0]) + # + # cb = axes[1].imshow(arr_ppu,vmin=arr.min(),vmax=arr.max()) + # plt.colorbar(cb, ax=axes[1]) + # + # cb = axes[2].imshow(diff) + # plt.colorbar(cb,ax=axes[2]) + # plt.show() + # exit() + if __name__ == "__main__": - thresh_pars_test() + #ppu_geostats_test(".") + while True: + thresh_pars_test() #obs_ensemble_quantile_test() #geostat_draws_test("temp") # ac_draw_test("temp") diff --git a/etc/environment.yml b/etc/environment.yml index 44788090..ed369d97 100644 --- a/etc/environment.yml +++ b/etc/environment.yml @@ -22,4 +22,5 @@ dependencies: - flopy - modflow-devtools - scikit-learn -# - pip \ No newline at end of file + - pip + - ffmpeg diff --git a/examples/Schurexample_henry.ipynb b/examples/Schurexample_henry.ipynb index 3a0bda73..e168df43 100644 --- a/examples/Schurexample_henry.ipynb +++ b/examples/Schurexample_henry.ipynb @@ -260,9 +260,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "df.plot(kind=\"bar\",figsize=(10,5))\n", diff --git a/examples/freyberg_mf6/freyberg6.ims b/examples/freyberg_mf6/freyberg6.ims index 581f7fc1..cced539c 100644 --- a/examples/freyberg_mf6/freyberg6.ims +++ b/examples/freyberg_mf6/freyberg6.ims @@ -4,7 +4,7 @@ BEGIN Options END Options BEGIN Nonlinear - OUTER_HCLOSE 0.01000000 + OUTER_HCLOSE 1.00000000E-06 OUTER_MAXIMUM 100 UNDER_RELAXATION dbd UNDER_RELAXATION_THETA 0.85000000 @@ -19,8 +19,8 @@ END Nonlinear BEGIN LINEAR INNER_MAXIMUM 50 - INNER_HCLOSE 1.00000000E-04 - INNER_RCLOSE 0.10000000 + INNER_HCLOSE 1.00000000E-06 + INNER_RCLOSE 0.0010000000 LINEAR_ACCELERATION bicgstab PRECONDITIONER_LEVELS 1 PRECONDITIONER_DROP_TOLERANCE 0.00100000 diff --git a/examples/modflow_to_pest_like_a_boss.ipynb b/examples/modflow_to_pest_like_a_boss.ipynb index 8d363135..0e377f25 100644 --- a/examples/modflow_to_pest_like_a_boss.ipynb +++ b/examples/modflow_to_pest_like_a_boss.ipynb @@ -18,9 +18,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", @@ -53,9 +51,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "egpath = Path(\".\").absolute()\n", @@ -99,9 +95,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "#load the existing model and save it in a new dir and make sure it runs\n", @@ -127,9 +121,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "hds = flopy.utils.HeadFile(os.path.join(ml.model_ws,\"freyberg.hds\"))\n", @@ -170,9 +162,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "hob_df = pyemu.gw_utils.modflow_hob_to_instruction_file(os.path.join(ml.model_ws,ml.name+\".hob.out\"))" @@ -188,9 +178,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "hob_df.head()" @@ -208,9 +196,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "# the flux budget output filename that will be written during each forward run\n", @@ -224,9 +210,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "df_wb.head()" @@ -260,9 +244,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "prefix_dict= {0:[\"hk1\",\"sy1\",\"rech1\"]}\n" @@ -278,9 +260,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pp_cells = 3\n", @@ -298,9 +278,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pp_df.index = pp_df.parnme\n", @@ -318,9 +296,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "hk_pp = pyemu.pp_utils.pp_file_to_dataframe(os.path.join(ml.model_ws,\"hk1pp.dat\"))" @@ -329,9 +305,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "hk_pp.head()" @@ -347,9 +321,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "a = pp_cells * ml.dis.delr.array[0] * 3.0\n", @@ -368,9 +340,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "ok = pyemu.geostats.OrdinaryKrige(geostruct=gs,point_data=hk_pp)" @@ -390,9 +360,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "sr = pyemu.helpers.SpatialReference.from_namfile(os.path.join(ml.model_ws, ml.namefile),\n", @@ -410,9 +378,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "ok.to_grid_factors_file(os.path.join(ml.model_ws,\"pp.fac\"))" @@ -441,9 +407,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "sr.xcentergrid[0,0], sr.ycentergrid[0,0]" @@ -452,9 +416,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "hk_pp.iloc[0,:].values" @@ -479,9 +441,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "ext_path = os.path.join(ml.model_ws,\"ref\")\n", @@ -508,9 +468,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "drn_df = pd.read_csv(os.path.join(bak_path,drain_files[0]),\n", @@ -540,9 +498,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "tpl_files = [os.path.join(ml.model_ws,f) for f in os.listdir(ml.model_ws) if f.endswith(\".tpl\")]\n", @@ -560,9 +516,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "ins_files = [os.path.join(ml.model_ws,f) for f in os.listdir(ml.model_ws) if f.endswith(\".ins\")]\n", @@ -580,9 +534,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst = pyemu.Pst.from_io_files(tpl_files,input_files,ins_files,output_files)" @@ -598,9 +550,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst.parameter_data.head()" @@ -616,9 +566,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst.observation_data.head()" @@ -634,9 +582,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "hob_df.head()" @@ -652,9 +598,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "hob_df.index = hob_df.obsnme\n", @@ -664,9 +608,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst.observation_data.loc[hob_df.index,\"obsval\"] = hob_df.obsval\n", @@ -685,9 +627,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst.parameter_data.head()" @@ -703,9 +643,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "avg = ml.drn.stress_period_data[0][\"cond\"].mean()\n", @@ -728,9 +666,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "par.loc[pp_df.parnme,\"pargp\"] = pp_df.pargp" @@ -746,9 +682,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst.model_command" @@ -764,9 +698,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst.model_command = [\"python forward_run.py\"]" @@ -782,9 +714,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst.write(os.path.join(ml.model_ws,\"pest.pst\"))" @@ -806,9 +736,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "shutil.copy2(os.path.join(\"Freyberg_transient\",\"forward_run.py\"),os.path.join(ml.model_ws,\"forward_run.py\"))" @@ -826,9 +754,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pyemu.utils.helpers.zero_order_tikhonov(pst,par_groups=[\"drn_cond\"])\n", @@ -845,9 +771,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pp_groups = pp_df.groupby(\"pargp\").groups\n", @@ -860,9 +784,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst.prior_information" @@ -871,9 +793,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst.control_data.pestmode = \"regularization\"" @@ -896,9 +816,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst.pestpp_options[\"svd_pack\"] = \"redsvd\"\n", @@ -915,9 +833,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst.write(\"freyberg_reg.pst\")" diff --git a/examples/pstfrom_mf6.ipynb b/examples/pstfrom_mf6.ipynb index e4964eb1..f1e12d6e 100644 --- a/examples/pstfrom_mf6.ipynb +++ b/examples/pstfrom_mf6.ipynb @@ -17,9 +17,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "import os\n", @@ -60,9 +58,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "id_arr = np.loadtxt(os.path.join(org_model_ws,\"freyberg6.dis_idomain_layer3.txt\"))\n", @@ -103,9 +99,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "sim = flopy.mf6.MFSimulation.load(sim_ws=tmp_model_ws)\n", @@ -122,9 +116,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "sr = pyemu.helpers.SpatialReference.from_namfile(\n", @@ -143,9 +135,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "template_ws = \"freyberg6_template\"\n", @@ -167,9 +157,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "df = pd.read_csv(os.path.join(tmp_model_ws,\"heads.csv\"),index_col=0)\n", @@ -186,9 +174,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "hds_df = pf.add_observations(\"heads.csv\",insfile=\"heads.csv.ins\",index_cols=\"time\",\n", @@ -206,9 +192,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "[f for f in os.listdir(template_ws) if f.endswith(\".ins\")]" @@ -226,9 +210,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "df = pd.read_csv(os.path.join(tmp_model_ws, \"sfr.csv\"), index_col=0)\n", @@ -252,9 +234,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "v = pyemu.geostats.ExpVario(contribution=1.0,a=1000)\n", @@ -265,9 +245,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "grid_gs.plot()\n", @@ -277,9 +255,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "temporal_gs.plot()\n", @@ -296,9 +272,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "ib = m.dis.idomain[0].array" @@ -314,9 +288,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "hk_arr_files = [f for f in os.listdir(tmp_model_ws) if \"npf_k_\" in f and f.endswith(\".txt\")]\n", @@ -335,9 +307,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pf.add_parameters(filenames=\"freyberg6.npf_k_layer1.txt\",par_type=\"grid\",\n", @@ -357,9 +327,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "[f for f in os.listdir(template_ws) if f.endswith(\".tpl\")]" @@ -397,9 +365,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pf.add_parameters(filenames=\"freyberg6.npf_k_layer3.txt\",par_type=\"pilotpoints\",\n", @@ -418,18 +384,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "xmn = m.modelgrid.xvertices.min()\n", @@ -458,9 +420,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pf.add_parameters(filenames=\"freyberg6.npf_k_layer2.txt\",par_type=\"pilotpoints\",\n", @@ -483,9 +443,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "_ = [print(line.rstrip()) for line in open(\"helpers.py\",'r').readlines()]" @@ -508,9 +466,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "assert os.path.exists(\"special_outputs.dat.ins\")\n", @@ -528,9 +484,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pf.add_py_function(\"helpers.py\",\"process_model_outputs()\",is_pre_cmd=False)" @@ -546,9 +500,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "out_file = special_ins_filename.replace(\".ins\",\"\")\n", @@ -573,9 +525,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst = pf.build_pst()" @@ -593,9 +543,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "[f for f in os.listdir(template_ws) if f.endswith(\".py\")]" @@ -604,9 +552,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "_ = [print(line.rstrip()) for line in open(os.path.join(template_ws,\"forward_run.py\"))]" @@ -624,9 +570,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "# only execute this block once!\n", @@ -637,9 +581,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "_ = [print(line.rstrip()) for line in open(os.path.join(template_ws,\"forward_run.py\"))]" @@ -667,9 +609,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pf.add_parameters(filenames=\"freyberg6.npf_k_layer3.txt\",par_type=\"grid\",\n", @@ -688,9 +628,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst = pf.build_pst()\n", @@ -719,9 +657,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "wel_files = [f for f in os.listdir(tmp_model_ws) if \"wel_stress_period\" in f and f.endswith(\".txt\")]\n", @@ -731,9 +667,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pd.read_csv(os.path.join(tmp_model_ws,wel_files[0]),header=None)" @@ -767,9 +701,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst = pf.build_pst()\n", @@ -789,9 +721,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "plt.imshow(x[-25:,-25:])" @@ -820,9 +750,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pst = pf.build_pst()\n", @@ -860,9 +788,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "tpl_filename = os.path.join(template_ws,\"special_pars.dat.tpl\")\n", @@ -875,9 +801,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pf.pst.add_parameters(tpl_filename,pst_path=\".\")" @@ -895,9 +819,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "par = pf.pst.parameter_data\n", @@ -917,9 +839,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "par.loc[pf.pst.par_names[5:10],\"parlbnd\"]" @@ -928,9 +848,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "par.loc[pf.pst.par_names[5:10],\"parlbnd\"] = 0.25\n", @@ -958,9 +876,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pe = pf.draw(num_reals=100,use_specsim=True)" @@ -969,9 +885,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "pe.to_csv(os.path.join(template_ws,\"prior.csv\"))" @@ -980,9 +894,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "print(pe.loc[:,pst.adj_par_names[0]])\n", @@ -1182,18 +1094,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [] } diff --git a/examples/pstfrom_mf6_ppu.ipynb b/examples/pstfrom_mf6_ppu.ipynb new file mode 100644 index 00000000..4d9d30cc --- /dev/null +++ b/examples/pstfrom_mf6_ppu.ipynb @@ -0,0 +1,823 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setting up a PEST interface from MODFLOW6 using the `PstFrom` class with `PyPestUtils` for advanced pilot point parameterization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import shutil\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import pyemu\n", + "import flopy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.append(os.path.join(\"..\",\"..\",\"pypestutils\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pypestutils as ppu" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An existing MODFLOW6 model is in the directory `freyberg_mf6`. Lets check it out:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "org_model_ws = os.path.join('freyberg_mf6')\n", + "os.listdir(org_model_ws)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can see that all the input array and list data for this model have been written \"externally\" - this is key to using the `PstFrom` class. \n", + "\n", + "Let's quickly viz the model top just to remind us of what we are dealing with:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "id_arr = np.loadtxt(os.path.join(org_model_ws,\"freyberg6.dis_idomain_layer3.txt\"))\n", + "top_arr = np.loadtxt(os.path.join(org_model_ws,\"freyberg6.dis_top.txt\"))\n", + "top_arr[id_arr==0] = np.nan\n", + "plt.imshow(top_arr)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's copy those files to a temporary location just to make sure we don't goof up those original files:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tmp_model_ws = \"temp_pst_from_ppu\"\n", + "if os.path.exists(tmp_model_ws):\n", + " shutil.rmtree(tmp_model_ws)\n", + "shutil.copytree(org_model_ws,tmp_model_ws)\n", + "os.listdir(tmp_model_ws)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we need just a tiny bit of info about the spatial discretization of the model - this is needed to work out separation distances between parameters for build a geostatistical prior covariance matrix later.\n", + "\n", + "Here we will load the flopy sim and model instance just to help us define some quantities later - flopy is not required to use the `PstFrom` class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sim = flopy.mf6.MFSimulation.load(sim_ws=tmp_model_ws)\n", + "m = sim.get_model(\"freyberg6\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we use the simple `SpatialReference` pyemu implements to help us spatially locate parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sr = pyemu.helpers.SpatialReference.from_namfile(\n", + " os.path.join(tmp_model_ws, \"freyberg6.nam\"),\n", + " delr=m.dis.delr.array, delc=m.dis.delc.array)\n", + "sr" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can instantiate a `PstFrom` class instance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "template_ws = \"freyberg6_template\"\n", + "pf = pyemu.utils.PstFrom(original_d=tmp_model_ws, new_d=template_ws,\n", + " remove_existing=True,\n", + " longnames=True, spatial_reference=sr,\n", + " zero_based=False,start_datetime=\"1-1-2018\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Observations\n", + "\n", + "So now that we have a `PstFrom` instance, but its just an empty container at this point, so we need to add some PEST interface \"observations\" and \"parameters\". Let's start with observations using MODFLOW6 head. These are stored in `heads.csv`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_csv(os.path.join(tmp_model_ws,\"heads.csv\"),index_col=0)\n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The main entry point for adding observations is (surprise) `PstFrom.add_observations()`. This method works on the list-type observation output file. We need to tell it what column is the index column (can be string if there is a header or int if no header) and then what columns contain quantities we want to monitor (e.g. \"observe\") in the control file - in this case we want to monitor all columns except the index column:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hds_df = pf.add_observations(\"heads.csv\",insfile=\"heads.csv.ins\",index_cols=\"time\",\n", + " use_cols=list(df.columns.values),prefix=\"hds\",)\n", + "hds_df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that it returned a dataframe with lots of useful info: the observation names that were formed (`obsnme`), the values that were read from `heads.csv` (`obsval`) and also some generic weights and group names. At this point, no control file has been created, we have simply prepared to add this observations to the control file later. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "[f for f in os.listdir(template_ws) if f.endswith(\".ins\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Nice! We also have a PEST-style instruction file for those obs.\n", + "\n", + "Now lets do the same for SFR observations:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_csv(os.path.join(tmp_model_ws, \"sfr.csv\"), index_col=0)\n", + "sfr_df = pf.add_observations(\"sfr.csv\", insfile=\"sfr.csv.ins\", index_cols=\"time\", use_cols=list(df.columns.values))\n", + "sfr_df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sweet as! Now that we have some observations, let's add parameters!\n", + "\n", + "## Pilot points and `PyPestUtils`\n", + "\n", + "This notebook is mostly meant to demonstrate some advanced pilot point parameterization that is possible with `PyPestUtils`, so we will only focus on HK and VK pilot point parameters. This is just to keep the example short. In practice, please please please parameterize boundary conditions too!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "v = pyemu.geostats.ExpVario(contribution=1.0,a=5000,bearing=0,anisotropy=1)\n", + "pp_gs = pyemu.geostats.GeoStruct(variograms=v, transform='log')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pp_gs.plot()\n", + "print(\"spatial variogram\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's get the idomain array to use as a zone array - this keeps us from setting up parameters in inactive model cells:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ib = m.dis.idomain[0].array" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Find HK files for the upper and lower model layers (assuming model layer 2 is a semi-confining unit)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hk_arr_files = [f for f in os.listdir(tmp_model_ws) if \"npf_k_\" in f and f.endswith(\".txt\") and \"layer2\" not in f]\n", + "hk_arr_files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "arr_file = \"freyberg6.npf_k_layer1.txt\"\n", + "tag = arr_file.split('.')[1].replace(\"_\",\"-\")\n", + "pf.add_parameters(filenames=arr_file,par_type=\"pilotpoints\",\n", + " par_name_base=tag,pargp=tag,zone_array=ib,\n", + " upper_bound=10.,lower_bound=0.1,ult_ubound=100,ult_lbound=0.01,\n", + " pp_options={\"pp_space\":3},geostruct=pp_gs)\n", + "#let's also add the resulting hk array that modflow sees as observations\n", + "# so we can make easy plots later...\n", + "pf.add_observations(arr_file,prefix=tag,\n", + " obsgp=tag,zone_array=ib)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you are familiar with how `PstFrom` has worked historically, we handed off the process to solve for the factor file (which requires solving the kriging equations for each active node) to a pure python (well, with pandas and numpy). This was ok for toy models, but hella slow for big ugly models. If you look at the log entries above, you should see that the instead, `PstFrom` successfully handed off the solve to `PyPestUtils`, which is exponentially faster for big models. sweet ez! " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tpl_files = [f for f in os.listdir(template_ws) if f.endswith(\".tpl\")]\n", + "tpl_files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open(os.path.join(template_ws,tpl_files[0]),'r') as f:\n", + " for _ in range(2):\n", + " print(f.readline().strip())\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "So those might look like pretty redic parameter names, but they contain heaps of metadata to help you post process things later..." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "So those are you standard pilot points for HK in layer 1 - same as it ever was..." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Geostatistical hyper-parameters\n", + "\n", + "For the HK layer 1 pilot points, we used a standard geostatistical structure - the ever popular exponential variogram. But what if the properties that define that variogram were themselves uncertain? Like what is the anisotropy ellipse varied in space across the model domain? What does this imply? Well, technically speaking, those variogram properties can be conceptualized as \"hyper parameters\" in that they influence the underlying parameters (in this case, the pilot points) in hierarchical sense. That is, the bearing of the anisotropy of the variogram changes, then the resulting interpolation from the pilot points to grid changes. But where it gets really deep is that we need to define correlation structures for these spatially varying hyper pars, so they themselves have plausible spatial patterns...Seen that movie inception?!\n", + "\n", + "In `PyPestUtils`, we can supply the pilot-point-to-grid interpolation process with arrays of hyper-parameter values, one array for each variogram property. The result of this hyper parameter mess is referred to as a non-stationary spatial parameterization. buckle up...\n", + "\n", + "First let's define some additional geostatistical structures:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "value_v = pyemu.geostats.ExpVario(contribution=1, a=5000, anisotropy=2, bearing=0.0)\n", + "value_gs = pyemu.geostats.GeoStruct(variograms=value_v)\n", + "bearing_v = pyemu.geostats.ExpVario(contribution=1,a=5000,anisotropy=2,bearing=90.0)\n", + "bearing_gs = pyemu.geostats.GeoStruct(variograms=bearing_v)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "arr_file = \"freyberg6.npf_k_layer3.txt\"\n", + "tag = arr_file.split('.')[1].replace(\"_\",\"-\")\n", + "pf.add_parameters(filenames=arr_file,par_type=\"pilotpoints\",\n", + " par_name_base=tag,pargp=tag,zone_array=ib,\n", + " upper_bound=10.,lower_bound=0.1,ult_ubound=100,ult_lbound=0.01,\n", + " pp_options={\"pp_space\":3,\"prep_hyperpars\":True},geostruct=value_gs,\n", + " apply_order=2)\n", + "pf.add_observations(arr_file,prefix=tag,\n", + " obsgp=tag,zone_array=ib)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hyperpar_files = [f for f in os.listdir(pf.new_d) if tag in f]\n", + "hyperpar_files" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "when we supplied the \"prep_hyperpars\" as `True` above, that triggered `PstFrom` to do something different. Instead of solving for the pilot point kriging factors as before, now, we have array-based files for the geostatistical hyper parameters, as well as some additional quantities we need to \"apply\" these hyper parameter at runtime. This is a key difference: When the pilot point variogram is changing for each model run, we need to re-solve for the kriging factors for each model run...\n", + "\n", + "We snuck in something else too - see that `apply_order` argument? That is how we can control what order of files being processed by the run-time multiplier parameter function. Since we are going to parameterize the hyper parameters and there is an implicit order between these hyper parameters and the underlying pilot points, we need to make sure the hyper parameters are processed first. \n", + "\n", + "Lets setup some hyper parameters for estimation. We will use a constant for the anisotropy ratio, but use pilot points for the bearing:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "afile = 'npf-k-layer3.aniso.dat'\n", + "tag = afile.split('.')[0].replace(\"_\",\"-\")+\"-aniso\"\n", + "pf.add_parameters(afile,par_type=\"constant\",par_name_base=tag,\n", + " pargp=tag,lower_bound=-1.0,upper_bound=1.0,\n", + " apply_order=1,\n", + " par_style=\"a\",transform=\"none\",initial_value=0.0)\n", + "pf.add_observations(afile, prefix=tag, obsgp=tag)\n", + "bfile = 'npf-k-layer3.bearing.dat'\n", + "tag = bfile.split('.')[0].replace(\"_\",\"-\")+\"-bearing\"\n", + "pf.add_parameters(bfile, par_type=\"pilotpoints\", par_name_base=tag,\n", + " pargp=tag, pp_space=6,lower_bound=-45,upper_bound=45,\n", + " par_style=\"a\",transform=\"none\",\n", + " pp_options={\"try_use_ppu\":True},\n", + " apply_order=1,geostruct=bearing_gs)\n", + "pf.add_observations(bfile, prefix=tag, obsgp=tag) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that the `apply_order` for these hyper pars is 1 so that any processing for these quantities happens before the actual underlying pilot points are processed" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## \"These go to 11\" - amp'ing things up with categorization\n", + "\n", + "Sometimes, the world we want to simulate might be better represented as categorical instead continuous. That is, rather than smoothly varying property fields, we want fields that are either a high value or a low value (please dont ask for more than 2 categories!). In this case, depending on how you plan to assimilate data (that is, what inversion algorithm you are planning to you), we can accomodate this preference for categorical fields. \n", + "\n", + "This is pretty advanced and also dense. There is another example notebook the describes the categorization process in detail. Here we will just blast thru it....\n", + "\n", + "lets setup non-stationary categorical parameterization for the VK of layer 2 (the semi confining unit). We can conceptualize this as a semi-confining unit that has \"windows\" in it that connects the two aquifers. Where there is not a window, the Vk will be very low, where there is a window, the VK will be much higher. Let's also assume the windows in the confining unit where created when a stream eroded thru it, so the shape of these windows will be higher-order (not derived from a standard geostatistical 2-point process), but rather from connected features.\n", + "\n", + "In what follows, we setup this complex parameterization. We also add lots of aux observations to lets plot and viz the steps in this parameterization process." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "arr_file = \"freyberg6.npf_k33_layer2.txt\"\n", + "print(arr_file)\n", + "k = int(arr_file.split(\".\")[1][-1]) - 1\n", + "pth_arr_file = os.path.join(pf.new_d,arr_file)\n", + "arr = np.loadtxt(pth_arr_file)\n", + "cat_dict = {1:[0.4,arr.mean()],2:[0.6,arr.mean()]}\n", + "\n", + "#this is where we initialize the categorization process - it will operate on the \n", + "# layer 2 VK array just before MODFLOW runs\n", + "thresharr,threshcsv = pyemu.helpers.setup_threshold_pars(pth_arr_file,cat_dict=cat_dict,\n", + " testing_workspace=pf.new_d,inact_arr=ib)\n", + "\n", + "# the corresponding apply function\n", + "pf.pre_py_cmds.append(\"pyemu.helpers.apply_threshold_pars('{0}')\".format(os.path.split(threshcsv)[1]))\n", + "prefix = arr_file.split('.')[1].replace(\"_\",\"-\")\n", + "\n", + "pth_arr_file = os.path.join(pf.new_d,arr_file)\n", + "arr = np.loadtxt(pth_arr_file)\n", + "\n", + "tag = arr_file.split('.')[1].replace(\"_\",\"-\") + \"_pp\"\n", + "prefix = arr_file.split('.')[1].replace(\"_\",\"-\")\n", + "#setup pilot points with hyper pars for the thresholding array (the array that will drive the \n", + "# categorization process). Notice the apply_order arg being used \n", + "pf.add_parameters(filenames=os.path.split(thresharr)[1],par_type=\"pilotpoints\",transform=\"none\",\n", + " par_name_base=tag+\"-threshpp_k:{0}\".format(k),\n", + " pargp=tag + \"-threshpp_k:{0}\".format(k),\n", + " lower_bound=0.0,upper_bound=2.0,par_style=\"m\",\n", + " pp_options={\"try_use_ppu\":False,\"prep_hyperpars\":True,\"pp_space\":5},\n", + " apply_order=2,geostruct=value_gs\n", + " )\n", + "\n", + "tag = arr_file.split('.')[1].replace(\"_\",\"-\")\n", + "# a constant parameter for the anisotropy of the thresholding array\n", + "# Notice the apply_order arg being used\n", + "tfiles = [f for f in os.listdir(pf.new_d) if tag in f]\n", + "afile = [f for f in tfiles if \"aniso\" in f][0]\n", + "pf.add_parameters(afile,par_type=\"constant\",par_name_base=tag+\"-aniso\",\n", + " pargp=tag+\"-aniso\",lower_bound=-1.0,upper_bound=1.0,\n", + " apply_order=1,\n", + " par_style=\"a\",transform=\"none\",initial_value=0.0)\n", + "# obs for the anisotropy field\n", + "pf.add_observations(afile, prefix=tag+\"-aniso\", obsgp=tag+\"-aniso\")\n", + "\n", + "# pilot points for the bearing array of the geostructure of the thresholding array\n", + "# Notice the apply_order arg being used\n", + "bfile = [f for f in tfiles if \"bearing\" in f][0]\n", + "pf.add_parameters(bfile, par_type=\"pilotpoints\", par_name_base=tag + \"-bearing\",\n", + " pargp=tag + \"-bearing\", pp_space=6,lower_bound=-45,upper_bound=45,\n", + " par_style=\"a\",transform=\"none\",\n", + " pp_options={\"try_use_ppu\":True},\n", + " apply_order=1,geostruct=bearing_gs)\n", + "# obs for the bearing array\n", + "pf.add_observations(bfile, prefix=tag + \"-bearing\", obsgp=tag + \"-bearing\") \n", + "\n", + "# list style parameters for the quantities used in the categorization process\n", + "# We will manipulate these initial values and bounds later\n", + "pf.add_parameters(filenames=os.path.split(threshcsv)[1], par_type=\"grid\",index_cols=[\"threshcat\"],\n", + " use_cols=[\"threshproportion\",\"threshfill\"],\n", + " par_name_base=[prefix+\"threshproportion_k:{0}\".format(k),prefix+\"threshfill_k:{0}\".format(k)],\n", + " pargp=[prefix+\"threshproportion_k:{0}\".format(k),prefix+\"threshfill_k:{0}\".format(k)],\n", + " lower_bound=[0.1,0.1],upper_bound=[10.0,10.0],transform=\"none\",par_style='d')\n", + "\n", + "# obs of the resulting Vk array that MODFLOW uses\n", + "pf.add_observations(arr_file,prefix=tag,\n", + " obsgp=tag,zone_array=ib)\n", + "\n", + "# observations of the categorized array\n", + "pf.add_observations(arr_file+\".threshcat.dat\", prefix=\"tcatarr-\" + prefix+\"_k:{0}\".format(k),\n", + " obsgp=\"tcatarr-\" + prefix+\"_k:{0}\".format(k),zone_array=ib)\n", + "\n", + "# observations of the thresholding array\n", + "pf.add_observations(arr_file + \".thresharr.dat\",\n", + " prefix=tag+'-thresharr',\n", + " obsgp=tag+'-thresharr', zone_array=ib)\n", + "\n", + "# observations of the results of the thresholding process\n", + "df = pd.read_csv(threshcsv.replace(\".csv\",\"_results.csv\"),index_col=0)\n", + "pf.add_observations(os.path.split(threshcsv)[1].replace(\".csv\",\"_results.csv\"),index_cols=\"threshcat\",use_cols=df.columns.tolist(),prefix=prefix+\"-results_k:{0}\".format(k),\n", + " obsgp=prefix+\"-results_k:{0}\".format(k),ofile_sep=\",\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### build the control file, pest interface files, and forward run script\n", + "At this point, we have some parameters and some observations, so we can create a control file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.mod_sys_cmds.append(\"mf6\")\n", + "pf.pre_py_cmds.insert(0,\"import sys\")\n", + "pf.pre_py_cmds.insert(1,\"sys.path.append(os.path.join('..','..','..','pypestutils'))\")\n", + "pst = pf.build_pst()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "_ = [print(line.rstrip()) for line in open(os.path.join(template_ws,\"forward_run.py\"))]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting initial parameter bounds and values\n", + "\n", + "Here is some gory detail regarding defining the hyper parameters for both layer 3 HK and layer 2 VK..." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#set the initial and bounds for the fill values\n", + "par = pst.parameter_data\n", + "\n", + "apar = par.loc[par.pname.str.contains(\"aniso\"),:]\n", + "bpar = par.loc[par.pname.str.contains(\"bearing\"), :]\n", + "par.loc[apar.parnme.str.contains(\"layer3\").index,\"parval1\"] = 3\n", + "par.loc[apar.parnme.str.contains(\"layer3\").index,\"parlbnd\"] = 1\n", + "par.loc[apar.parnme.str.contains(\"layer3\").index,\"parubnd\"] = 5\n", + "\n", + "par.loc[apar.parnme.str.contains(\"layer2\").index,\"parval1\"] = 2\n", + "par.loc[apar.parnme.str.contains(\"layer2\").index,\"parlbnd\"] = 0\n", + "par.loc[apar.parnme.str.contains(\"layer2\").index,\"parubnd\"] = 4\n", + "\n", + "par.loc[bpar.parnme.str.contains(\"layer3\").index,\"parval1\"] = 0\n", + "par.loc[bpar.parnme.str.contains(\"layer3\").index,\"parlbnd\"] = -90\n", + "par.loc[bpar.parnme.str.contains(\"layer3\").index,\"parubnd\"] = 90\n", + "\n", + "par.loc[bpar.parnme.str.contains(\"layer2\").index,\"parval1\"] = 0\n", + "par.loc[bpar.parnme.str.contains(\"layer2\").index,\"parlbnd\"] = -90\n", + "par.loc[bpar.parnme.str.contains(\"layer2\").index,\"parubnd\"] = 90\n", + "\n", + "cat1par = par.loc[par.apply(lambda x: x.threshcat==\"0\" and x.usecol==\"threshfill\",axis=1),\"parnme\"]\n", + "cat2par = par.loc[par.apply(lambda x: x.threshcat == \"1\" and x.usecol == \"threshfill\", axis=1), \"parnme\"]\n", + "assert cat1par.shape[0] == 1\n", + "assert cat2par.shape[0] == 1\n", + "\n", + "cat1parvk = [p for p in cat1par if \"k:1\" in p]\n", + "cat2parvk = [p for p in cat2par if \"k:1\" in p]\n", + "for lst in [cat2parvk,cat1parvk]:\n", + " assert len(lst) > 0\n", + "\n", + "#these are the values that will fill the two categories of VK - \n", + "# one is low (clay) and one is high (sand - the windows)\n", + "par.loc[cat1parvk, \"parval1\"] = 0.0001\n", + "par.loc[cat1parvk, \"parubnd\"] = 0.01\n", + "par.loc[cat1parvk, \"parlbnd\"] = 0.000001\n", + "par.loc[cat1parvk, \"partrans\"] = \"log\"\n", + "par.loc[cat2parvk, \"parval1\"] = 0.1\n", + "par.loc[cat2parvk, \"parubnd\"] = 1\n", + "par.loc[cat2parvk, \"parlbnd\"] = 0.01\n", + "par.loc[cat2parvk, \"partrans\"] = \"log\"\n", + "\n", + "\n", + "cat1par = par.loc[par.apply(lambda x: x.threshcat == \"0\" and x.usecol == \"threshproportion\", axis=1), \"parnme\"]\n", + "cat2par = par.loc[par.apply(lambda x: x.threshcat == \"1\" and x.usecol == \"threshproportion\", axis=1), \"parnme\"]\n", + "\n", + "assert cat1par.shape[0] == 1\n", + "assert cat2par.shape[0] == 1\n", + "\n", + "#these are the proportions of clay and sand in the resulting categorical array\n", + "#really under the hood, only the first one is used, so we can fix the other.\n", + "par.loc[cat1par, \"parval1\"] = 0.95\n", + "par.loc[cat1par, \"parubnd\"] = 1.0\n", + "par.loc[cat1par, \"parlbnd\"] = 0.9\n", + "par.loc[cat1par,\"partrans\"] = \"none\"\n", + "\n", + "# since the apply method only looks that first proportion, we can just fix this one\n", + "par.loc[cat2par, \"parval1\"] = 1\n", + "par.loc[cat2par, \"parubnd\"] = 1\n", + "par.loc[cat2par, \"parlbnd\"] = 1\n", + "par.loc[cat2par,\"partrans\"] = \"fixed\"\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Generating a prior parameter ensemble, then run and viz a real" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(122341)\n", + "pe = pf.draw(num_reals=100)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pe.to_csv(os.path.join(template_ws,\"prior.csv\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "real = 0\n", + "pst_name = \"real_{0}.pst\".format(real)\n", + "pst.parameter_data.loc[pst.adj_par_names,\"parval1\"] = pe.loc[real,pst.adj_par_names].values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pst.control_data.noptmax = 0\n", + "pst.write(os.path.join(pf.new_d,pst_name))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pyemu.os_utils.run(\"pestpp-ies {0}\".format(pst_name),cwd=pf.new_d)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pst.set_res(os.path.join(pf.new_d,pst_name.replace(\".pst\",\".base.rei\")))\n", + "res = pst.res\n", + "obs = pst.observation_data\n", + "grps = [o for o in obs.obgnme.unique() if o.startswith(\"npf\") and \"result\" not in o and \"aniso\" not in o]\n", + "grps" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gobs = obs.loc[obs.obgnme.isin(grps),:].copy()\n", + "gobs[\"i\"] = gobs.i.astype(int)\n", + "gobs[\"j\"] = gobs.j.astype(int)\n", + "gobs[\"k\"] = gobs.obgnme.apply(lambda x: int(x.split('-')[2].replace(\"layer\",\"\")) - 1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "uk = gobs.k.unique()\n", + "uk.sort()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for k in uk:\n", + " kobs = gobs.loc[gobs.k==k,:]\n", + " ug = kobs.obgnme.unique()\n", + " ug.sort()\n", + " fig,axes = plt.subplots(1,4,figsize=(20,6))\n", + " axes = np.atleast_1d(axes)\n", + " for ax in axes:\n", + " ax.set_frame_on(False)\n", + " ax.set_yticks([])\n", + " ax.set_xticks([])\n", + " for g,ax in zip(ug,axes):\n", + " gkobs = kobs.loc[kobs.obgnme==g,:]\n", + " \n", + " arr = np.zeros_like(top_arr)\n", + " arr[gkobs.i,gkobs.j] = res.loc[gkobs.obsnme,\"modelled\"].values\n", + " ax.set_aspect(\"equal\")\n", + " label = \"\"\n", + " if \"bearing\" not in g and \"aniso\" not in g:\n", + " arr = np.log10(arr)\n", + " label = \"$log_{10}$\"\n", + " cb = ax.imshow(arr)\n", + " plt.colorbar(cb,ax=ax,label=label)\n", + " ax.set_title(\"layer: {0} group: {1}\".format(k+1,g),loc=\"left\",fontsize=15)\n", + " \n", + " plt.tight_layout()\n", + " plt.show()\n", + " plt.close(fig)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Stunning isnt it?! There is clealy a lot subjectivity in the form of defining the prior for the hyper parameters required to use these non-stationary geostats, but they do afford more opportunities to express (stochastic) expert knowledge. To be honest, there was a lot of experimenting with this notebook to get these figures to look this way - playing with variograms and parameter inital values and bounds a lot. You encouraged to do the same! scroll back up, change things, and \"restart kernel and run all\" - this will help build some better intution, promise...." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/understanding_array_thresholding.ipynb b/examples/understanding_array_thresholding.ipynb new file mode 100644 index 00000000..0e69c4b9 --- /dev/null +++ b/examples/understanding_array_thresholding.ipynb @@ -0,0 +1,367 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a3b87237-8575-4798-a392-278aebf77801", + "metadata": {}, + "source": [ + "# Deep dive into thresholding continuous fields to make categorical fields\n", + "\n", + "In this notebook we will explore the categorization process that is in pyEMU. Note this process was inspired by Todaro and other (2023) \"Experimental sandbox tracer tests to characterize a two-facies aquifer via an ensemble smoother\" [https://doi.org/10.1007/s10040-023-02662-1](https://doi.org/10.1007/s10040-023-02662-1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37e0fb1f-51b0-442c-88e1-2812b54ad902", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import shutil\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import pyemu" + ] + }, + { + "cell_type": "markdown", + "id": "62977378-37d4-4a2a-8801-b0ade16e7440", + "metadata": {}, + "source": [ + "First, lets generate just a single multivariate gaussian field. We will use this as the \"original array\" thru out the rest of this notebook:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72b9a89c-84d8-4109-8dc7-d18714817191", + "metadata": {}, + "outputs": [], + "source": [ + "nrow = ncol = 50\n", + "delx = np.ones(ncol)\n", + "dely = np.ones(nrow)\n", + "v = pyemu.geostats.ExpVario(contribution=1.0,a=500)\n", + "gs = pyemu.geostats.GeoStruct(variograms=v)\n", + "ss = pyemu.geostats.SpecSim2d(delx=delx,dely=dely,geostruct=gs)\n", + "np.random.seed(122341)\n", + "org_arr = ss.draw_arrays(1,mean_value=10)[0,:,:]\n", + "assert org_arr.min() > 0.0\n", + "cb = plt.imshow(org_arr)\n", + "_ = plt.colorbar(cb)" + ] + }, + { + "cell_type": "markdown", + "id": "f7094e59-f763-41eb-988d-da9193249ebe", + "metadata": {}, + "source": [ + "Now let's setup a workspace" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "322ca92f-06ce-4492-a43a-c864e75b6df7", + "metadata": {}, + "outputs": [], + "source": [ + "ws = \"temp_thresh\"\n", + "if os.path.exists(ws):\n", + " shutil.rmtree(ws)\n", + "os.makedirs(ws)" + ] + }, + { + "cell_type": "markdown", + "id": "faabba44-0fa6-49d2-9213-5b5c261c02dc", + "metadata": {}, + "source": [ + "And save the original array in that workspace" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "209f5fbc-4f2d-48c5-b3d6-7495c5cac86c", + "metadata": {}, + "outputs": [], + "source": [ + "orgarr_file = os.path.join(ws,\"orgarr.dat\")\n", + "np.savetxt(orgarr_file,org_arr)" + ] + }, + { + "cell_type": "markdown", + "id": "14a06478-8c58-4fe0-9ce1-b3a2808315f2", + "metadata": {}, + "source": [ + "The categorization process in pyEMU currently only supports 2 categories/facies. So we need to define a `dict` that contains the proportions of each category and the initial value used to fill each category. For example purposes, we will use the extrema of the original array as the fill values, so that we will end up with a categorical array that has only two unique values: the min and max of the original array:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a86bcae0-e99b-4abd-8a70-3574f6a8e3a9", + "metadata": {}, + "outputs": [], + "source": [ + "cat_dict = {1:[0.95,org_arr.min()],2:[0.05,org_arr.max()]}\n", + "thresharr_file,threshcsv_file = pyemu.helpers.setup_threshold_pars(orgarr_file,cat_dict=cat_dict,\n", + " testing_workspace=ws)" + ] + }, + { + "cell_type": "markdown", + "id": "0cf681cc-cd3f-48ec-9b01-a0c78f34ac77", + "metadata": {}, + "source": [ + "Now lets see what was created as part of the setup process:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55aefaae-cad3-4b3c-a026-271de70d3737", + "metadata": {}, + "outputs": [], + "source": [ + "os.listdir(ws)" + ] + }, + { + "cell_type": "markdown", + "id": "8d46d36f-b377-4c53-b1b2-82eff652a2d0", + "metadata": {}, + "source": [ + "Notice the what we have are files with the original array name and some suffix. Let's check them out:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47047de9-27f0-4fad-aeb4-895129c7acca", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_csv(threshcsv_file)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "323471e6-728d-42e6-8b4c-49e6b7e30e72", + "metadata": {}, + "outputs": [], + "source": [ + "catarr_file = orgarr_file+\".threshcat.dat\"\n", + "print(catarr_file)\n", + "assert os.path.exists(catarr_file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1defbda2-e13b-4a8a-98bd-a038f4718cf9", + "metadata": {}, + "outputs": [], + "source": [ + "thresharr_file = orgarr_file+\".thresharr.dat\"\n", + "print(thresharr_file)\n", + "assert os.path.exists(thresharr_file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ffc78187-afee-4665-94da-895da6b1f6cc", + "metadata": {}, + "outputs": [], + "source": [ + "threshcsv_file_results = orgarr_file+'.threshprops_results.csv'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e80d025-891f-4ca7-943b-ddb708286707", + "metadata": {}, + "outputs": [], + "source": [ + "def load_and_plot(save_name=None):\n", + " cat_arr = np.loadtxt(catarr_file)\n", + " new_arr = np.loadtxt(orgarr_file)\n", + " thresh_arr = np.loadtxt(thresharr_file)\n", + " thresh_arr = (thresh_arr-thresh_arr.min())/thresh_arr.max()\n", + " ddf = pd.read_csv(threshcsv_file_results)\n", + " cat1_prop = ddf.loc[0,\"proportion\"]/ ddf.loc[:,\"proportion\"].sum()\n", + " cat2_prop = ddf.loc[1,\"proportion\"]/ ddf.loc[:,\"proportion\"].sum()\n", + " cat1_thresh = ddf.loc[0,\"threshold\"]\n", + " cat2_thresh = ddf.loc[1,\"threshold\"]\n", + " fig,axes = plt.subplots(1,3,figsize=(10,2.5))\n", + " cb = axes[0].imshow(org_arr)\n", + " plt.colorbar(cb,ax=axes[0])\n", + " #cb = axes[2].imshow(cat_arr)#,vmin=org_arr.min(),vmax=org_arr.max())\n", + " #plt.colorbar(cb,ax=axes[2])\n", + " cb = axes[2].imshow(new_arr,vmin=org_arr.min(),vmax=org_arr.max())\n", + " plt.colorbar(cb,ax=axes[2])\n", + " cb = axes[1].imshow(thresh_arr)#,vmin=org_arr.min(),vmax=org_arr.max())\n", + " plt.colorbar(cb,ax=axes[1])\n", + " axes[1].contour(thresh_arr,levels=[cat1_thresh,cat2_thresh],colors=['w',\"w\"])\n", + " axes[0].set_title(\"original array\\n\",loc=\"left\",fontsize=8)\n", + " axes[1].set_title(\"thresholding array\\ncat 1 threshold:{0:3.4f}\".\\\n", + " format(cat1_thresh),loc=\"left\",fontsize=8)\n", + " #axes[2].set_title(\"categorized array\\ncat 1 proportion: {0:3.4f}\".\\\n", + " # format(cat1_prop)\\\n", + " # ,loc=\"left\",fontsize=10)\n", + " axes[2].set_title(\"new array\\n{1:2.2f}% max, {0:2.2f}% min\".format(cat1_prop*100,cat2_prop*100)\n", + " ,loc=\"left\",fontsize=8)\n", + " plt.tight_layout()\n", + " for ax in axes:\n", + " ax.set_yticks([])\n", + " ax.set_xticks([])\n", + " plt.tight_layout()\n", + " if save_name is not None:\n", + " plt.savefig(save_name)\n", + " else: \n", + " plt.show()\n", + " plt.close(fig)\n", + " return cat_arr,new_arr,ddf" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e6858a-9892-4afc-a5d9-0840ca9ad99b", + "metadata": {}, + "outputs": [], + "source": [ + "cat_arr,new_arr,newnew_df = load_and_plot()\n", + "newnew_df" + ] + }, + { + "cell_type": "markdown", + "id": "968ee1a0-6d37-4f14-bcd3-c00a4c9e87f1", + "metadata": {}, + "source": [ + "So there it is! The original array for reference, the \"thresholding array\" (which is just a scaled and normed verson of the original array) and the resulting \"new array\".\n", + "\n", + "\n", + "Now let's experiment - feel free to change the quantities in `new_df`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee5c2cc6-d9f1-4546-a55d-24d812a53b2e", + "metadata": {}, + "outputs": [], + "source": [ + "new_df = df.copy()\n", + "new_df.loc[0,\"threshproportion\"] = .25\n", + "new_df.to_csv(threshcsv_file)\n", + "pyemu.helpers.apply_threshold_pars(threshcsv_file)\n", + "_,_,newnew_df = load_and_plot()\n", + "newnew_df.iloc[0]" + ] + }, + { + "cell_type": "markdown", + "id": "1c451fea-bd3a-46b2-b7b8-dddd30947fb8", + "metadata": {}, + "source": [ + "Now lets sweep over a range of category 1 proportions and make some figs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ffdadd80-4b39-4b03-b91a-280f4273aaff", + "metadata": {}, + "outputs": [], + "source": [ + "cat1_props = np.linspace(0.01,0.99,100)\n", + "cat1_props" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "330c370a-e286-4cda-aeb3-ec35aa7c1efe", + "metadata": {}, + "outputs": [], + "source": [ + "for i,prop in enumerate(cat1_props):\n", + " new_df = df.copy()\n", + " new_df.loc[0,\"threshproportion\"] = prop\n", + " new_df.to_csv(threshcsv_file)\n", + " pyemu.helpers.apply_threshold_pars(threshcsv_file)\n", + " save_name = os.path.join(ws,\"fig_{0:04d}.png\".format(i))\n", + " _,_,newnew_df = load_and_plot(save_name=save_name)\n", + " print(i,\" \",end='')\n" + ] + }, + { + "cell_type": "markdown", + "id": "0b48cf4e-7f48-4b47-9fe5-9940f855cb2f", + "metadata": {}, + "source": [ + "And if you have `ffmpeg` installed, we can make an sweet-as animated gif:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c1144ba-7410-43df-896d-2525c093c1e2", + "metadata": {}, + "outputs": [], + "source": [ + "fps = 15\n", + "pyemu.os_utils.run(\"ffmpeg -i fig_{0:04d}.png -vf palettegen=256 palette.png\".format(int(len(cat1_props)/2)),cwd=ws)\n", + "pyemu.os_utils.run(\"ffmpeg -r {0} -y -s 1920X1080 -i fig_%04d.png -i palette.png -filter_complex \\\"scale=720:-1:flags=lanczos[x];[x][1:v]paletteuse\\\" fancy.gif\".format(fps),\n", + " cwd=ws)" + ] + }, + { + "cell_type": "markdown", + "id": "7bfa7e9d-c49b-4ad2-b223-7d161598de5e", + "metadata": {}, + "source": [ + "![SegmentLocal](temp_thresh/fancy.gif \"segment1\")" + ] + }, + { + "cell_type": "markdown", + "id": "59eb4b05-9d3f-4baa-9193-169d7385b355", + "metadata": {}, + "source": [ + "So how does this work within the PEST world? Well we can treat the thresholding array as an array we want to parameterize (maybe with pilot points?) as well as parameterizing the fill values and proportions in the \"threshprops.csv\" file. This will let us manipulate the shape of the resulting categorical array that forward model will use as an input. In turn, this yield variability in the simulated response of the system. And away we go!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyemu/utils/geostats.py b/pyemu/utils/geostats.py index 41f45dcb..d03ce17e 100644 --- a/pyemu/utils/geostats.py +++ b/pyemu/utils/geostats.py @@ -3,6 +3,7 @@ from __future__ import print_function import os import copy +import shutil from datetime import datetime import multiprocessing as mp import warnings @@ -872,6 +873,8 @@ def calc_factors_grid( var_filename=None, forgive=False, num_threads=1, + try_use_ppu=False, + ppu_factor_filename = "factors.dat" ): """calculate kriging factors (weights) for a structured grid. @@ -905,14 +908,21 @@ def calc_factors_grid( is raised for failed matrix inversion. num_threads (`int`): number of multiprocessing workers to use to try to speed up kriging in python. Default is 1. - + try_use_ppu (`bool`): flag to try to use `PyPestUtils` to solve the kriging equations. If `true`, + and if `from pypestutils.pestutilslib import PestUtilsLib` does not raise an exception, + the `PestUtilsLib.calc_kriging_factors_2d()` is used. Otherwise, the `OrdinaryKrige` + implementation is used. + ppu_factor_filename (`str`): the name of the factor file that will be created if + `PestUtilsLib.calc_kriging_factors_2d()` is used. Default is "factors.dat". Unused if + `try_use_ppu` is `False`. Returns: `pandas.DataFrame`: a dataframe with information summarizing the ordinary kriging - process for each grid node + process for each grid node. Not returned if `try_use_ppu` is `True`. Note: this method calls OrdinaryKrige.calc_factors() this method is the main entry point for grid-based kriging factor generation + Example:: @@ -940,6 +950,72 @@ def calc_factors_grid( "spatial_reference does not have proper attributes:{0}".format(str(e)) ) + + + use_ppu = False + if try_use_ppu: + try: + from pypestutils.pestutilslib import PestUtilsLib + use_ppu = True + except Exception as e: + pass + + + if use_ppu: + if x.ndim == 1: + x = np.atleast_2d(x).transpose() + y = np.atleast_2d(y).transpose() + print("...pypestutils detected and being used for kriging solve, trust us, you want this!") + ecs = self.point_data.x.values + ncs = self.point_data.y.values + zns = 1#np.ones_like(ecs,dtype=int) + if "zone" in self.point_data.columns: + zns = self.point_data.zone.values.astype(int) + ect = x.ravel() + nct = y.ravel() + znt = 1 + if zone_array is not None: + znt = zone_array.ravel().astype(int) + + #reset any missing values in znt to a zns value - + # doesnt matter in the end, just results in more nodes + # being solved for... + znt_unique = np.unique(znt) + zns_unique = np.unique(zns) + for uz in znt_unique: + if uz not in zns_unique: + znt[znt==uz] = zns_unique[0] + + assert len(self.geostruct.variograms) == 1 + v = self.geostruct.variograms[0] + if isinstance(v,ExpVario): + vartype = 2 + elif isinstance(v,SphVario): + vartype = 1 + elif isinstance(v, GauVario): + vartype = 3 + else: + raise NotImplementedError("unsupported variogram type: {0}".format(str(type(v)))) + krigtype = 1 #hard coded to ordinary + factorfiletype = 1 #hard coded to ascii for now since we have to rewrite the fac file + + plib = PestUtilsLib() + num_interp_pts = plib.calc_kriging_factors_2d(self.point_data.x.values, + self.point_data.y.values,zns, + x.ravel(),y.ravel(),znt, + vartype,krigtype, + v.a,v.anisotropy,v.bearing, + search_radius,maxpts_interp, + minpts_interp,ppu_factor_filename, + factorfiletype) + plib.free_all_memory() + assert os.path.exists(ppu_factor_filename) + reformat_factorfile(x.shape[0], x.shape[1], self.point_data, self.geostruct, ppu_factor_filename) + + print("...ppu_factor_filename '{0}' created, factors calculated for {1} points".\ + format(ppu_factor_filename,num_interp_pts)) + return num_interp_pts + if var_filename is not None: if self.spatial_reference.grid_type=='vertex': arr = ( @@ -2380,7 +2456,7 @@ def fac2real( type(pp_file) ) ) - assert os.path.exists(factors_file), "factors file not found" + assert os.path.exists(factors_file), "factors file {0} not found".format(factors_file) f_fac = open(factors_file, "r") fpp_file = f_fac.readline() if pp_file is None and pp_data is None: @@ -2456,3 +2532,29 @@ def _parse_factor_line(line): # fac = float(raw[ifac+1]) # fac_data[pnum] = fac return inode, itrans, fac_data + + +def reformat_factorfile(nrow,ncol,point_data,geostruct,ppu_factor_filename): + f_in = open(ppu_factor_filename,'r') + f_out = open("temp.fac",'w') + f_out.write("points.junk\nzone.junk\n") + f_out.write("{0} {1}\n".format(ncol,nrow)) + f_out.write("{0}\n".format(point_data.shape[0])) + [f_out.write("{0}\n".format(name)) for name in point_data.name] + t = 0 + if geostruct.transform == "log": + t = 1 + t = str(t) + f_in.readline() + f_in.readline() + for line in f_in: + raw = line.strip().split() + raw.insert(1,t) + line = " ".join(raw) + f_out.write(line+"\n") + f_in.close() + f_out.close() + shutil.copy2("temp.fac",ppu_factor_filename) + os.remove("temp.fac") + + diff --git a/pyemu/utils/helpers.py b/pyemu/utils/helpers.py index 1b73b6b8..c3312a85 100644 --- a/pyemu/utils/helpers.py +++ b/pyemu/utils/helpers.py @@ -1749,7 +1749,7 @@ def apply_list_and_array_pars(arr_par_file="mult2model_info.csv", chunk_len=50): Args: arr_par_file (str): chunk_len (`int`): the number of files to process per multiprocessing - chunk in appl_array_pars(). default is 50. + chunk in apply_array_pars(). default is 50. Returns: @@ -1770,17 +1770,39 @@ def apply_list_and_array_pars(arr_par_file="mult2model_info.csv", chunk_len=50): lambda x: os.path.join(*x.replace("\\","/").split("/")) if isinstance(x,str) else x ) - arr_pars = df.loc[df.index_cols.isna()].copy() - list_pars = df.loc[df.index_cols.notna()].copy() - # extract lists from string in input df - list_pars["index_cols"] = list_pars.index_cols.apply(literal_eval) - list_pars["use_cols"] = list_pars.use_cols.apply(literal_eval) - list_pars["lower_bound"] = list_pars.lower_bound.apply(literal_eval) - list_pars["upper_bound"] = list_pars.upper_bound.apply(literal_eval) - - # TODO check use_cols is always present - apply_genericlist_pars(list_pars, chunk_len=chunk_len) - apply_array_pars(arr_pars, chunk_len=chunk_len) + if "apply_order" in df.columns: + df["apply_order"] = df.apply_order.astype(float) + uapply_values = df.apply_order.unique() + uapply_values.sort() + else: + df["apply_order"] = 999 + uapply_values = [999] + for apply_value in uapply_values: + ddf = df.loc[df.apply_order==apply_value,:].copy() + assert ddf.shape[0] > 0 + arr_pars = ddf.loc[ddf.index_cols.isna()].copy() + list_pars = ddf.loc[ddf.index_cols.notna()].copy() + # extract lists from string in input df + list_pars["index_cols"] = list_pars.index_cols.apply(literal_eval) + list_pars["use_cols"] = list_pars.use_cols.apply(literal_eval) + list_pars["lower_bound"] = list_pars.lower_bound.apply(literal_eval) + list_pars["upper_bound"] = list_pars.upper_bound.apply(literal_eval) + + if "pre_apply_function" in ddf.columns: + calls = ddf.pre_apply_function.dropna() + for call in calls: + print("...evaluating pre-apply function '{0}'".format(call)) + eval(call) + + # TODO check use_cols is always present + apply_genericlist_pars(list_pars, chunk_len=chunk_len) + apply_array_pars(arr_pars, chunk_len=chunk_len) + + if "post_apply_function" in ddf.columns: + calls = ddf.post_apply_function.dropna() + for call in calls: + print("...evaluating post-apply function '{0}'".format(call)) + eval(call) def _process_chunk_fac2real(chunk, i): @@ -1981,14 +2003,6 @@ def apply_array_pars(arr_par="arr_pars.csv", arr_par_file=None, chunk_len=50): print("number of chunks to process:", len(chunks)) if len(chunks) == 1: _process_chunk_array_files(chunks[0], 0, df) - # procs = [] - # for chunk in chunks: # now only spawn processor for each chunk - # p = mp.Process(target=_process_chunk_model_files, args=[chunk, df]) - # p.start() - # procs.append(p) - # for p in procs: - # r = p.get(False) - # p.join() else: with mp.get_context("spawn").Pool( processes=min(mp.cpu_count(), 60)) as pool: @@ -3842,7 +3856,14 @@ def setup_threshold_pars(orgarr_file,cat_dict,testing_workspace=".",inact_arr=No format(len(cat_dict))) prop_tags,prop_vals,fill_vals = [],[],[] - for key,(proportion,fill_val) in cat_dict.items(): + #print(cat_dict[1]) + #for key,(proportion,fill_val) in cat_dict.items(): + keys = list(cat_dict.keys()) + keys.sort() + for key in keys: + proportion = cat_dict[key][0] + fill_val = cat_dict[key][1] + if int(key) not in cat_dict: raise Exception("integer type of key '{0}' not found in target_proportions_dict".format(key)) prop_tags.append(int(key)) @@ -3864,10 +3885,16 @@ def setup_threshold_pars(orgarr_file,cat_dict,testing_workspace=".",inact_arr=No csv_file = orgarr_file+".threshprops.csv" df.to_csv(csv_file,index=False) + # test that it seems to be working + rel_csv_file = csv_file + + rel_csv_file = os.path.relpath(csv_file,start=testing_workspace) bd = os.getcwd() os.chdir(testing_workspace) - apply_threshold_pars(os.path.split(csv_file)[1]) + + #apply_threshold_pars(os.path.split(csv_file)[1]) + apply_threshold_pars(rel_csv_file) os.chdir(bd) return thresharr_file,csv_file @@ -3885,8 +3912,10 @@ def apply_threshold_pars(csv_file): thresarr_file = csv_file.replace("props.csv","arr.dat") tarr = np.loadtxt(thresarr_file) if np.any(tarr < 0): - print(tarr) - raise Exception("negatives in thresholding array {0}".format(thresarr_file)) + tmin = tarr.min() + tarr += tmin + 1 + #print(tarr) + #raise Exception("negatives in thresholding array {0}".format(thresarr_file)) #norm tarr tarr = (tarr - tarr.min()) / tarr.max() orgarr_file = csv_file.replace(".threshprops.csv","") @@ -3911,19 +3940,24 @@ def apply_threshold_pars(csv_file): target_prop = tvals[0] tol = 1.0e-10 - if tarr.std() < tol: + if tarr.std() < 1e-5: + print("WARNING: thresholding array {0} has very low standard deviation".format(thresarr_file)) print(" using a homogenous array with first category fill value {0}".format(tfill[0])) farr = np.zeros_like(tarr) + tfill[0] if iarr is not None: - farr[iarr == 0] = -1.0e+30 - tarr[iarr == 0] = -1.0e+30 - df.loc[tcat[0], "threshold"] = tarr.mean() - df.loc[tcat[1], "threshold"] = tarr.mean() + farr[iarr == 0] = np.nan + tarr[iarr == 0] = np.nan + df.loc[tcat[0], "threshold"] = np.nanmean(tarr) + df.loc[tcat[1], "threshold"] = np.nanmean(tarr) df.loc[tcat[0], "proportion"] = 1 df.loc[tcat[1], "proportion"] = 0 + if iarr is not None: + farr[iarr == 0] = -1e30 + tarr[iarr == 0] = -1e30 + df.to_csv(csv_file.replace(".csv", "_results.csv")) np.savetxt(orgarr_file, farr, fmt="%15.6E") np.savetxt(tarr_file, tarr, fmt="%15.6E") @@ -3934,8 +3968,8 @@ def apply_threshold_pars(csv_file): # a classic: gr = (np.sqrt(5.) + 1.) / 2. - a = tarr.min() - b = tarr.max() + a = np.nanmin(tarr) + b = np.nanmax(tarr) c = b - ((b - a) / gr) d = a + ((b - a) / gr) @@ -3944,7 +3978,7 @@ def apply_threshold_pars(csv_file): if iarr is not None: # this keeps inact from interfering with calcs later... - tarr[iarr == 0] = 1.0e+30 + tarr[iarr == 0] = np.nan tiarr = iarr.copy() tiarr[tiarr <= 0] = 0 tiarr[tiarr > 0] = 1 @@ -3985,8 +4019,8 @@ def get_current_prop(_cur_thresh): tarr[tarr <= thresh] = tcat[1] if iarr is not None: - farr[iarr==0] = -1.0e+30 - tarr[iarr == 0] = -1.0e+30 + farr[iarr==0] = -1e+30 + tarr[iarr==0] = -1e+30 df.loc[tcat[0],"threshold"] = thresh df.loc[tcat[1], "threshold"] = 1.0 - thresh df.loc[tcat[0], "proportion"] = prop diff --git a/pyemu/utils/pp_utils.py b/pyemu/utils/pp_utils.py index 9e832782..774e7ba2 100644 --- a/pyemu/utils/pp_utils.py +++ b/pyemu/utils/pp_utils.py @@ -1,12 +1,14 @@ """Pilot point support utilities """ +from __future__ import division, print_function + import os import copy import numpy as np import pandas as pd import warnings - +import pyemu pd.options.display.max_colwidth = 100 from pyemu.pst.pst_utils import SFMT, IFMT, FFMT, pst_config @@ -227,8 +229,9 @@ def setup_pilotpoints_grid( else: # cycle through rows and cols - for i in range(start_row, ib.shape[0] - start_row, every_n_cell): - for j in range(start_col, ib.shape[1] - start_col, every_n_cell): + # allow to run closer to outside edge rather than leaving a gap + for i in range(start_row, ib.shape[0] - start_row//2, every_n_cell): + for j in range(start_col, ib.shape[1] - start_col//2, every_n_cell): # skip if this is an inactive cell if ib[i, j] <= 0: # this will account for MF6 style ibound as well continue @@ -348,7 +351,7 @@ def setup_pilotpoints_grid( def pp_file_to_dataframe(pp_filename): - """read a pilot point file to a pandas Dataframe + """read a space-delim pilot point file to a pandas Dataframe Args: pp_filename (`str`): path and name of an existing pilot point file @@ -703,4 +706,231 @@ def get_zoned_ppoints_for_vertexgrid(spacing, zone_array, mg, zone_number=None, # select ppoint coords within area ppoints = [(p.x, p.y) for p in grid_points if polygon.covers(p) ] assert len(ppoints)>0 - return ppoints \ No newline at end of file + return ppoints + + +def parse_pp_options_with_defaults(pp_kwargs, threads=10, log=True, logger=None, **depr_kwargs): + default_dict = dict(pp_space=10, # default pp spacing + use_pp_zones=False, # don't setup pp by zone, as default + spatial_reference=None, # if not passed pstfrom will use class attrib + # factor calc options + try_use_ppu=True, # default to using ppu + num_threads=threads, # fallback if num_threads not in pp_kwargs, only used if ppu fails + # factor calc options, incl at run time. + minpts_interp=1, + maxpts_interp=20, + search_radius=1e10, + # ult lims + fill_value=1.0 if log else 0., + lower_limit=1.0e-30 if log else -1.0e30, + upper_limit=1.0e30 + ) + # for run time options we need to be strict about dtypes + default_dtype = dict(# pp_space=10, # default pp spacing + # use_pp_zones=False, # don't setup pp by zone, as default + # spatial_reference=None, # if not passed pstfrom will use class attrib + # # factor calc options + # try_use_ppu=True, # default to using ppu + # num_threads=threads, # fallback if num_threads not in pp_kwargs, only used if ppu fails + # # factor calc options, incl at run time. + minpts_interp=int, + maxpts_interp=int, + search_radius=float, + # ult lims + fill_value=float, + lower_limit=float, + upper_limit=float) + + # parse deprecated kwargs first + for key, val in depr_kwargs.items(): + if val is not None: + if key in pp_kwargs: + if logger is not None: + logger.lraise(f"'{key}' passed but its also in 'pp_options") + if logger is not None: + logger.warn(f"Directly passing '{key}' has been deprecated and will eventually be removed" + + f", please use pp_options['{key}'] instead.") + pp_kwargs[key] = val + + # fill defaults + for key, val in default_dict.items(): + if key not in pp_kwargs: + if logger is not None: + logger.statement(f"'{key}' not set in pp_options, " + f"Setting to default value: [{val}]") + pp_kwargs[key] = val + + for key, typ in default_dtype.items(): + pp_kwargs[key] = typ(pp_kwargs[key]) + + return pp_kwargs + + +def prep_pp_hyperpars(file_tag,pp_filename,pp_info,out_filename,grid_dict, + geostruct,arr_shape,pp_options,zone_array=None, + ws = "."): + try: + from pypestutils.pestutilslib import PestUtilsLib + except Exception as e: + raise Exception("prep_pp_hyperpars() error importing pypestutils: '{0}'".format(str(e))) + + illegal_chars = [i for i in r"/:*?<>\|"] + for i in illegal_chars: + if i in file_tag: + print("warning: replacing illegal character '{0}' with '-' in file_tag name '{1}'".format(i,file_tag)) + file_tag = file_tag.replace(i,"-") + + gridinfo_filename = file_tag + ".gridinfo.dat" + corrlen_filename = file_tag + ".corrlen.dat" + bearing_filename = file_tag + ".bearing.dat" + aniso_filename = file_tag + ".aniso.dat" + zone_filename = file_tag + ".zone.dat" + + nodes = list(grid_dict.keys()) + nodes.sort() + with open(os.path.join(ws,gridinfo_filename), 'w') as f: + f.write("node,x,y\n") + for node in nodes: + f.write("{0},{1},{2}\n".format(node, grid_dict[node][0], grid_dict[node][1])) + + corrlen = np.zeros(arr_shape) + geostruct.variograms[0].a + np.savetxt(os.path.join(ws,corrlen_filename), corrlen, fmt="%20.8E") + bearing = np.zeros(arr_shape) + geostruct.variograms[0].bearing + np.savetxt(os.path.join(ws,bearing_filename), bearing, fmt="%20.8E") + aniso = np.zeros(arr_shape) + geostruct.variograms[0].anisotropy + np.savetxt(os.path.join(ws,aniso_filename), aniso, fmt="%20.8E") + + if zone_array is None: + zone_array = np.ones(shape,dtype=int) + np.savetxt(os.path.join(ws,zone_filename),zone_array,fmt="%5d") + + + # fnx_call = "pyemu.utils.apply_ppu_hyperpars('{0}','{1}','{2}','{3}','{4}'". \ + # format(pp_filename, gridinfo_filename, out_filename, corrlen_filename, + # bearing_filename) + # fnx_call += "'{0}',({1},{2}))".format(aniso_filename, arr_shape[0], arr_shape[1]) + + # apply_ppu_hyperpars(pp_filename, gridinfo_filename, out_filename, corrlen_filename, + # bearing_filename, aniso_filename, arr_shape) + + config_df = pd.DataFrame(columns=["value"]) + config_df.index.name = "key" + + config_df.loc["pp_filename", "value"] = pp_filename # this might be in pp_options too in which case does this get stomped on? + config_df.loc["out_filename","value"] = out_filename + config_df.loc["corrlen_filename", "value"] = corrlen_filename + config_df.loc["bearing_filename", "value"] = bearing_filename + config_df.loc["aniso_filename", "value"] = aniso_filename + config_df.loc["gridinfo_filename", "value"] = gridinfo_filename + config_df.loc["zone_filename", "value"] = zone_filename + + config_df.loc["vartransform","value"] = geostruct.transform + v = geostruct.variograms[0] + vartype = 2 + if isinstance(v, pyemu.geostats.ExpVario): + pass + elif isinstance(v, pyemu.geostats.SphVario): + vartype = 1 + elif isinstance(v, pyemu.geostats.GauVario): + vartype = 3 + else: + raise NotImplementedError("unsupported variogram type: {0}".format(str(type(v)))) + krigtype = 1 #ordinary + config_df.loc["vartype","value"] = vartype + config_df.loc["krigtype","value"] = krigtype + config_df.loc["shape", "value"] = arr_shape + + keys = list(pp_options.keys()) + keys.sort() + for k in keys: + config_df.loc[k,"value"] = pp_options[k] + + #config_df.loc["function_call","value"] = fnx_call + config_df_filename = file_tag + ".config.csv" + config_df.loc["config_df_filename",:"value"] = config_df_filename + + config_df.to_csv(os.path.join(ws, config_df_filename)) + # this is just a temp input file needed for testing... + #pp_info.to_csv(os.path.join(ws,pp_filename),sep=" ",header=False) + pyemu.pp_utils.write_pp_file(os.path.join(ws,pp_filename),pp_info) + bd = os.getcwd() + os.chdir(ws) + try: + apply_ppu_hyperpars(config_df_filename) + except Exception as e: + os.chdir(bd) + raise RuntimeError(f"apply_ppu_hyperpars() error: {e}") + os.chdir(bd) + return config_df + + +def apply_ppu_hyperpars(config_df_filename): + try: + from pypestutils.pestutilslib import PestUtilsLib + except Exception as e: + raise Exception("apply_ppu_hyperpars() error importing pypestutils: '{0}'".format(str(e))) + + config_df = pd.read_csv(config_df_filename,index_col=0) + config_dict = config_df["value"].to_dict() + vartransform = config_dict.get("vartransform", "none") + config_dict = parse_pp_options_with_defaults(config_dict, threads=None, log=vartransform=='log') + + out_filename = config_dict["out_filename"] + #pp_info = pd.read_csv(config_dict["pp_filename"],sep="\s+") + pp_info = pyemu.pp_utils.pp_file_to_dataframe(config_dict["pp_filename"]) + grid_df = pd.read_csv(config_dict["gridinfo_filename"]) + corrlen = np.loadtxt(config_dict["corrlen_filename"]) + bearing = np.loadtxt(config_dict["bearing_filename"]) + aniso = np.loadtxt(config_dict["aniso_filename"]) + zone = np.loadtxt(config_dict["zone_filename"]) + + lib = PestUtilsLib() + fac_fname = out_filename+".temp.fac" + if os.path.exists(fac_fname): + os.remove(fac_fname) + fac_ftype = "text" + npts = lib.calc_kriging_factors_2d( + pp_info.x.values, + pp_info.y.values, + pp_info.zone.values, + grid_df.x.values.flatten(), + grid_df.y.values.flatten(), + zone.flatten().astype(int), + int(config_dict.get("vartype",1)), + int(config_dict.get("krigtype",1)), + corrlen.flatten(), + aniso.flatten(), + bearing.flatten(), + # defaults should be in config_dict -- the fallbacks here should not be hit now + config_dict.get("search_dist",config_dict.get("search_radius", 1e10)), + config_dict.get("maxpts_interp",50), + config_dict.get("minpts_interp",1), + fac_fname, + fac_ftype, + ) + + # this is now filled as a default in config_dict if not in config file, + # default value dependent on vartransform (ie. 1 for log 0 for non log) + noint = config_dict.get("fill_value",pp_info.loc[:, "parval1"].mean()) + + result = lib.krige_using_file( + fac_fname, + fac_ftype, + zone.size, + int(config_dict.get("krigtype", 1)), + vartransform, + pp_info["parval1"].values, + noint, + noint, + ) + assert npts == result["icount_interp"] + result = result["targval"] + #shape = tuple([int(s) for s in config_dict["shape"]]) + tup_string = config_dict["shape"] + shape = tuple(int(x) for x in tup_string[1:-1].split(',')) + result = result.reshape(shape) + np.savetxt(out_filename,result,fmt="%20.8E") + os.remove(fac_fname) + lib.free_all_memory() + + return result diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index b3921bc9..ffa6fd9e 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -227,6 +227,7 @@ def __init__( self.original_file_d = None self.mult_file_d = None self.tpl_d = self.new_d + self.pp_solve_num_threads = int(pp_solve_num_threads) if tpl_subfolder is not None: self.tpl_d = Path(self.new_d, tpl_subfolder) self.remove_existing = bool(remove_existing) @@ -239,7 +240,6 @@ def __init__( start_datetime = _get_datetime_from_str(start_datetime) self.start_datetime = start_datetime self.geostruct = None - self.pp_solve_num_threads = int(pp_solve_num_threads) self.par_struct_dict = {} # self.par_struct_dict_l = {} @@ -291,6 +291,14 @@ def __init__( self.chunk_len = int(chunk_len) self.py_functions = set() + self.pre_py_cmds.insert( + 0, + "pyemu.helpers.apply_list_and_array_pars(" + "arr_par_file='mult2model_info.csv',chunk_len={0})".format( + self.chunk_len + ), + ) + @property def parfile_relations(self): @@ -667,7 +675,7 @@ def draw(self, num_reals=100, sigma_range=6, use_specsim=False, scale_offset=Tru delc = self.spatial_reference.delc except Exception as e: pass - + # method moved to helpers pe = pyemu.helpers.draw_by_group(self.pst, num_reals=num_reals, sigma_range=sigma_range, use_specsim=use_specsim, scale_offset=scale_offset, struct_dict=struct_dict, @@ -701,7 +709,6 @@ def build_pst(self, filename=None, update=False, version=1): The new pest control file is assigned an NOPTMAX value of 0 """ - import cProfile pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) @@ -1284,8 +1291,7 @@ def add_py_function( ) func_lines = [] search_str = "def " + function_name + "(" - abet_set = set(string.ascii_uppercase) - abet_set.update(set(string.ascii_lowercase)) + abet_set = set(string.printable) - {' '} with open(file_name, "r") as f: while True: line = f.readline() @@ -1825,8 +1831,8 @@ def add_parameters( use_cols=None, use_rows=None, pargp=None, - pp_space=10, - use_pp_zones=False, + pp_space=None, + use_pp_zones=None, num_eig_kl=100, spatial_reference=None, geostruct=None, @@ -1841,6 +1847,9 @@ def add_parameters( comment_char=None, par_style="multiplier", initial_value=None, + pp_options=None, + apply_order=999, + apply_function=None ): """ Add list or array style model input files to PstFrom object. @@ -1900,18 +1909,9 @@ def add_parameters( using multiple `add_parameters()` calls (e.g. temporal pars) with common geostructs. pp_space (`float`, `int`,`str` or `pd.DataFrame`): Spatial pilot point information. - If `float` or `int`, AND `spatial_reference` is of type VertexGrid, it is the spacing in model length untis between pilot points. - If `int` it is the spacing in rows and cols of where to place pilot points. - If `pd.DataFrame`, then this arg is treated as a prefined set of pilot points - and in this case, the dataframe must have "name", "x", "y", and optionally "zone" columns. - If `str`, then an attempt is made to load a dataframe from a csv file (if `pp_space` ends with ".csv"), - shapefile (if `pp_space` ends with ".shp") or from a pilot points file. If `pp_space` is None, - an integer spacing of 10 is used. Default is None + DEPRECATED : use pp_options['pp_space'] instead. use_pp_zones (`bool`): a flag to use the greater-than-zero values - in the zone_array as pilot point zones. - If False, zone_array values greater than zero are treated as a - single zone. This argument is only used if `pp_space` is None - or `int`. Default is False. + DEPRECATED : use pp_options['use_pp_zones'] instead. num_eig_kl: TODO - impliment with KL pars spatial_reference (`pyemu.helpers.SpatialReference`): If different spatial reference required for pilotpoint setup. @@ -1945,13 +1945,52 @@ def add_parameters( This is not additive with `mfile_skip` option. Warning: currently comment lines within list-style tabular data will be lost. - par_style (`str`): either "m"/"mult"/"multiplier", "a"/"add"/"addend", or "d"/"direct" where the former sets up - up a multiplier and addend parameters process against the existing model input - array and the former sets up a template file to write the model + par_style (`str`): either "m"/"mult"/"multiplier", "a"/"add"/"addend", or "d"/"direct" where + the former sets up a multiplier and addend parameters process against the existing + model input array and the former sets up a template file to write the model input file directly. Default is "multiplier". - initial_value (`float`): the value to set for the `parval1` value in the control file Default is 1.0 + pp_options (`dict`): Various options to control pilot point options. + + Can include: + + * `try_use_ppu` (`bool`) : Flag to attempt to use `PyPestUtils` library to setup and apply pilot points. + Recommended but requires pypestutils in build environment (and forward run env). + (try `conda install pypestutils` or `pip install pypestutils`) + + * `pp_space` (multiple) : Spatial pilot point information. + + If `pp_space` is `float` or `int` type, AND `spatial_reference` is of type VertexGrid : + it is the spacing in model length units between pilot points. + + If `pp_space` is `int` type: it is the spacing in rows and cols of where to place pilot points. + + If `pp_space` is `pd.DataFrame` type: then this arg is treated as a prefined set of pilot points + and in this case, the dataframe must have "name", "x", "y", and optionally "zone" columns. + + If `pp_space` is `str` or path-like: then an attempt is made to load a dataframe from a csv file + (if `pp_space` ends with ".csv"), shapefile (if `pp_space` ends with ".shp") or from a pilot points + file. + + If `pp_space` is None : an integer spacing of 10 is used. Default is None + + * `use_pp_zones` (`bool`) : A flag to use the greater-than-zero values in the `zone_array` as pilot point + zones. If False: zone_array values greater than zero are treated as a single zone. + This argument is only used if `pp_space` is None or `int`. Default is False. + + * `spatial_reference` (`pyemu.helpers.SpatialReference`): If different + spatial reference required for pilot point setup. If None spatial reference passed to `PstFrom()` + will be used for pilot points + + * `prep_hyperpars` (`bool`) : Flag to setup and use pilot point hyper parameters. + (ie anisotropy, bearing, "a") with `PyPestUtils`. Only functions if using `PyPestUtils` (i.e. + `try_use_ppu` is True and pypestutils is successfully located). + + apply_order (`int`): the optional order to process this set of parameters at runtime. + Default is 999. + apply_function (`str`): a python function to call during the apply process at runtime. + Default is None. Returns: `pandas.DataFrame`: dataframe with info for new parameters @@ -1972,11 +2011,15 @@ def add_parameters( # TODO need more support for temporal pars? # - As another partype using index_cols or an additional time_cols + if apply_function is not None: + raise NotImplementedError("apply_function is not implemented") + pp_mult_dict = {} + # TODO support passing par_file (i,j)/(x,y) directly where information # is not contained in model parameter file - e.g. no i,j columns self.add_pars_callcount += 1 self.ijwarned[self.add_pars_callcount] = False - + config_df_filename = None if transform is None: if par_style in ["a", "add", "addend"]: transform = 'none' @@ -2184,13 +2227,13 @@ def add_parameters( # multiplier file name will be taken first par group, if passed # (the same multipliers will apply to all pars passed in this call) # Remove `:` for filenames - # multiplier file needs instance number - # regardless of whether instance is to be included + # multiplier file needs instance number + # regardless of whether instance is to be included # in the parameter names if i == 0: inst = self._next_count(par_name_base[i] +\ chk_prefix) - par_name_store = (par_name_base[0] + + par_name_store = (par_name_base[0] + fmt.format(inst)).replace(":", "") # if instance is to be included in the parameter names # add the instance suffix to the parameter name base @@ -2236,8 +2279,8 @@ def add_parameters( ult_lbound = self.ult_lbound_fill lbfill = "first" - pp_filename = None # setup placeholder variables - fac_filename = None + # pp_filename = None # setup placeholder variables + # fac_filename = None nxs = None # Process model parameter files to produce appropriate pest pars if index_cols is not None: # Assume list/tabular type input files @@ -2267,7 +2310,7 @@ def add_parameters( par_type.startswith("grid") or par_type.startswith("p") ) and geostruct is not None: get_xy = self.get_xy - df, nxs = write_list_tpl( + pp_df, nxs = write_list_tpl( filenames, dfs, par_name_base, @@ -2291,13 +2334,13 @@ def add_parameters( ) nxs = {fname: nx for fname, nx in zip(filenames, nxs)} assert ( - np.mod(len(df), len(use_cols)) == 0.0 + np.mod(len(pp_df), len(use_cols)) == 0.0 ), "Parameter dataframe wrong shape for number of cols {0}" "".format( use_cols ) # variables need to be passed to each row in df - lower_bound = np.tile(lower_bound, int(len(df) / ncol)) - upper_bound = np.tile(upper_bound, int(len(df) / ncol)) + lower_bound = np.tile(lower_bound, int(len(pp_df) / ncol)) + upper_bound = np.tile(upper_bound, int(len(pp_df) / ncol)) self.logger.log( "writing list-style template file '{0}'".format(tpl_filename) ) @@ -2315,7 +2358,7 @@ def add_parameters( "{0} for {1}".format(tpl_filename, par_name_base) ) # Generate array type template - also returns par data - df = write_array_tpl( + pp_df = write_array_tpl( name=par_name_base[0], tpl_filename=tpl_filename, suffix="", @@ -2342,194 +2385,97 @@ def add_parameters( "pilot-points", "pp" }: - if par_style == "d": + from pyemu.utils import pp_utils + # setup pilotpoint style pars + if par_style == "d": # not currently supporting direct self.logger.lraise( "pilot points not supported for 'direct' par_style" ) + # Setup pilotpoints for array type par files self.logger.log("setting up pilot point parameters") - # finding spatial references for for setting up pilot points - if spatial_reference is None: - # if none passed with add_pars call - self.logger.statement( - "No spatial reference " "(containing cell spacing) passed." - ) - if self.spatial_reference is not None: - # using global sr on PstFrom object - self.logger.statement( - "OK - using spatial reference " "in parent object." - ) - spatial_reference = self.spatial_reference - spatial_reference_type = spatial_reference.grid_type - else: - # uhoh - self.logger.lraise( - "No spatial reference in parent " - "object either. " - "Can't set-up pilotpoints" - ) - # check that spatial reference lines up with the original array dimensions - structured = False - if not isinstance(spatial_reference, dict): + + # get pp_options from passed args + # we have a few args that only relate to pp setup + # -- can now group these into pp_options arg, + # just define these "to be deprecated" in separate dict for now + depr_pp_args = {k: v for k, v in locals().items() if k in + ['pp_space', 'use_pp_zones', 'spatial_reference']} + # pull out pp_options or set to empty dict + pp_options = dict([]) if pp_options is None else pp_options + + # We might need a functioning zone_array for what is to come + # this is a change from previous where we were relying on zone_array + # being None to indicate not setting up by zone. Now should be leaning more + # on the "use_pp_zones" arg in pp_options + if zone_array is None: # but need dummy zone array + nr, nc = file_dict[list(file_dict.keys())[0]].shape + zone_array = np.ones((nr, nc), dtype=int) + + # don't want to have to pass too much in on this pp_options dict, + # so define pp_filename here + pp_options['pp_filename'] = "{0}pp.dat".format(par_name_store) # todo could also be a pp_kwarg + # also set parname base from what is extracted above. + pnb = par_name_base[0] + pnb = "pname:{1}_ptype:pp_pstyle:{0}".format(par_style, pnb) + pp_options['pp_basename'] = pnb + # pp_options passed as a dict (and returned) after modification + pp_options = pp_utils.parse_pp_options_with_defaults( + pp_options, + self.pp_solve_num_threads, + transform =='log', + self.logger, + **depr_pp_args + ) + + # zone array is reset to ar>1=1 if not using pp zones + pp_options = self._prep_pp_args(zone_array, pp_options) + # collect zone_array back (used later) + zone_array = pp_options['zone_array'] + # pp_utils.setup_pilotpoints_grid will write a tpl file + # with the name take from pp_filename_dict. (pp_filename+".tpl") + # and pp_filename comes from "{0}pp.dat".format(par_name_store) + # par_name_store comes from par_name base with instants increment + # better to make tpl consistent between method? + tpl_filename = pp_options['pp_tpl'] = self.tpl_d / (pp_options['pp_filename'] + ".tpl") + # in_filepst is a general variable used to fill input file list + in_filepst = pp_filename = pp_options['pp_filename'] + + # Additional check that spatial reference lines up with the original array dimensions + # and defining grid type and struct or unstruct -- for use later + spatial_reference = pp_options['spatial_reference'] + if isinstance(spatial_reference, dict): # unstruct + structured = False + else: # this is then a structured grid + spatial_reference_type = spatial_reference.grid_type structured = True + # slightly different check needed for vertex type + if spatial_reference_type == 'vertex': + checkref = {0: ['ncpl', spatial_reference.ncpl]} + else: + checkref = {0: ['nrow', spatial_reference.nrow], + 1: ['ncol', spatial_reference.ncol]} + # loop files and dimensions for mod_file, ar in file_dict.items(): orgdata = ar.shape - if spatial_reference_type == 'vertex': - assert orgdata[0] == spatial_reference.ncpl, ( - "Spatial reference ncpl not equal to original data ncpl for\n" - + os.path.join( - *os.path.split(self.original_file_d)[1:], mod_file - ) - ) - else: - assert orgdata[0] == spatial_reference.nrow, ( - "Spatial reference nrow not equal to original data nrow for\n" - + os.path.join( - *os.path.split(self.original_file_d)[1:], mod_file - ) - ) - assert orgdata[1] == spatial_reference.ncol, ( - "Spatial reference ncol not equal to original data ncol for\n" + for i, chk in checkref.items(): + assert orgdata[i] == chk[1], ( + f"Spatial reference {chk[0]} not equal to original data {chk[0]} for\n" + os.path.join( *os.path.split(self.original_file_d)[1:], mod_file ) ) - # (stolen from helpers.PstFromFlopyModel()._pp_prep()) - # but only settting up one set of pps at a time - pnb = par_name_base[0] - pnb = "pname:{1}_ptype:pp_pstyle:{0}".format(par_style, pnb) - pp_dict = {0: pnb} - pp_filename = "{0}pp.dat".format(par_name_store) - # pst inputfile (for tpl->in pair) is - # par_name_storepp.dat table (in pst ws) - in_filepst = pp_filename - pp_filename_dict = {pnb: in_filepst} - tpl_filename = self.tpl_d / (pp_filename + ".tpl") - # tpl_filename = get_relative_filepath(self.new_d, tpl_filename) - pp_locs = None - if pp_space is None: # default spacing if not passed - self.logger.warn("pp_space is None, using 10...\n") - pp_space = 10 - else: - if not use_pp_zones and (isinstance(pp_space, (int, np.integer))): - # if not using pp zones will set up pp for just one - # zone (all non zero) -- for active domain... - if zone_array is None: - nr, nc = file_dict[list(file_dict.keys())[0]].shape - zone_array = np.ones((nr,nc)) - zone_array[zone_array > 0] = 1 # so can set all - # gt-zero to 1 - if isinstance(pp_space, float): - pp_space = int(pp_space) - elif isinstance(pp_space, (int, np.integer)): - pass - elif isinstance(pp_space, str): - if pp_space.lower().strip().endswith(".csv"): - self.logger.statement( - "trying to load pilot point location info from csv file '{0}'".format( - self.new_d / Path(pp_space) - ) - ) - pp_locs = pd.read_csv(self.new_d / pp_space) - - elif pp_space.lower().strip().endswith(".shp"): - self.logger.statement( - "trying to load pilot point location info from shapefile '{0}'".format( - self.new_d / Path(pp_space) - ) - ) - pp_locs = pyemu.pp_utils.pilot_points_from_shapefile( - str(self.new_d / Path(pp_space)) - ) - else: - self.logger.statement( - "trying to load pilot point location info from pilot point file '{0}'".format( - self.new_d / Path(pp_space) - ) - ) - pp_locs = pyemu.pp_utils.pp_file_to_dataframe( - self.new_d / pp_space - ) - self.logger.statement( - "pilot points found in file '{0}' will be transferred to '{1}' for parameterization".format( - pp_space, pp_filename - ) - ) - elif isinstance(pp_space, pd.DataFrame): - pp_locs = pp_space - else: - self.logger.lraise( - "unrecognized 'pp_space' value, should be int, csv file, pp file or dataframe, not '{0}'".format( - type(pp_space) - ) - ) - if pp_locs is not None: - cols = pp_locs.columns.tolist() - if "name" not in cols: - self.logger.lraise("'name' col not found in pp dataframe") - if "x" not in cols: - self.logger.lraise("'x' col not found in pp dataframe") - if "y" not in cols: - self.logger.lraise("'y' col not found in pp dataframe") - if "zone" not in cols: - self.logger.warn( - "'zone' col not found in pp dataframe, adding generic zone" - ) - pp_locs.loc[:, "zone"] = 1 - - elif zone_array is not None: - # check that all the zones in the pp df are in the zone array - missing = [] - for uz in pp_locs.zone.unique(): - if int(uz) not in zone_array: - missing.append(str(uz)) - if len(missing) > 0: - self.logger.lraise( - "the following pp zone values were not found in the zone array: {0}".format( - ",".join(missing) - ) - ) - - for uz in np.unique(zone_array): - if uz < 1: - continue - if uz not in pp_locs.zone.values: - missing.append(str(uz)) - if len(missing) > 0: - self.logger.warn( - "the following zones don't have any pilot points:{0}".format( - ",".join(missing) - ) - ) - - if not structured and pp_locs is None: - self.logger.lraise( - "pilot point type parameters with an unstructured grid requires 'pp_space' " - "contain explict pilot point information" - ) - - if not structured and zone_array is not None: - # self.logger.lraise("'zone_array' not supported for unstructured grids and pilot points") - if "zone" not in pp_locs.columns: - self.logger.lraise( - "'zone' not found in pp info dataframe and 'zone_array' passed" - ) - uvals = np.unique(zone_array) - zvals = set([int(z) for z in pp_locs.zone.tolist()]) - missing = [] - for uval in uvals: - if int(uval) not in zvals and int(uval) != 0: - missing.append(str(int(uval))) - if len(missing) > 0: - self.logger.warn( - "the following values in the zone array were not found in the pp info: {0}".format( - ",".join(missing) - ) - ) + # Use pp_options kwargs dict in pp setup + pp_df = self._setup_pp_df(**pp_options) + # set par group -- already defined above + pp_df.loc[:, "pargp"] = pargp + self.logger.statement("pilot point 'pargp':{0}".format(",".join(pargp))) + self.logger.log("setting up pilot point parameters") + # start working on interp factor calcs + # check on geostruct for pilotpoints -- something is required! if geostruct is None: # need a geostruct for pilotpoints - # can use model default, if provided if self.geostruct is None: # but if no geostruct passed... if not structured: @@ -2542,11 +2488,43 @@ def add_parameters( "using ExpVario with contribution=1 " "and a=(pp_space*max(delr,delc))" ) - # set up a default - could probably do something better if pp locs are passed - if not isinstance(pp_space, (int, np.integer)): + # set up a default - + # TODO could probably do something better if pp locs are passed + # How about this?!?? + # if not isinstance(pp_options["pp_space"], (int, np.integer)): + # self.logger.warn( + # "pp_space is not defined, " + # "attempting to extract pp_dist (a) from " + # "pp_locs" + # ) + # try: + # pp_dist = pp_options["pp_locs"][['x', 'y']].apply( + # lambda xy: sorted( # sortign to extract min non zero + # ((pp_options["pp_locs"][['x', 'y']] - xy) ** 2).sum(axis=1) ** 0.5)[1], axis=1 + # ).mean() + # except: + # self.logger.warn( + # "Unable to extract pp_dist from pp_locs, " + # "reverting to dist defined by 10 cells." + # ) + # # default to 10 cells spacing + # pp_dist = 10 * float( + # max( + # spatial_reference.delr.max(), + # spatial_reference.delc.max(), + # ) + # ) + # else: + # pp_dist = pp_options["pp_space"] * float( + # max( + # spatial_reference.delr.max(), + # spatial_reference.delc.max(), + # ) + # ) + if not isinstance(pp_options["pp_space"], (int, np.integer)): space = 10 else: - space = pp_space + space = pp_options["pp_space"] pp_dist = space * float( max( spatial_reference.delr.max(), @@ -2581,137 +2559,167 @@ def add_parameters( else: pp_geostruct = geostruct - if pp_locs is None: - # Set up pilot points - - df = pyemu.pp_utils.setup_pilotpoints_grid( - sr=spatial_reference, - ibound=zone_array, - use_ibound_zones=use_pp_zones, - prefix_dict=pp_dict, - every_n_cell=pp_space, - pp_dir=self.new_d, - tpl_dir=self.tpl_d, - shapename=str(self.new_d / "{0}.shp".format(par_name_store)), - pp_filename_dict=pp_filename_dict, - ) - else: - - df = pyemu.pp_utils.pilot_points_to_tpl( - pp_locs, - tpl_filename, - pnb, - ) - df["tpl_filename"] = tpl_filename - df["pp_filename"] = pp_filename - df.loc[:, "pargp"] = pargp - df.set_index("parnme", drop=False, inplace=True) - # df includes most of the par info for par_dfs and also for - # relate_parfiles - self.logger.statement( - "{0} pilot point parameters created".format(df.shape[0]) - ) - # should be only one group at a time - pargp = df.pargp.unique() - self.logger.statement("pilot point 'pargp':{0}".format(",".join(pargp))) - self.logger.log("setting up pilot point parameters") - # Calculating pp factors - pg = pargp[0] - # this reletvively quick - ok_pp = pyemu.geostats.OrdinaryKrige(pp_geostruct, df) - # build krige reference information on the fly - used to help - # prevent unnecessary krig factor calculation - pp_info_dict = { - "pp_data": ok_pp.point_data.loc[:, ["x", "y", "zone"]], - "cov": ok_pp.point_cov_df, - "zn_ar": zone_array, - "sr": spatial_reference, - "pstyle":par_style, - "transform":transform - } - fac_processed = False - for facfile, info in self._pp_facs.items(): # check against - # factors already calculated - if ( - info["pp_data"].equals(pp_info_dict["pp_data"]) - and info["cov"].equals(pp_info_dict["cov"]) - and np.array_equal(info["zn_ar"], pp_info_dict["zn_ar"]) - and pp_info_dict["pstyle"] == info["pstyle"] - and pp_info_dict["transform"] == info["transform"] - - ): - if type(info["sr"]) == type(spatial_reference): - if isinstance(spatial_reference, dict): - if len(info["sr"]) != len(spatial_reference): - continue - else: - continue - - fac_processed = True # don't need to re-calc same factors - fac_filename = facfile # relate to existing fac file - self.logger.statement("reusing factors") - break - if not fac_processed: - # TODO need better way of naming sequential fac_files? - self.logger.log("calculating factors for pargp={0}".format(pg)) - fac_filename = self.new_d / "{0}pp.fac".format(par_name_store) - var_filename = fac_filename.with_suffix(".var.dat") - self.logger.statement( - "saving krige variance file:{0}".format(var_filename) - ) - self.logger.statement( - "saving krige factors file:{0}".format(fac_filename) - ) - # store info on pilotpoints - self._pp_facs[fac_filename] = pp_info_dict - # this is slow (esp on windows) so only want to do this - # when required + pg = pargp + # getting hyperpars request + prep_pp_hyperpars = pp_options.get("prep_hyperpars",False) + pp_locs = pp_options["pp_locs"] + pp_mult_dict = {} + if prep_pp_hyperpars: if structured: - ok_pp.calc_factors_grid( - spatial_reference, - var_filename=var_filename, - zone_array=zone_array, - num_threads=self.pp_solve_num_threads, - ) - ok_pp.to_grid_factors_file(fac_filename) + grid_dict = {} + for inode,(xx,yy) in enumerate(zip(spatial_reference.xcentergrid.flatten(), + spatial_reference.ycentergrid.flatten())): + grid_dict[inode] = (xx,yy) else: - # put the sr dict info into a df - # but we only want to use the n - if zone_array is not None: - for zone in np.unique(zone_array): - if int(zone) == 0: - continue + grid_dict = spatial_reference + #prep_pp_hyperpars(file_tag,pp_filename,out_filename,grid_dict, + # geostruct,arr_shape,pp_options,zone_array=None) + if structured: + shape = spatial_reference.xcentergrid.shape + else: + shape = (1,len(grid_dict)) + + config_df = pp_utils.prep_pp_hyperpars( + pg, + pp_filename, + pp_df, + os.path.join("mult", mlt_filename), + grid_dict, + pp_geostruct, + shape, + pp_options, + zone_array=zone_array, + ws=self.new_d + ) + #todo: add call to apply func ahead of call to mult func + config_df_filename = config_df.loc["config_df_filename","value"] + #self.pre_py_cmds.insert(0,"pyemu.utils.apply_ppu_hyperpars('{0}')".\ + # format(config_df_filename)) + + #if "pypestutils" not in self.extra_py_imports: + # self.extra_py_imports.append("pypestutils") + print(config_df_filename) + config_func_str = "pyemu.utils.pp_utils.apply_ppu_hyperpars('{0}')".\ + format(config_df_filename) + pp_mult_dict["pre_apply_function"] = config_func_str + else: + # this reletvively quick + ok_pp = pyemu.geostats.OrdinaryKrige(pp_geostruct, pp_df) + # build krige reference information on the fly - used to help + # prevent unnecessary krig factor calculation + pp_info_dict = { + "pp_data": ok_pp.point_data.loc[:, ["x", "y", "zone"]], + "cov": ok_pp.point_cov_df, + "zn_ar": zone_array, + "sr": spatial_reference, + "pstyle":par_style, + "transform":transform + } + fac_processed = False + for facfile, info in self._pp_facs.items(): # check against + # factors already calculated + if ( + info["pp_data"].equals(pp_info_dict["pp_data"]) + and info["cov"].equals(pp_info_dict["cov"]) + and np.array_equal(info["zn_ar"], pp_info_dict["zn_ar"]) + and pp_info_dict["pstyle"] == info["pstyle"] + and pp_info_dict["transform"] == info["transform"] + + ): + if type(info["sr"]) == type(spatial_reference): + if isinstance(spatial_reference, dict): + if len(info["sr"]) != len(spatial_reference): + continue + else: + continue + fac_processed = True # don't need to re-calc same factors + fac_filename = facfile # relate to existing fac file + self.logger.statement("reusing factors") + break + if not fac_processed: + # TODO need better way of naming sequential fac_files? + self.logger.log("calculating factors for pargp={0}".format(pg)) + fac_filename = self.new_d / "{0}pp.fac".format(par_name_store) + var_filename = fac_filename.with_suffix(".var.dat") + self.logger.statement( + "saving krige variance file:{0}".format(var_filename) + ) + self.logger.statement( + "saving krige factors file:{0}".format(fac_filename) + ) + # store info on pilotpoints + self._pp_facs[fac_filename] = pp_info_dict + # this is slow (esp on windows) so only want to do this + # when required + if structured: + + ret_val = ok_pp.calc_factors_grid( + spatial_reference, + var_filename=var_filename, + zone_array=zone_array, + num_threads=pp_options.get("num_threads",self.pp_solve_num_threads), + minpts_interp=pp_options.get("minpts_interp",1), + maxpts_interp=pp_options.get("maxpts_interp",20), + search_radius=pp_options.get("search_radius",1e10), + try_use_ppu=pp_options.get("try_use_ppu",True), + #ppu_factor_filename=pp_options.get("ppu_factor_filename","factors.dat") + ppu_factor_filename=fac_filename + ) + if not isinstance(ret_val,int): + ok_pp.to_grid_factors_file(fac_filename) + else: + # put the sr dict info into a df + # but we only want to use the n + if zone_array is not None: + for zone in np.unique(zone_array): + if int(zone) == 0: + continue + data = [] + for node, (x, y) in spatial_reference.items(): + if zone_array[0, node] == zone: + data.append([node, x, y]) + if len(data) == 0: + continue + node_df = pd.DataFrame(data, columns=["node", "x", "y"]) + ok_pp.calc_factors( + node_df.x, + node_df.y, + num_threads=pp_options.get("num_threads", self.pp_solve_num_threads), + minpts_interp=pp_options.get("minpts_interp", 1), + maxpts_interp=pp_options.get("maxpts_interp", 20), + search_radius=pp_options.get("search_radius", 1e10), + pt_zone=zone, + idx_vals=node_df.node.astype(int), + ) + ok_pp.to_grid_factors_file( + fac_filename, ncol=len(spatial_reference) + ) + else: data = [] for node, (x, y) in spatial_reference.items(): - if zone_array[0, node] == zone: - data.append([node, x, y]) - if len(data) == 0: - continue + data.append([node, x, y]) node_df = pd.DataFrame(data, columns=["node", "x", "y"]) - ok_pp.calc_factors( - node_df.x, - node_df.y, - num_threads=self.pp_solve_num_threads, - pt_zone=zone, - idx_vals=node_df.node.astype(int), + ok_pp.calc_factors(node_df.x, node_df.y, + num_threads=pp_options.get("num_threads", self.pp_solve_num_threads)) + ok_pp.to_grid_factors_file( + fac_filename, ncol=node_df.shape[0] ) - ok_pp.to_grid_factors_file( - fac_filename, ncol=len(spatial_reference) - ) - else: - data = [] - for node, (x, y) in spatial_reference.items(): - data.append([node, x, y]) - node_df = pd.DataFrame(data, columns=["node", "x", "y"]) - ok_pp.calc_factors(node_df.x, node_df.y, num_threads=self.pp_solve_num_threads) - ok_pp.to_grid_factors_file( - fac_filename, ncol=node_df.shape[0] - ) + self.logger.log("calculating factors for pargp={0}".format(pg)) + # if pilotpoint need to store more info + assert fac_filename is not None, "missing pilot-point input filename" + pp_mult_dict["fac_file"] = os.path.relpath(fac_filename, self.new_d) + pp_mult_dict["pp_file"] = pp_filename + if transform == "log": + pp_mult_dict["pp_fill_value"] = pp_options.get("fill_value", 1.0) + pp_mult_dict["pp_lower_limit"] = pp_options.get("lower_limit", 1.0e-30) + pp_mult_dict["pp_upper_limit"] = pp_options.get("upper_limit", 1.0e30) + else: + pp_mult_dict["pp_fill_value"] = pp_options.get("fill_value", 0.0) + pp_mult_dict["pp_lower_limit"] = pp_options.get("lower_limit", -1.0e30) + pp_mult_dict["pp_upper_limit"] = pp_options.get("upper_limit", 1.0e30) - self.logger.log("calculating factors for pargp={0}".format(pg)) - # TODO - other par types - JTW? elif par_type == "kl": self.logger.lraise("array type 'kl' not implemented") else: @@ -2727,8 +2735,8 @@ def add_parameters( if datetime is not None: # add time info to par_dfs - df["datetime"] = datetime - df["timedelta"] = t_offest + pp_df["datetime"] = datetime + pp_df["timedelta"] = t_offest # accumulate information that relates mult_files (set-up here and # eventually filled by PEST) to actual model files so that actual # model input file can be generated @@ -2763,34 +2771,24 @@ def add_parameters( mult_dict["chkpar"] = nxs[mod_file] if par_style in ["m", "a"]: mult_dict["mlt_file"] = Path(self.mult_file_d.name, mlt_filename) - - if pp_filename is not None: - # if pilotpoint need to store more info - assert fac_filename is not None, "missing pilot-point input filename" - mult_dict["fac_file"] = os.path.relpath(fac_filename, self.new_d) - mult_dict["pp_file"] = pp_filename - if transform == "log": - mult_dict["pp_fill_value"] = 1.0 - mult_dict["pp_lower_limit"] = 1.0e-30 - mult_dict["pp_upper_limit"] = 1.0e30 - else: - mult_dict["pp_fill_value"] = 0.0 - mult_dict["pp_lower_limit"] = -1.0e30 - mult_dict["pp_upper_limit"] = 1.0e30 + # add pp specific info + mult_dict.update(pp_mult_dict) if zone_filename is not None: mult_dict["zone_file"] = zone_filename relate_parfiles.append(mult_dict) relate_pars_df = pd.DataFrame(relate_parfiles) + relate_pars_df["apply_order"] = apply_order + # store on self for use in pest build etc self._parfile_relations.append(relate_pars_df) # add cols required for pst.parameter_data - df.loc[:, "partype"] = par_type - df.loc[:, "partrans"] = transform - df.loc[:, "parubnd"] = upper_bound - df.loc[:, "parlbnd"] = lower_bound + pp_df.loc[:, "partype"] = par_type + pp_df.loc[:, "partrans"] = transform + pp_df.loc[:, "parubnd"] = upper_bound + pp_df.loc[:, "parlbnd"] = lower_bound if par_style != "d": - df.loc[:, "parval1"] = initial_value + pp_df.loc[:, "parval1"] = initial_value # df.loc[:,"tpl_filename"] = tpl_filename # store tpl --> in filename pair @@ -2803,10 +2801,10 @@ def add_parameters( # add pars to par_data list BH: is this what we want? # - BH: think we can get away with dropping duplicates? - missing = set(par_data_cols) - set(df.columns) + missing = set(par_data_cols) - set(pp_df.columns) for field in missing: # fill missing pst.parameter_data cols with defaults - df[field] = pyemu.pst_utils.pst_config["par_defaults"][field] - df = df.drop_duplicates(subset="parnme") # drop pars that appear multiple times + pp_df[field] = pyemu.pst_utils.pst_config["par_defaults"][field] + pp_df = pp_df.drop_duplicates(subset="parnme") # drop pars that appear multiple times # df = df.loc[:, par_data_cols] # just storing pst required cols # - need to store more for cov builder (e.g. x,y) # TODO - check when self.par_dfs gets used @@ -2815,19 +2813,19 @@ def add_parameters( # only add pardata for new parameters # some parameters might already be in a par_df if they occur # in more than one instance (layer, for example) - new_parnmes = set(df['parnme']).difference(self.unique_parnmes) - df = df.loc[df['parnme'].isin(new_parnmes)].copy() - self.par_dfs.append(df) + new_parnmes = set(pp_df['parnme']).difference(self.unique_parnmes) + pp_df = pp_df.loc[pp_df['parnme'].isin(new_parnmes)].copy() + self.par_dfs.append(pp_df) self.unique_parnmes.update(new_parnmes) # pivot df to list of df per par group in this call # (all groups will be related to same geostruct) # TODO maybe use different marker to denote a relationship between pars # at the moment relating pars using common geostruct and pargp but may # want to reserve pargp for just PEST - if "covgp" not in df.columns: - gp_dict = {g: [d] for g, d in df.groupby("pargp")} + if "covgp" not in pp_df.columns: + gp_dict = {g: [d] for g, d in pp_df.groupby("pargp")} else: - gp_dict = {g: [d] for g, d in df.groupby("covgp")} + gp_dict = {g: [d] for g, d in pp_df.groupby("covgp")} # df_list = [d for g, d in df.groupby('pargp')] if geostruct is not None and ( par_type.lower() not in ["constant", "zone"] or datetime is not None @@ -2852,7 +2850,7 @@ def add_parameters( self.par_struct_dict[geostruct][gp].extend(gppars) # self.par_struct_dict_l[geostruct].extend(list(gp_dict.values())) else: # TODO some rules for if geostruct is not passed.... - if "x" in df.columns: + if "x" in pp_df.columns: pass # TODO warn that it looks like spatial pars but no geostruct? # if self.geostruct is not None: @@ -2891,6 +2889,198 @@ def add_parameters( self.logger.warn( "pst object not available, " "new control file will be written" ) + return pp_df + + + def _prep_pp_args(self, zone_array, pp_kwargs=None): + if pp_kwargs is None: + pp_kwargs = dict([]) + + if not pp_kwargs["use_pp_zones"]: + # will set up pp for just one + # zone (all non zero) -- for active domain... + zone_array[zone_array > 0] = 1 # so can set all + # gt-zero to 1 + use_ppu = pp_kwargs.get('try_use_ppu', True) + hyper = pp_kwargs.get('prep_hyperpars', False) + if hyper: + if not use_ppu: + self.logger.warn("`prep_hyperpars` requested but `try_use_ppu` set to False.\n" + "Setting `try_use_ppu` to True and testing import") + use_ppu = True + if use_ppu: + try: + import pypestutils as ppu + except ImportError as e: + if hyper: + raise ImportError(f"`prep_hyperpars` needs pypestutils : {e}") + else: + self.logger.warn("`use_ppu` requested but failed to import\n" + "falling back to pyemu pp methods") + + # extra check for spatial ref + if pp_kwargs.get('spatial_reference', None) is None: + self.logger.statement( + "No spatial reference " "(containing cell spacing) passed." + ) + if self.spatial_reference is not None: + # using global sr on PstFrom object + self.logger.statement( + "OK - using spatial reference " "in parent object." + ) + pp_kwargs['spatial_reference'] = self.spatial_reference + else: + # uhoh + self.logger.lraise( + "No spatial reference passed and none in parent object." + "Can't set-up pilotpoint pars" + ) + + # get pp_locs arg from pp_space + pp_filename = pp_kwargs['pp_filename'] + pp_space = pp_kwargs['pp_space'] + # default pp_locs to None # todo -- could this be a pp_kwargs passed to add pars? + pp_locs = None + if isinstance(pp_space, float): + pp_space = int(pp_space) + elif isinstance(pp_space, str): + if pp_space.strip().lower().endswith(".csv"): + self.logger.statement( + "trying to load pilot point location info from csv file '{0}'".format( + self.new_d / Path(pp_space) + ) + ) + pp_locs = pd.read_csv(self.new_d / pp_space) + + elif pp_space.strip().lower().endswith(".shp"): + self.logger.statement( + "trying to load pilot point location info from shapefile '{0}'".format( + self.new_d / Path(pp_space) + ) + ) + pp_locs = pyemu.pp_utils.pilot_points_from_shapefile( + str(self.new_d / Path(pp_space)) + ) + else: + self.logger.statement( + "trying to load pilot point location info from pilot point file '{0}'".format( + self.new_d / Path(pp_space) + ) + ) + pp_locs = pyemu.pp_utils.pp_file_to_dataframe( + self.new_d / Path(pp_space) + ) + self.logger.statement( + "pilot points found in file '{0}' will be transferred to '{1}' for parameterization".format( + pp_space, pp_filename + ) + ) + elif isinstance(pp_space, pd.DataFrame): + pp_locs = pp_space + elif not isinstance(pp_space, (int, np.integer)): + self.logger.lraise( + "unrecognized pp_options['pp_space'] value, should be int, csv file, pp file or dataframe, not '{0}'".format( + type(pp_space) + ) + ) + if pp_locs is None: + if isinstance(pp_kwargs['spatial_reference'], dict): # then we are unstructured and need pp_locs + self.logger.lraise( + "pilot point type parameters with an unstructured grid requires 'pp_space' " + "contain explict pilot point information" + ) + else: # check pp_locs (if not None) + cols = pp_locs.columns.tolist() + if "name" not in cols: + self.logger.lraise("'name' col not found in pp dataframe") + if "x" not in cols: + self.logger.lraise("'x' col not found in pp dataframe") + if "y" not in cols: + self.logger.lraise("'y' col not found in pp dataframe") + if "zone" not in cols: + self.logger.warn( + "'zone' col not found in pp dataframe, adding generic zone" + ) + pp_locs.loc[:, "zone"] = 1 + if pp_kwargs["use_pp_zones"]: + # check that all the zones in the pp_locs are in the zone array + missing = set(pp_locs.zone.unique()) - set(np.unique(zone_array)) + if len(missing) > 0: + self.logger.lraise( + "the following pp zone values were not found in the zone array: {0}".format( + ",".join(str(m) for m in missing) + ) + ) + missing = set(np.unique(zone_array)) - set(pp_locs.zone.unique()) - {0} + if len(missing) > 0: + self.logger.warn( + "the following zones (in zone_array) don't have any pilot points:{0}".format( + ",".join(str(m) for m in missing) + ) + ) + pp_kwargs.update(dict(pp_space=pp_space, pp_locs=pp_locs, + zone_array=zone_array)) + + return pp_kwargs + + def _setup_pp_df( + self, + pp_filename=None, + pp_basename=None, + pp_space=None, + pp_locs=None, + use_pp_zones=None, + zone_array=None, + spatial_reference=None, + pp_tpl=None, + **kwargs + + ): + # a few essentials + assert pp_filename is not None, "No arg passed for pp_filename" + assert pp_basename is not None, "No arg passed for pp_basename" + assert pp_tpl is not None, "No arg passed for pp_tpl" + if pp_locs is None: + # some more essentials if not passed pp_locs + assert pp_space is not None, "If pp_locs is not pp_space should be int." + assert use_pp_zones is not None, "If pp_locs is not use_pp_zones should be bool." + assert spatial_reference is not None, "If pp_locs is not spatial_reference should be passed." + # define a shape file -- incidental + shp_fname = str(self.new_d / "{0}.shp".format(pp_filename)) + # Set up pilot points + pp_dict = {0: pp_basename} + pp_filename_dict = {pp_basename: pp_filename} + df = pyemu.pp_utils.setup_pilotpoints_grid( + sr=spatial_reference, + ibound=zone_array, + use_ibound_zones=use_pp_zones, + prefix_dict=pp_dict, + every_n_cell=pp_space, + pp_dir=self.new_d, + tpl_dir=self.tpl_d, + shapename=shp_fname, + pp_filename_dict=pp_filename_dict, + ) + else: + # build tpl from pp_locs + # todo -- do we lose flexibility here regarding where tpls are saved + # should the tpl file we pass be Path(self.tpl_d)/pp_tpl? + df = pyemu.pp_utils.pilot_points_to_tpl( + pp_locs, + pp_tpl, + pp_basename, + ) + if "tpl_filename" not in df.columns: + df["tpl_filename"] = pp_tpl + if "pp_filename" not in df.columns: + df["pp_filename"] = pp_filename + df.set_index("parnme", drop=False, inplace=True) + # pp_locs = df + # df includes most of the par info for par_dfs and also for + # relate_parfiles + self.logger.statement( + "{0} pilot point parameters created".format(df.shape[0]) + ) return df def _load_listtype_file( diff --git a/pyproject.toml b/pyproject.toml index 8335094d..c85d1e5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,8 @@ optional = [ "pyshp", "scipy", "shapely", - "jinja2" # for to_latex options + "jinja2", # for to_latex options + "pypestutils" ] test = [ "coveralls",