From 6b3003ec0b73bfc3d668fa834e2cd4bb35ee832d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 16 Jan 2025 15:56:37 -0500 Subject: [PATCH 01/20] #315 Add fre list tools - list experiments - list platforms --- fre/list/frelist.py | 28 ++++++++---- fre/list/frelistexample.py | 18 -------- fre/list/list_experiments_script.py | 69 +++++++++++++++++++++++++++++ fre/list/list_platforms_script.py | 64 ++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 26 deletions(-) delete mode 100644 fre/list/frelistexample.py create mode 100644 fre/list/list_experiments_script.py create mode 100644 fre/list/list_platforms_script.py diff --git a/fre/list/frelist.py b/fre/list/frelist.py index fea3b6ea..03498cee 100644 --- a/fre/list/frelist.py +++ b/fre/list/frelist.py @@ -1,20 +1,32 @@ ''' fre list ''' import click - -from .frelistexample import list_test_function +from fre.list import list_experiments_script +from fre.list import list_platforms_script @click.group(help=click.style(" - access fre list subcommands", fg=(232,204,91))) def list_cli(): ''' entry point to fre list click commands ''' @list_cli.command() -@click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.') -@click.pass_context -def function(context, uppercase): - # pylint: disable=unused-argument - """ - Execute fre list test """ - context.forward(list_test_function) +@click.option("-y", + "--yamlfile", + type=str, + help="YAML file to be used for parsing", + required=True) +def list_exps(yamlfile): + """ - List experiments available""" + list_experiments_script.list_experiments_subtool(yamlfile) + +@list_cli.command() +@click.option("-y", + "--yamlfile", + type=str, + help="YAML file to be used for parsing", + required=True) +def list_platforms(yamlfile): + """ - List platforms available """ + list_platforms_script.list_platforms_subtool(yamlfile) if __name__ == "__main__": list_cli() diff --git a/fre/list/frelistexample.py b/fre/list/frelistexample.py deleted file mode 100644 index 7e40e195..00000000 --- a/fre/list/frelistexample.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -experimentation file for integrating one file's functions into main prototype fre file -authored by Bennett.Chang@noaa.gov | bcc2761 -NOAA | GFDL -""" - -import click - -@click.command() -def list_test_function(uppercase=None): - """Execute fre list testfunction2.""" - statement = "testingtestingtestingtesting" - if uppercase: - statement = statement.upper() - click.echo(statement) - -if __name__ == '__main__': - list_test_function() diff --git a/fre/list/list_experiments_script.py b/fre/list/list_experiments_script.py new file mode 100644 index 00000000..d962fa3e --- /dev/null +++ b/fre/list/list_experiments_script.py @@ -0,0 +1,69 @@ +""" +Script combines the model yaml with exp, platform, and target to list experiment information. +""" +import os +from pathlib import Path +import click +import yaml +import fre.yamltools.combine_yamls as cy + +def join_constructor(loader, node): + """ + Allows FRE properties defined + in main yaml to be concatenated. + """ + seq = loader.construct_sequence(node) + return ''.join([str(i) for i in seq]) + +class NoAliasDumper(yaml.SafeDumper): + def ignore_aliases(self, data): + return True + +def yaml_load(yamlfile): + """ + Load the yamlfile + """ + with open(yamlfile, 'r') as yf: + y = yaml.load(yf,Loader=yaml.Loader) + + return y + +def quick_combine(yml, exp, platform, target): + """ + Create intermediate combined model and exp. yaml + """ + # Combine model / experiment + comb = cy.init_pp_yaml(yml,exp,platform,target) + comb.combine_model() + +def clean(combined): + """ + Remove intermediate combined yaml. + """ + if Path(combined).exists(): + os.remove(combined) + print(f"{combined} removed.") + +def list_experiments_subtool(yamlfile): + """ + List the post-processing experiments available + """ + # Regsiter tag handler + yaml.add_constructor('!join', join_constructor) + + e = "exp_placeholder" + p = "p_placeholder" + t = "t_placeholder" + + combined=f"combined-{e}.yaml" + # Combine model / experiment + quick_combine(yamlfile,e,p,t) + + # Print experiment names + c = yaml_load(combined) + + print("\nPost-processing experiments available:") + for i in c.get("experiments"): + print(f' - {i.get("name")}') + print("\n") + clean(combined) diff --git a/fre/list/list_platforms_script.py b/fre/list/list_platforms_script.py new file mode 100644 index 00000000..180a29c1 --- /dev/null +++ b/fre/list/list_platforms_script.py @@ -0,0 +1,64 @@ +""" +Script combines the model yaml with exp, platform, and target to list experiment information. +""" + +import os +import click +import yaml +import fre.yamltools.combine_yamls as cy +from pathlib import Path + +def join_constructor(loader, node): + """ + Allows FRE properties defined + in main yaml to be concatenated. + """ + seq = loader.construct_sequence(node) + return ''.join([str(i) for i in seq]) + +#class NoAliasDumper(yaml.SafeDumper): +# def ignore_aliases(self, data): +# return True + +def yaml_load(yamlfile): + """ + Load the yamlfile + """ + with open(yamlfile, 'r') as yf: + y = yaml.load(yf,Loader=yaml.Loader) + + return y + +def quick_combine(yml, exp, platform, target,combined): + """ + """ + # Combine model / experiment + comb = cy.init_compile_yaml(yml,platform,target) + comb.combine_model() + comb.combine_platforms() + +def clean(combined): + if Path(combined).exists(): + os.remove(combined) + print(f"{combined} removed.") + +def list_platforms_subtool(yamlfile): + # Regsiter tag handler + yaml.add_constructor('!join', join_constructor) + + e = yamlfile.split("/")[-1].split(".")[0] #"exp_placeholder" + p = "p_placeholder" + t = "t_placeholder" + + combined=f"combined-am5.yaml" + # Combine model / experiment + quick_combine(yamlfile,e,p,t,combined) + + # Print experiment names + c = yaml_load(combined) + + print("\nPlatforms available:") + for i in c.get("platforms"): + print(f' - {i.get("name")}') + print("\n") + clean(combined) From 365e2c806825fb8098400044e2d2edc0f9440687 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 22 Jan 2025 10:43:06 -0500 Subject: [PATCH 02/20] #315 Change `list` to `list_` --- fre/fre.py | 2 +- fre/{list => list_}/__init__.py | 0 fre/{list => list_}/frelist.py | 10 +++++----- fre/{list => list_}/list_experiments_script.py | 6 +++--- fre/{list => list_}/list_platforms_script.py | 0 5 files changed, 9 insertions(+), 9 deletions(-) rename fre/{list => list_}/__init__.py (100%) rename fre/{list => list_}/frelist.py (82%) rename fre/{list => list_}/list_experiments_script.py (94%) rename fre/{list => list_}/list_platforms_script.py (100%) diff --git a/fre/fre.py b/fre/fre.py index 61ebb73c..ca5e086b 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -29,7 +29,7 @@ cls = LazyGroup, lazy_subcommands = {"pp": ".pp.frepp.pp_cli", "catalog": ".catalog.frecatalog.catalog_cli", - "list": ".list.frelist.list_cli", + "list": ".list_.frelist.list_cli", "check": ".check.frecheck.check_cli", "run": ".run.frerun.run_cli", "test": ".test.fretest.test_cli", diff --git a/fre/list/__init__.py b/fre/list_/__init__.py similarity index 100% rename from fre/list/__init__.py rename to fre/list_/__init__.py diff --git a/fre/list/frelist.py b/fre/list_/frelist.py similarity index 82% rename from fre/list/frelist.py rename to fre/list_/frelist.py index 03498cee..167ed47e 100644 --- a/fre/list/frelist.py +++ b/fre/list_/frelist.py @@ -1,8 +1,8 @@ -''' fre list ''' +''' fre lister ''' import click -from fre.list import list_experiments_script -from fre.list import list_platforms_script +from fre.list_ import list_experiments_script +from fre.list_ import list_platforms_script @click.group(help=click.style(" - access fre list subcommands", fg=(232,204,91))) def list_cli(): @@ -14,7 +14,7 @@ def list_cli(): type=str, help="YAML file to be used for parsing", required=True) -def list_exps(yamlfile): +def exps(yamlfile): """ - List experiments available""" list_experiments_script.list_experiments_subtool(yamlfile) @@ -24,7 +24,7 @@ def list_exps(yamlfile): type=str, help="YAML file to be used for parsing", required=True) -def list_platforms(yamlfile): +def platforms(yamlfile): """ - List platforms available """ list_platforms_script.list_platforms_subtool(yamlfile) diff --git a/fre/list/list_experiments_script.py b/fre/list_/list_experiments_script.py similarity index 94% rename from fre/list/list_experiments_script.py rename to fre/list_/list_experiments_script.py index d962fa3e..da13129a 100644 --- a/fre/list/list_experiments_script.py +++ b/fre/list_/list_experiments_script.py @@ -15,9 +15,9 @@ def join_constructor(loader, node): seq = loader.construct_sequence(node) return ''.join([str(i) for i in seq]) -class NoAliasDumper(yaml.SafeDumper): - def ignore_aliases(self, data): - return True +#class NoAliasDumper(yaml.SafeDumper): +# def ignore_aliases(self, data): +# return True def yaml_load(yamlfile): """ diff --git a/fre/list/list_platforms_script.py b/fre/list_/list_platforms_script.py similarity index 100% rename from fre/list/list_platforms_script.py rename to fre/list_/list_platforms_script.py From 0a5988239051a7793bcec9634f8adbe20a0a3f98 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 22 Jan 2025 11:34:03 -0500 Subject: [PATCH 03/20] #315 Add cli unit tests for `list` tool --- fre/tests/test_fre_list_cli.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/fre/tests/test_fre_list_cli.py b/fre/tests/test_fre_list_cli.py index 80c5af2b..6ef05760 100644 --- a/fre/tests/test_fre_list_cli.py +++ b/fre/tests/test_fre_list_cli.py @@ -20,3 +20,35 @@ def test_cli_fre_list_opt_dne(): ''' fre list optionDNE ''' result = runner.invoke(fre.fre, args=["list", "optionDNE"]) assert result.exit_code == 2 + +## fre list exps +def test_cli_fre_list_exps(): + ''' fre list exps ''' + result = runner.invoke(fre.fre, args=["list", "exps"]) + assert result.exit_code == 2 + +def test_cli_fre_list_exps_help(): + ''' fre list exps --help ''' + result = runner.invoke(fre.fre, args=["list", "exps", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_list_exps_opt_dne(): + ''' fre list exps optionDNE ''' + result = runner.invoke(fre.fre, args=["list", "exps", "optionDNE"]) + assert result.exit_code == 2 + +## fre list platforms +def test_cli_fre_list_platforms(): + ''' fre list platforms ''' + result = runner.invoke(fre.fre, args=["list", "platforms"]) + assert result.exit_code == 2 + +def test_cli_fre_list_exps_help(): + ''' fre list platforms --help ''' + result = runner.invoke(fre.fre, args=["list", "platforms", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_list_exps_opt_dne(): + ''' fre list platforms optionDNE ''' + result = runner.invoke(fre.fre, args=["list", "platforms", "optionDNE"]) + assert result.exit_code == 2 From 4d8c9182831a265aacf0d54210bb369d360cdab4 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 22 Jan 2025 12:03:32 -0500 Subject: [PATCH 04/20] #315 Add path to yaml file --- fre/list_/list_platforms_script.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index 180a29c1..e7092232 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -50,12 +50,14 @@ def list_platforms_subtool(yamlfile): p = "p_placeholder" t = "t_placeholder" - combined=f"combined-am5.yaml" + combined=f"combined-{e}.yaml" + yamlpath = os.path.dirname(yamlfile) + # Combine model / experiment quick_combine(yamlfile,e,p,t,combined) # Print experiment names - c = yaml_load(combined) + c = yaml_load(os.path.join(yamlpath,combined)) print("\nPlatforms available:") for i in c.get("platforms"): From 3e7081e61424469ee2582becb089dae37ca8549b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 22 Jan 2025 12:04:22 -0500 Subject: [PATCH 05/20] #315 Add tests for listing experiments and platforms --- .../null_example/combined-null_model_bad.yaml | 4 ++ fre/list_/tests/null_example/compile.yaml | 48 ++++++++++++++++ fre/list_/tests/null_example/null_model.yaml | 57 +++++++++++++++++++ fre/list_/tests/null_example/platforms.yaml | 20 +++++++ .../tests/test_list_experiments_script.py | 35 ++++++++++++ fre/list_/tests/test_list_platforms_script.py | 40 +++++++++++++ 6 files changed, 204 insertions(+) create mode 100644 fre/list_/tests/null_example/combined-null_model_bad.yaml create mode 100644 fre/list_/tests/null_example/compile.yaml create mode 100644 fre/list_/tests/null_example/null_model.yaml create mode 100644 fre/list_/tests/null_example/platforms.yaml create mode 100644 fre/list_/tests/test_list_experiments_script.py create mode 100644 fre/list_/tests/test_list_platforms_script.py diff --git a/fre/list_/tests/null_example/combined-null_model_bad.yaml b/fre/list_/tests/null_example/combined-null_model_bad.yaml new file mode 100644 index 00000000..58d14512 --- /dev/null +++ b/fre/list_/tests/null_example/combined-null_model_bad.yaml @@ -0,0 +1,4 @@ +name: &name "fre/list_/tests/null_example/null_model_bad" +platform: &platform "p_placeholder" +target: &target "t_placeholder" + diff --git a/fre/list_/tests/null_example/compile.yaml b/fre/list_/tests/null_example/compile.yaml new file mode 100644 index 00000000..68c151f8 --- /dev/null +++ b/fre/list_/tests/null_example/compile.yaml @@ -0,0 +1,48 @@ +compile: + experiment: "null_model_full" + container_addlibs: + baremetal_linkerflags: + src: + - component: "FMS" + repo: "https://github.com/NOAA-GFDL/FMS.git" + cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML -DHAVE_GETTID" + otherFlags: "-fallow-argument-mismatch" # only needed for gcc + branch: *branch + - component: "atmos_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/atmos_null.git" + branch: *branch + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: *FMSincludes + - component: "land_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/land_null.git" + branch: *branch + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: *FMSincludes + - component: "ice_param" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ice_param.git" + branch: *branch + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: *FMSincludes + - component: "ocean_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ocean_null.git" + branch: *branch + otherFlags: *FMSincludes + - component: "ice_null" + requires: ["FMS", "ice_param", "ocean_null"] + repo: "https://github.com/NOAA-GFDL/ice_null.git" + branch: *branch + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: *FMSincludes + - component: "coupler" + requires: ["FMS", "ocean_null", "atmos_null", "land_null", "ice_param", "ocean_null", "ice_null"] + repo: "https://github.com/NOAA-GFDL/FMScoupler.git" + branch: *branch + otherFlags: *FMSincludes + cppdefs: "-D_USE_LEGACY_LAND_ -Duse_AM3_physics -DINTERNAL_FILE_NML" + paths: [ "coupler/full", + "coupler/shared" ] + diff --git a/fre/list_/tests/null_example/null_model.yaml b/fre/list_/tests/null_example/null_model.yaml new file mode 100644 index 00000000..9c473735 --- /dev/null +++ b/fre/list_/tests/null_example/null_model.yaml @@ -0,0 +1,57 @@ +# reusable variables +fre_properties: + - &VERSION "null" + - &FRE_STEM !join [null_model/, *VERSION] + + # amip + - &EXP_AMIP_START "19790101T0000Z" + - &EXP_AMIP_END "20200101T0000Z" + - &ANA_AMIP_START "19800101T0000Z" + - &ANA_AMIP_END "20200101T0000Z" + + - &PP_AMIP_CHUNK96 "P1Y" + - &PP_AMIP_CHUNK384 "P1Y" + - &PP_XYINTERP96 "180,288" + - &PP_XYINTERP384 "720,1152" + + # compile information + - &release "f1a1r1" + - &INTEL "intel-classic" + - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +# compile information +build: + compileYaml: "compile.yaml" + platformYaml: "platforms.yaml" + +shared: + # directories shared across tools + directories: &shared_directories + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + + # shared pp settings + postprocess: + settings: &shared_settings + site: "ppan" + switches: &shared_switches + do_statics: True + do_timeavgs: True + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + +experiments: + - name: "null_model_0" + pp: + - name: "null_model_1" + pp: + - name: "null_model_2" + pp: + - name: "null_model_3" + pp: diff --git a/fre/list_/tests/null_example/platforms.yaml b/fre/list_/tests/null_example/platforms.yaml new file mode 100644 index 00000000..e7d9e5ae --- /dev/null +++ b/fre/list_/tests/null_example/platforms.yaml @@ -0,0 +1,20 @@ +platforms: + - name: ncrc5.intel23 + compiler: intel + modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] + modules: [!join [*INTEL, "/2023.2.0"],"fre/bronx-21",cray-hdf5/1.12.2.11, cray-netcdf/4.9.0.11] + mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] + modelRoot: ${HOME}/fremake_canopy/test + - name: hpcme.2023 + compiler: intel + RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"] + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "ecpe4s/noaa-intel-prototype:2023.09.25" + mkTemplate: "/apps/mkmf/templates/hpcme-intel21.mk" + - name: ci.gnu + compiler: gnu + mkTemplate: /__w/fre-cli/fre-cli/mkmf/templates/linux-ubuntu-xenial-gnu.mk + modelRoot: ${TEST_BUILD_DIR}/fremake_canopy/test diff --git a/fre/list_/tests/test_list_experiments_script.py b/fre/list_/tests/test_list_experiments_script.py new file mode 100644 index 00000000..c182c80c --- /dev/null +++ b/fre/list_/tests/test_list_experiments_script.py @@ -0,0 +1,35 @@ +""" +Test fre list exps +""" +#import os +#import shutil +import pytest +from pathlib import Path +from fre.list_ import list_experiments_script + +# SET-UP +TEST_DIR = Path("fre/list_/tests") +NM_EXAMPLE = Path("null_example") +YAMLFILE = "null_model.yaml" +BADYAMLFILE = "null_model_bad.yaml" + +# yaml file checks +def test_modelyaml_exists(): + ''' Make sure model yaml exists ''' + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() + +def test_exp_list(capfd): + ''' test list exps ''' + list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + + #Capture output + out,err=capfd.readouterr() + if "Post-processing experiments available" in out: + assert True + else: + assert False + +@pytest.mark.xfail() +def test_exps_list_badyaml(): + ''' test failure of list exps tool given a bad yaml file path ''' + list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{BADYAMLFILE}") diff --git a/fre/list_/tests/test_list_platforms_script.py b/fre/list_/tests/test_list_platforms_script.py new file mode 100644 index 00000000..e8719eda --- /dev/null +++ b/fre/list_/tests/test_list_platforms_script.py @@ -0,0 +1,40 @@ +""" +Test fre list exps +""" +#import os +#import shutil +import pytest +from pathlib import Path +from fre.list_ import list_platforms_script + +# SET-UP +TEST_DIR = Path("fre/list_/tests") +NM_EXAMPLE = Path("null_example") +YAMLFILE = "null_model.yaml" +BADYAMLFILE = "null_model_bad.yaml" + +# yaml file checks +def test_modelyaml_exists(): + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() + +def test_compileyaml_exists(): + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/compile.yaml").exists() + +def test_platformyaml_exists(): + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/platforms.yaml").exists() + +def test_exp_list(capfd): + ''' test list exps ''' + list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + + #Capture output + out,err=capfd.readouterr() + if "Platforms available" in out: + assert True + else: + assert False + +@pytest.mark.xfail() +def test_exps_list_badyaml(): + ''' test failure of list exps tool given a bad yaml file path ''' + list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{BADYAMLFILE}") From 0b6f808041bd99a8e52689da2e3a5c8ec9921318 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 23 Jan 2025 16:25:34 -0500 Subject: [PATCH 06/20] #315 Address some pylint messages --- fre/list_/list_experiments_script.py | 2 +- fre/list_/list_platforms_script.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/fre/list_/list_experiments_script.py b/fre/list_/list_experiments_script.py index da13129a..dcb80672 100644 --- a/fre/list_/list_experiments_script.py +++ b/fre/list_/list_experiments_script.py @@ -3,7 +3,6 @@ """ import os from pathlib import Path -import click import yaml import fre.yamltools.combine_yamls as cy @@ -31,6 +30,7 @@ def yaml_load(yamlfile): def quick_combine(yml, exp, platform, target): """ Create intermediate combined model and exp. yaml + This is done to avoid an "undefined alias" error """ # Combine model / experiment comb = cy.init_pp_yaml(yml,exp,platform,target) diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index e7092232..c296a6c6 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -3,10 +3,9 @@ """ import os -import click +from pathlib import Path import yaml import fre.yamltools.combine_yamls as cy -from pathlib import Path def join_constructor(loader, node): """ @@ -29,8 +28,10 @@ def yaml_load(yamlfile): return y -def quick_combine(yml, exp, platform, target,combined): +def quick_combine(yml, platform, target): """ + Combine the intermediate model and platforms yaml. + This is done to avoid an "undefined alias" error """ # Combine model / experiment comb = cy.init_compile_yaml(yml,platform,target) @@ -38,11 +39,17 @@ def quick_combine(yml, exp, platform, target,combined): comb.combine_platforms() def clean(combined): + """ + Remove intermediate combined yaml. + """ if Path(combined).exists(): os.remove(combined) print(f"{combined} removed.") def list_platforms_subtool(yamlfile): + """ + List the platforms available + """ # Regsiter tag handler yaml.add_constructor('!join', join_constructor) @@ -54,7 +61,7 @@ def list_platforms_subtool(yamlfile): yamlpath = os.path.dirname(yamlfile) # Combine model / experiment - quick_combine(yamlfile,e,p,t,combined) + quick_combine(yamlfile,p,t) # Print experiment names c = yaml_load(os.path.join(yamlpath,combined)) From b60de68e156d8de5e56944eaa638f88145dd5596 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 23 Jan 2025 16:30:21 -0500 Subject: [PATCH 07/20] #315 Did not mean to add this combined yaml --- fre/list_/tests/null_example/combined-null_model_bad.yaml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 fre/list_/tests/null_example/combined-null_model_bad.yaml diff --git a/fre/list_/tests/null_example/combined-null_model_bad.yaml b/fre/list_/tests/null_example/combined-null_model_bad.yaml deleted file mode 100644 index 58d14512..00000000 --- a/fre/list_/tests/null_example/combined-null_model_bad.yaml +++ /dev/null @@ -1,4 +0,0 @@ -name: &name "fre/list_/tests/null_example/null_model_bad" -platform: &platform "p_placeholder" -target: &target "t_placeholder" - From 084463c36d75604809b8d604a7c4f48a25bcab7f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 23 Jan 2025 16:51:53 -0500 Subject: [PATCH 08/20] #315 Change `list` to `list_` --- meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index 54ac50fe..8413d728 100644 --- a/meta.yaml +++ b/meta.yaml @@ -58,7 +58,7 @@ test: - fre.catalog - fre.check - fre.cmor - - fre.list + - fre.list_ - fre.make - fre.pp - fre.run From c5d33501a41c301f37d83e87a70e2fa5514f4465 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 23 Jan 2025 17:06:27 -0500 Subject: [PATCH 09/20] #315 Add some documentation for `fre list` tools --- docs/tools.rst | 6 ++++++ docs/tools/listtools.rst | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 docs/tools/listtools.rst diff --git a/docs/tools.rst b/docs/tools.rst index c658b211..7b564eca 100644 --- a/docs/tools.rst +++ b/docs/tools.rst @@ -63,3 +63,9 @@ fre yamltools ================= .. include:: tools/yamltools.rst + + +fre list +================= + +.. include:: tools/listtools.rst diff --git a/docs/tools/listtools.rst b/docs/tools/listtools.rst new file mode 100644 index 00000000..e4747234 --- /dev/null +++ b/docs/tools/listtools.rst @@ -0,0 +1,16 @@ +``exps`` +----------------- + +``fre list exps [options]`` + - Purpose: Lists available post-processing experiments included in the yaml configurations + - Options: + - `-y, --yamlfile [experiment yaml]` + +``platforms`` +----------------- + +``fre list platforms [options]`` + - Purpose: Lists available platforms included in the yaml configurations + - Options: + - `-y, --yamlfile [experiment yaml]` + From efc16326504b1c06adefbdb58b153b76fdd828a3 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 27 Jan 2025 09:49:12 -0500 Subject: [PATCH 10/20] #315 Comment thoughts --- fre/list_/list_experiments_script.py | 2 ++ fre/list_/list_platforms_script.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/fre/list_/list_experiments_script.py b/fre/list_/list_experiments_script.py index dcb80672..7e27a0ba 100644 --- a/fre/list_/list_experiments_script.py +++ b/fre/list_/list_experiments_script.py @@ -14,6 +14,8 @@ def join_constructor(loader, node): seq = loader.construct_sequence(node) return ''.join([str(i) for i in seq]) +# To look into: ignore undefined alias error msg for listing? +# Found this somewhere but don't fully understand yet #class NoAliasDumper(yaml.SafeDumper): # def ignore_aliases(self, data): # return True diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index c296a6c6..d8352c58 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -15,6 +15,8 @@ def join_constructor(loader, node): seq = loader.construct_sequence(node) return ''.join([str(i) for i in seq]) +# To look into: ignore undefined alias error msg for listing? +# Found this somewhere but don't fully understand yet #class NoAliasDumper(yaml.SafeDumper): # def ignore_aliases(self, data): # return True From 4a1eaa6eb10481864816cab48a02e7cee346bdc6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 27 Jan 2025 09:59:11 -0500 Subject: [PATCH 11/20] #315 Clean comments --- fre/list_/tests/test_list_experiments_script.py | 2 -- fre/list_/tests/test_list_platforms_script.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/fre/list_/tests/test_list_experiments_script.py b/fre/list_/tests/test_list_experiments_script.py index c182c80c..0768bec9 100644 --- a/fre/list_/tests/test_list_experiments_script.py +++ b/fre/list_/tests/test_list_experiments_script.py @@ -1,8 +1,6 @@ """ Test fre list exps """ -#import os -#import shutil import pytest from pathlib import Path from fre.list_ import list_experiments_script diff --git a/fre/list_/tests/test_list_platforms_script.py b/fre/list_/tests/test_list_platforms_script.py index e8719eda..e2854227 100644 --- a/fre/list_/tests/test_list_platforms_script.py +++ b/fre/list_/tests/test_list_platforms_script.py @@ -1,8 +1,6 @@ """ Test fre list exps """ -#import os -#import shutil import pytest from pathlib import Path from fre.list_ import list_platforms_script From f0ce1c6e8da0429061922ae1a1a73420b21a00c5 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 29 Jan 2025 14:10:00 -0500 Subject: [PATCH 12/20] #315 Change `e`, `p`, `t` options to None - `e` for list_platforms needs to remain how it is for now --- fre/list_/list_experiments_script.py | 6 +++--- fre/list_/list_platforms_script.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fre/list_/list_experiments_script.py b/fre/list_/list_experiments_script.py index 7e27a0ba..35b08069 100644 --- a/fre/list_/list_experiments_script.py +++ b/fre/list_/list_experiments_script.py @@ -53,9 +53,9 @@ def list_experiments_subtool(yamlfile): # Regsiter tag handler yaml.add_constructor('!join', join_constructor) - e = "exp_placeholder" - p = "p_placeholder" - t = "t_placeholder" + e = None + p = None + t = None combined=f"combined-{e}.yaml" # Combine model / experiment diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index d8352c58..f5a91321 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -55,9 +55,9 @@ def list_platforms_subtool(yamlfile): # Regsiter tag handler yaml.add_constructor('!join', join_constructor) - e = yamlfile.split("/")[-1].split(".")[0] #"exp_placeholder" - p = "p_placeholder" - t = "t_placeholder" + e = yamlfile.split("/")[-1].split(".")[0] + p = None + t = None combined=f"combined-{e}.yaml" yamlpath = os.path.dirname(yamlfile) From 17206663a001c19c550047fae1742a57d256dfab Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 29 Jan 2025 14:48:11 -0500 Subject: [PATCH 13/20] #315 Leverage null_model yaml files already created --- fre/list_/tests/null_example/compile.yaml | 48 ---------------- fre/list_/tests/null_example/null_model.yaml | 57 ------------------- fre/list_/tests/null_example/platforms.yaml | 20 ------- .../tests/test_list_experiments_script.py | 2 +- fre/list_/tests/test_list_platforms_script.py | 2 +- fre/make/tests/null_example/null_model.yaml | 22 +++++++ 6 files changed, 24 insertions(+), 127 deletions(-) delete mode 100644 fre/list_/tests/null_example/compile.yaml delete mode 100644 fre/list_/tests/null_example/null_model.yaml delete mode 100644 fre/list_/tests/null_example/platforms.yaml diff --git a/fre/list_/tests/null_example/compile.yaml b/fre/list_/tests/null_example/compile.yaml deleted file mode 100644 index 68c151f8..00000000 --- a/fre/list_/tests/null_example/compile.yaml +++ /dev/null @@ -1,48 +0,0 @@ -compile: - experiment: "null_model_full" - container_addlibs: - baremetal_linkerflags: - src: - - component: "FMS" - repo: "https://github.com/NOAA-GFDL/FMS.git" - cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML -DHAVE_GETTID" - otherFlags: "-fallow-argument-mismatch" # only needed for gcc - branch: *branch - - component: "atmos_null" - requires: ["FMS"] - repo: "https://github.com/NOAA-GFDL/atmos_null.git" - branch: *branch - cppdefs: "-DINTERNAL_FILE_NML" - otherFlags: *FMSincludes - - component: "land_null" - requires: ["FMS"] - repo: "https://github.com/NOAA-GFDL/land_null.git" - branch: *branch - cppdefs: "-DINTERNAL_FILE_NML" - otherFlags: *FMSincludes - - component: "ice_param" - requires: ["FMS"] - repo: "https://github.com/NOAA-GFDL/ice_param.git" - branch: *branch - cppdefs: "-DINTERNAL_FILE_NML" - otherFlags: *FMSincludes - - component: "ocean_null" - requires: ["FMS"] - repo: "https://github.com/NOAA-GFDL/ocean_null.git" - branch: *branch - otherFlags: *FMSincludes - - component: "ice_null" - requires: ["FMS", "ice_param", "ocean_null"] - repo: "https://github.com/NOAA-GFDL/ice_null.git" - branch: *branch - cppdefs: "-DINTERNAL_FILE_NML" - otherFlags: *FMSincludes - - component: "coupler" - requires: ["FMS", "ocean_null", "atmos_null", "land_null", "ice_param", "ocean_null", "ice_null"] - repo: "https://github.com/NOAA-GFDL/FMScoupler.git" - branch: *branch - otherFlags: *FMSincludes - cppdefs: "-D_USE_LEGACY_LAND_ -Duse_AM3_physics -DINTERNAL_FILE_NML" - paths: [ "coupler/full", - "coupler/shared" ] - diff --git a/fre/list_/tests/null_example/null_model.yaml b/fre/list_/tests/null_example/null_model.yaml deleted file mode 100644 index 9c473735..00000000 --- a/fre/list_/tests/null_example/null_model.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# reusable variables -fre_properties: - - &VERSION "null" - - &FRE_STEM !join [null_model/, *VERSION] - - # amip - - &EXP_AMIP_START "19790101T0000Z" - - &EXP_AMIP_END "20200101T0000Z" - - &ANA_AMIP_START "19800101T0000Z" - - &ANA_AMIP_END "20200101T0000Z" - - - &PP_AMIP_CHUNK96 "P1Y" - - &PP_AMIP_CHUNK384 "P1Y" - - &PP_XYINTERP96 "180,288" - - &PP_XYINTERP384 "720,1152" - - # compile information - - &release "f1a1r1" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" - -# compile information -build: - compileYaml: "compile.yaml" - platformYaml: "platforms.yaml" - -shared: - # directories shared across tools - directories: &shared_directories - history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] - ptmp_dir: "/xtmp/$USER/ptmp" - - # shared pp settings - postprocess: - settings: &shared_settings - site: "ppan" - switches: &shared_switches - do_statics: True - do_timeavgs: True - clean_work: True - do_refinediag: False - do_atmos_plevel_masking: True - do_preanalysis: False - do_analysis: True - -experiments: - - name: "null_model_0" - pp: - - name: "null_model_1" - pp: - - name: "null_model_2" - pp: - - name: "null_model_3" - pp: diff --git a/fre/list_/tests/null_example/platforms.yaml b/fre/list_/tests/null_example/platforms.yaml deleted file mode 100644 index e7d9e5ae..00000000 --- a/fre/list_/tests/null_example/platforms.yaml +++ /dev/null @@ -1,20 +0,0 @@ -platforms: - - name: ncrc5.intel23 - compiler: intel - modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [!join [*INTEL, "/2023.2.0"],"fre/bronx-21",cray-hdf5/1.12.2.11, cray-netcdf/4.9.0.11] - mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] - modelRoot: ${HOME}/fremake_canopy/test - - name: hpcme.2023 - compiler: intel - RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"] - modelRoot: /apps - container: True - containerBuild: "podman" - containerRun: "apptainer" - containerBase: "ecpe4s/noaa-intel-prototype:2023.09.25" - mkTemplate: "/apps/mkmf/templates/hpcme-intel21.mk" - - name: ci.gnu - compiler: gnu - mkTemplate: /__w/fre-cli/fre-cli/mkmf/templates/linux-ubuntu-xenial-gnu.mk - modelRoot: ${TEST_BUILD_DIR}/fremake_canopy/test diff --git a/fre/list_/tests/test_list_experiments_script.py b/fre/list_/tests/test_list_experiments_script.py index 0768bec9..46e413c4 100644 --- a/fre/list_/tests/test_list_experiments_script.py +++ b/fre/list_/tests/test_list_experiments_script.py @@ -6,7 +6,7 @@ from fre.list_ import list_experiments_script # SET-UP -TEST_DIR = Path("fre/list_/tests") +TEST_DIR = Path("fre/make/tests") NM_EXAMPLE = Path("null_example") YAMLFILE = "null_model.yaml" BADYAMLFILE = "null_model_bad.yaml" diff --git a/fre/list_/tests/test_list_platforms_script.py b/fre/list_/tests/test_list_platforms_script.py index e2854227..b7addf63 100644 --- a/fre/list_/tests/test_list_platforms_script.py +++ b/fre/list_/tests/test_list_platforms_script.py @@ -6,7 +6,7 @@ from fre.list_ import list_platforms_script # SET-UP -TEST_DIR = Path("fre/list_/tests") +TEST_DIR = Path("fre/make/tests") NM_EXAMPLE = Path("null_example") YAMLFILE = "null_model.yaml" BADYAMLFILE = "null_model_bad.yaml" diff --git a/fre/make/tests/null_example/null_model.yaml b/fre/make/tests/null_example/null_model.yaml index 2dcbcad4..e5c0f120 100644 --- a/fre/make/tests/null_example/null_model.yaml +++ b/fre/make/tests/null_example/null_model.yaml @@ -11,5 +11,27 @@ build: compileYaml: "compile.yaml" platformYaml: "platforms.yaml" +shared: # directories shared across tools + directories: &shared_directories + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + + # shared pp settings + postprocess: + settings: &shared_settings + site: "ppan" + switches: &shared_switches + clean_work: True + do_refinediag: False + do_analysis: True + experiments: - name: "null_model_full" + pp: + - name: "null_model_0" + pp: + - name: "null_model_1" + pp: + - name: "null_model_2" + pp: From 1747cd9edc77c6852cfe37c7051c7a811e1c2fb3 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 29 Jan 2025 15:00:21 -0500 Subject: [PATCH 14/20] #315 Use functions that have already been created - remove repetitive functions - use `pathlib` functions vs `os` --- fre/list_/list_experiments_script.py | 32 +++++++------------------- fre/list_/list_platforms_script.py | 34 ++++++++-------------------- 2 files changed, 17 insertions(+), 49 deletions(-) diff --git a/fre/list_/list_experiments_script.py b/fre/list_/list_experiments_script.py index 35b08069..b2b4512c 100644 --- a/fre/list_/list_experiments_script.py +++ b/fre/list_/list_experiments_script.py @@ -1,34 +1,16 @@ """ Script combines the model yaml with exp, platform, and target to list experiment information. """ -import os from pathlib import Path import yaml import fre.yamltools.combine_yamls as cy -def join_constructor(loader, node): - """ - Allows FRE properties defined - in main yaml to be concatenated. - """ - seq = loader.construct_sequence(node) - return ''.join([str(i) for i in seq]) - # To look into: ignore undefined alias error msg for listing? -# Found this somewhere but don't fully understand yet +# Found this somewhere but don't fully understand yet #class NoAliasDumper(yaml.SafeDumper): # def ignore_aliases(self, data): # return True -def yaml_load(yamlfile): - """ - Load the yamlfile - """ - with open(yamlfile, 'r') as yf: - y = yaml.load(yf,Loader=yaml.Loader) - - return y - def quick_combine(yml, exp, platform, target): """ Create intermediate combined model and exp. yaml @@ -38,12 +20,12 @@ def quick_combine(yml, exp, platform, target): comb = cy.init_pp_yaml(yml,exp,platform,target) comb.combine_model() -def clean(combined): +def remove(combined): """ Remove intermediate combined yaml. """ if Path(combined).exists(): - os.remove(combined) + Path(combined).unlink() print(f"{combined} removed.") def list_experiments_subtool(yamlfile): @@ -51,7 +33,7 @@ def list_experiments_subtool(yamlfile): List the post-processing experiments available """ # Regsiter tag handler - yaml.add_constructor('!join', join_constructor) + yaml.add_constructor('!join', cy.join_constructor) e = None p = None @@ -62,10 +44,12 @@ def list_experiments_subtool(yamlfile): quick_combine(yamlfile,e,p,t) # Print experiment names - c = yaml_load(combined) + c = cy.yaml_load(combined) print("\nPost-processing experiments available:") for i in c.get("experiments"): print(f' - {i.get("name")}') print("\n") - clean(combined) + + # Clean intermediate combined yaml + remove(combined) diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index f5a91321..63f5ccbd 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -2,34 +2,16 @@ Script combines the model yaml with exp, platform, and target to list experiment information. """ -import os from pathlib import Path import yaml import fre.yamltools.combine_yamls as cy -def join_constructor(loader, node): - """ - Allows FRE properties defined - in main yaml to be concatenated. - """ - seq = loader.construct_sequence(node) - return ''.join([str(i) for i in seq]) - # To look into: ignore undefined alias error msg for listing? # Found this somewhere but don't fully understand yet #class NoAliasDumper(yaml.SafeDumper): # def ignore_aliases(self, data): # return True -def yaml_load(yamlfile): - """ - Load the yamlfile - """ - with open(yamlfile, 'r') as yf: - y = yaml.load(yf,Loader=yaml.Loader) - - return y - def quick_combine(yml, platform, target): """ Combine the intermediate model and platforms yaml. @@ -40,12 +22,12 @@ def quick_combine(yml, platform, target): comb.combine_model() comb.combine_platforms() -def clean(combined): +def remove(combined): """ Remove intermediate combined yaml. """ if Path(combined).exists(): - os.remove(combined) + Path(combined).unlink() print(f"{combined} removed.") def list_platforms_subtool(yamlfile): @@ -53,23 +35,25 @@ def list_platforms_subtool(yamlfile): List the platforms available """ # Regsiter tag handler - yaml.add_constructor('!join', join_constructor) + yaml.add_constructor('!join', cy.join_constructor) - e = yamlfile.split("/")[-1].split(".")[0] + e = yamlfile.split("/")[-1].split(".")[0] p = None t = None combined=f"combined-{e}.yaml" - yamlpath = os.path.dirname(yamlfile) + yamlpath = Path(yamlfile).parent # Combine model / experiment quick_combine(yamlfile,p,t) # Print experiment names - c = yaml_load(os.path.join(yamlpath,combined)) + c = cy.yaml_load(f"{yamlpath}/{combined}") print("\nPlatforms available:") for i in c.get("platforms"): print(f' - {i.get("name")}') print("\n") - clean(combined) + + # Clean the intermediate combined yaml + remove(combined) From a1e8c93417dad807b6b3c0ccc21b363f3cf7946a Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 30 Jan 2025 15:47:11 -0500 Subject: [PATCH 15/20] #315 Add error check --- fre/list_/list_experiments_script.py | 14 +++++++++----- fre/list_/list_platforms_script.py | 9 ++++++--- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/fre/list_/list_experiments_script.py b/fre/list_/list_experiments_script.py index b2b4512c..cda7baa0 100644 --- a/fre/list_/list_experiments_script.py +++ b/fre/list_/list_experiments_script.py @@ -26,7 +26,10 @@ def remove(combined): """ if Path(combined).exists(): Path(combined).unlink() - print(f"{combined} removed.") + print("Remove intermediate combined yaml:\n", + f" {combined} removed.") + else: + raise ValueError(f"{combined} could not be found to remove.") def list_experiments_subtool(yamlfile): """ @@ -35,11 +38,12 @@ def list_experiments_subtool(yamlfile): # Regsiter tag handler yaml.add_constructor('!join', cy.join_constructor) - e = None - p = None - t = None + e = "None" + p = "None" + t = "None" + + combined = f"combined-{e}.yaml" - combined=f"combined-{e}.yaml" # Combine model / experiment quick_combine(yamlfile,e,p,t) diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index 63f5ccbd..ae1a4312 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -28,7 +28,10 @@ def remove(combined): """ if Path(combined).exists(): Path(combined).unlink() - print(f"{combined} removed.") + print("Remove intermediate combined yaml:\n", + f" {combined} removed.") + else: + raise ValueError(f"{combined} could not be found to remove.") def list_platforms_subtool(yamlfile): """ @@ -41,7 +44,7 @@ def list_platforms_subtool(yamlfile): p = None t = None - combined=f"combined-{e}.yaml" + combined = f"combined-{e}.yaml" yamlpath = Path(yamlfile).parent # Combine model / experiment @@ -56,4 +59,4 @@ def list_platforms_subtool(yamlfile): print("\n") # Clean the intermediate combined yaml - remove(combined) + #remove(f"{yamlpath}/{combined}") From 188bffe1c95075edfcdb52a995a769bc5cbad2c7 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 30 Jan 2025 15:47:41 -0500 Subject: [PATCH 16/20] #315 Update/add tests --- .../tests/test_list_experiments_script.py | 53 ++++++++++++++++-- fre/list_/tests/test_list_platforms_script.py | 56 +++++++++++++++++-- 2 files changed, 98 insertions(+), 11 deletions(-) diff --git a/fre/list_/tests/test_list_experiments_script.py b/fre/list_/tests/test_list_experiments_script.py index 46e413c4..96da5c48 100644 --- a/fre/list_/tests/test_list_experiments_script.py +++ b/fre/list_/tests/test_list_experiments_script.py @@ -3,13 +3,14 @@ """ import pytest from pathlib import Path +import yaml from fre.list_ import list_experiments_script # SET-UP TEST_DIR = Path("fre/make/tests") NM_EXAMPLE = Path("null_example") YAMLFILE = "null_model.yaml" -BADYAMLFILE = "null_model_bad.yaml" +EXP_NAME = "None" # yaml file checks def test_modelyaml_exists(): @@ -25,9 +26,49 @@ def test_exp_list(capfd): if "Post-processing experiments available" in out: assert True else: - assert False + assert False -@pytest.mark.xfail() -def test_exps_list_badyaml(): - ''' test failure of list exps tool given a bad yaml file path ''' - list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{BADYAMLFILE}") +def test_int_combine(capfd): + ''' test intermediate combine step is happening ''' + list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + + #Capture output + out,err=capfd.readouterr() + check_out = ["Combining yaml files", "model yaml"] + for i in check_out: + if i in out: + assert True + else: + assert False + +def test_nocombinedyaml(): + ''' test intermediate combined yaml was cleaned at end of listing ''' + assert Path(f"./combined-{EXP_NAME}.yaml").exists() == False + +# Test individual functions operating correctly: combine and clean +def test_correct_combine(): + ''' test that combined yaml includes necessary keys ''' + p = "None" + t = "None" + yamlfilepath = Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + + # Combine model / experiment + list_experiments_script.quick_combine(yamlfilepath,EXP_NAME,p,t) + assert Path(f"./combined-{EXP_NAME}.yaml").exists() + + with open(f"combined-{EXP_NAME}.yaml", 'r') as yf: + y = yaml.load(yf,Loader=yaml.Loader) + + req_keys = ["name","platform","target","fre_properties","experiments"] + for k in req_keys: + if k in y.keys(): + assert True + else: + assert False + +def test_yamlremove(): + ''' test intermediate combined yaml removed ''' + # Remove combined yaml file + list_experiments_script.remove(f"combined-{EXP_NAME}.yaml") + + assert Path(f"./combined-{EXP_NAME}.yaml").exists() == False diff --git a/fre/list_/tests/test_list_platforms_script.py b/fre/list_/tests/test_list_platforms_script.py index b7addf63..0ed57671 100644 --- a/fre/list_/tests/test_list_platforms_script.py +++ b/fre/list_/tests/test_list_platforms_script.py @@ -3,6 +3,7 @@ """ import pytest from pathlib import Path +import yaml from fre.list_ import list_platforms_script # SET-UP @@ -10,18 +11,22 @@ NM_EXAMPLE = Path("null_example") YAMLFILE = "null_model.yaml" BADYAMLFILE = "null_model_bad.yaml" +EXP_NAME = YAMLFILE.split(".")[0] # yaml file checks def test_modelyaml_exists(): + '''test if model yaml exists''' assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() def test_compileyaml_exists(): + '''test if compile yaml exists''' assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/compile.yaml").exists() def test_platformyaml_exists(): + '''test if platforms yaml exists''' assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/platforms.yaml").exists() -def test_exp_list(capfd): +def test_platforms_list(capfd): ''' test list exps ''' list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") @@ -32,7 +37,48 @@ def test_exp_list(capfd): else: assert False -@pytest.mark.xfail() -def test_exps_list_badyaml(): - ''' test failure of list exps tool given a bad yaml file path ''' - list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{BADYAMLFILE}") +def test_int_combine(capfd): + ''' test intermediate combine is happening ''' + list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + + #Capture output + out,err=capfd.readouterr() + check_out = ["Combining yaml files", "model yaml", "platforms yaml"] + for i in check_out: + if i in out: + assert True + else: + assert False + +def test_nocombinedyaml(): + ''' test intermediate combined yaml was cleaned ''' + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/combined-{EXP_NAME}.yaml").exists() == False + +# Test individual functions operating correctly: combine and clean +def test_correct_combine(): + ''' test that combined yaml includes necesary keys ''' + p = None + t = None + yamlfile = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + # Combine model / experiment + list_platforms_script.quick_combine(yamlfile,p,t) + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/combined-{EXP_NAME}.yaml").exists() + + comb_yamlfile = f"{TEST_DIR}/{NM_EXAMPLE}/combined-{EXP_NAME}.yaml" + with open(comb_yamlfile, 'r') as yf: + y = yaml.load(yf,Loader=yaml.Loader) + + req_keys = ["name","platform","target","fre_properties","platforms"] + for k in req_keys: + if k in y.keys(): + assert True + else: + assert False + +def test_yamlremove(): + ''' test intermediate combined yaml removed ''' + # Remove combined yaml file + list_platforms_script.remove(f"{TEST_DIR}/{NM_EXAMPLE}/combined-{EXP_NAME}.yaml") + + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/combined-{EXP_NAME}.yaml").exists() == False From cd1c4147e1148c2dac93d41dea4243103c17d9d3 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 30 Jan 2025 15:58:30 -0500 Subject: [PATCH 17/20] #315 Uncomment remove function --- fre/list_/list_platforms_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index ae1a4312..c04810a2 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -59,4 +59,4 @@ def list_platforms_subtool(yamlfile): print("\n") # Clean the intermediate combined yaml - #remove(f"{yamlpath}/{combined}") + remove(f"{yamlpath}/{combined}") From 961f96fb55b078cc1273682b805a6c78805491c2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 30 Jan 2025 16:00:12 -0500 Subject: [PATCH 18/20] #315 Update list_platforms tool --- fre/list_/list_platforms_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index c04810a2..dbc17b3e 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -41,8 +41,8 @@ def list_platforms_subtool(yamlfile): yaml.add_constructor('!join', cy.join_constructor) e = yamlfile.split("/")[-1].split(".")[0] - p = None - t = None + p = "None" + t = "None" combined = f"combined-{e}.yaml" yamlpath = Path(yamlfile).parent From ba11f2974ec8e0da837d073713639065c6a97002 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 4 Feb 2025 16:39:53 -0500 Subject: [PATCH 19/20] #315 Add combined yaml validation for list_platforms --- fre/list_/list_platforms_script.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index dbc17b3e..286ee0b2 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -4,6 +4,8 @@ from pathlib import Path import yaml +import json +from jsonschema import validate, ValidationError, SchemaError import fre.yamltools.combine_yamls as cy # To look into: ignore undefined alias error msg for listing? @@ -21,6 +23,7 @@ def quick_combine(yml, platform, target): comb = cy.init_compile_yaml(yml,platform,target) comb.combine_model() comb.combine_platforms() + comb.clean_yaml() def remove(combined): """ @@ -33,6 +36,23 @@ def remove(combined): else: raise ValueError(f"{combined} could not be found to remove.") +def validate_yaml(loaded_yaml): + """ + Validate the intermediate combined yaml + """ + # Validate combined yaml + frelist_dir = Path(__file__).resolve().parents[2] + schema_path = f"{frelist_dir}/fre/gfdl_msd_schemas/FRE/fre_make.json" + with open(schema_path, 'r') as s: + schema = json.load(s) + + print("\nValidating intermediate yaml:") + try: + validate(instance=loaded_yaml, schema=schema) + print(" Intermediate combined yaml VALID.") + except: + raise ValueError("\n\nIntermediate combined yaml NOT VALID.") + def list_platforms_subtool(yamlfile): """ List the platforms available @@ -51,10 +71,13 @@ def list_platforms_subtool(yamlfile): quick_combine(yamlfile,p,t) # Print experiment names - c = cy.yaml_load(f"{yamlpath}/{combined}") + yml = cy.yaml_load(f"{yamlpath}/{combined}") + + # Validate the yaml + validate_yaml(yml) print("\nPlatforms available:") - for i in c.get("platforms"): + for i in yml.get("platforms"): print(f' - {i.get("name")}') print("\n") From d89224df8d3a4322723ed904744f4eb8ae20073d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 4 Feb 2025 17:00:01 -0500 Subject: [PATCH 20/20] #315 Add validation test --- fre/list_/tests/test_list_platforms_script.py | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/fre/list_/tests/test_list_platforms_script.py b/fre/list_/tests/test_list_platforms_script.py index 0ed57671..aef98607 100644 --- a/fre/list_/tests/test_list_platforms_script.py +++ b/fre/list_/tests/test_list_platforms_script.py @@ -1,5 +1,5 @@ """ -Test fre list exps +Test fre list platforms """ import pytest from pathlib import Path @@ -9,6 +9,8 @@ # SET-UP TEST_DIR = Path("fre/make/tests") NM_EXAMPLE = Path("null_example") +PLATFORM = "None" +TARGET = "None" YAMLFILE = "null_model.yaml" BADYAMLFILE = "null_model_bad.yaml" EXP_NAME = YAMLFILE.split(".")[0] @@ -27,7 +29,7 @@ def test_platformyaml_exists(): assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/platforms.yaml").exists() def test_platforms_list(capfd): - ''' test list exps ''' + ''' test list platforms ''' list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") #Capture output @@ -57,25 +59,45 @@ def test_nocombinedyaml(): # Test individual functions operating correctly: combine and clean def test_correct_combine(): ''' test that combined yaml includes necesary keys ''' - p = None - t = None - yamlfile = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" # Combine model / experiment - list_platforms_script.quick_combine(yamlfile,p,t) + list_platforms_script.quick_combine(yamlfile_path,PLATFORM,TARGET) assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/combined-{EXP_NAME}.yaml").exists() comb_yamlfile = f"{TEST_DIR}/{NM_EXAMPLE}/combined-{EXP_NAME}.yaml" with open(comb_yamlfile, 'r') as yf: y = yaml.load(yf,Loader=yaml.Loader) - req_keys = ["name","platform","target","fre_properties","platforms"] + req_keys = ["name","platform","target","platforms"] for k in req_keys: if k in y.keys(): assert True else: assert False +def test_yamlvalidate(capfd): + ''' test yaml is being validated ''' + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + # Combine model / experiment + list_platforms_script.quick_combine(yamlfile_path,PLATFORM,TARGET) + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/combined-{EXP_NAME}.yaml").exists() + + comb_yamlfile = f"{TEST_DIR}/{NM_EXAMPLE}/combined-{EXP_NAME}.yaml" + with open(comb_yamlfile, 'r') as yf: + y = yaml.load(yf,Loader=yaml.Loader) + + # Validate and capture output + list_platforms_script.validate_yaml(y) + out,err=capfd.readouterr() + if "Intermediate combined yaml VALID." in out: + assert True + else: + assert False + +#def test_not_valid_yaml(): + def test_yamlremove(): ''' test intermediate combined yaml removed ''' # Remove combined yaml file