315.fre list tool

docs/tools.rst

@@ -63,3 +63,9 @@ fre yamltools
 =================
 
 .. include:: tools/yamltools.rst
+
+
+fre list
+=================
+
+.. include:: tools/listtools.rst

docs/tools/listtools.rst

@@ -0,0 +1,16 @@
+``exps``
+-----------------
+
+``fre list exps [options]``
+   - Purpose: Lists available post-processing experiments included in the yaml configurations
+   - Options:
+        - `-y, --yamlfile [experiment yaml]`
+
+``platforms``
+-----------------
+
+``fre list platforms [options]``
+   - Purpose: Lists available platforms included in the yaml configurations
+   - Options:
+        - `-y, --yamlfile [experiment yaml]`
+

fre/fre.py

@@ -28,7 +28,7 @@
     cls = LazyGroup,
     lazy_subcommands = {"pp": ".pp.frepp.pp_cli",
                        "catalog": ".catalog.frecatalog.catalog_cli",
-                       "list": ".list.frelist.list_cli",
+                       "list": ".list_.frelist.list_cli",
                        "check": ".check.frecheck.check_cli",
                        "run": ".run.frerun.run_cli",
                        "test": ".test.fretest.test_cli",

fre/list_/frelist.py

@@ -0,0 +1,32 @@
+''' fre lister '''
+
+import click
+from fre.list_ import list_experiments_script
+from fre.list_ import list_platforms_script
+
+@click.group(help=click.style(" - access fre list subcommands", fg=(232,204,91)))
+def list_cli():
+    ''' entry point to fre list click commands '''
+
+@list_cli.command()
+@click.option("-y",
+              "--yamlfile",
+              type=str,
+              help="YAML file to be used for parsing",
+              required=True)
+def exps(yamlfile):
+    """ - List experiments  available"""
+    list_experiments_script.list_experiments_subtool(yamlfile)
+
+@list_cli.command()
+@click.option("-y",
+              "--yamlfile",
+              type=str,
+              help="YAML file to be used for parsing",
+              required=True)
+def platforms(yamlfile):
+    """ - List platforms available """
+    list_platforms_script.list_platforms_subtool(yamlfile)
+
+if __name__ == "__main__":
+    list_cli()

fre/list_/list_experiments_script.py

@@ -0,0 +1,71 @@
+"""
+Script combines the model yaml with exp, platform, and target to list experiment information.
+"""
+import os
+from pathlib import Path
+import yaml
+import fre.yamltools.combine_yamls as cy
+
+def join_constructor(loader, node):
+    """
+    Allows FRE properties defined
+    in main yaml to be concatenated.
+    """
+    seq = loader.construct_sequence(node)
+    return ''.join([str(i) for i in seq])
+
+# To look into: ignore undefined alias error msg for listing?
+# Found this somewhere but don't fully understand yet 
+#class NoAliasDumper(yaml.SafeDumper):
+#    def ignore_aliases(self, data):
+#        return True
+
+def yaml_load(yamlfile):
+    """
+    Load the yamlfile
+    """
+    with open(yamlfile, 'r') as yf:
+        y = yaml.load(yf,Loader=yaml.Loader)
+
+    return y
+
+def quick_combine(yml, exp, platform, target):
+    """
+    Create intermediate combined model and exp. yaml
+    This is done to avoid an "undefined alias" error
+    """
+    # Combine model / experiment
+    comb = cy.init_pp_yaml(yml,exp,platform,target)
+    comb.combine_model()
+
+def clean(combined):
+    """
+    Remove intermediate combined yaml.
+    """
+    if Path(combined).exists():
+        os.remove(combined)
+        print(f"{combined} removed.")
+
+def list_experiments_subtool(yamlfile):
+    """
+    List the post-processing experiments available
+    """
+    # Regsiter tag handler
+    yaml.add_constructor('!join', join_constructor)
+
+    e = "exp_placeholder"
+    p = "p_placeholder"
+    t = "t_placeholder"
+
+    combined=f"combined-{e}.yaml"
+    # Combine model / experiment
+    quick_combine(yamlfile,e,p,t)
+
+    # Print experiment names
+    c = yaml_load(combined)
+
+    print("\nPost-processing experiments available:")
+    for i in c.get("experiments"):
+        print(f'   - {i.get("name")}')
+    print("\n")
+    clean(combined)

fre/list_/list_platforms_script.py

@@ -0,0 +1,75 @@
+"""
+Script combines the model yaml with exp, platform, and target to list experiment information.
+"""
+
+import os
+from pathlib import Path
+import yaml
+import fre.yamltools.combine_yamls as cy
+
+def join_constructor(loader, node):
+    """
+    Allows FRE properties defined
+    in main yaml to be concatenated.
+    """
+    seq = loader.construct_sequence(node)
+    return ''.join([str(i) for i in seq])
+
+# To look into: ignore undefined alias error msg for listing?
+# Found this somewhere but don't fully understand yet
+#class NoAliasDumper(yaml.SafeDumper):
+#    def ignore_aliases(self, data):
+#        return True
+
+def yaml_load(yamlfile):
+    """
+    Load the yamlfile
+    """
+    with open(yamlfile, 'r') as yf:
+        y = yaml.load(yf,Loader=yaml.Loader)
+
+    return y
+
+def quick_combine(yml, platform, target):
+    """
+    Combine the intermediate model and platforms yaml.
+    This is done to avoid an "undefined alias" error
+    """
+    # Combine model / experiment
+    comb = cy.init_compile_yaml(yml,platform,target)
+    comb.combine_model()
+    comb.combine_platforms()
+
+def clean(combined):
+    """
+    Remove intermediate combined yaml.
+    """
+    if Path(combined).exists():
+        os.remove(combined)
+        print(f"{combined} removed.")
+
+def list_platforms_subtool(yamlfile):
+    """
+    List the platforms available
+    """
+    # Regsiter tag handler
+    yaml.add_constructor('!join', join_constructor)
+
+    e = yamlfile.split("/")[-1].split(".")[0] #"exp_placeholder"
+    p = "p_placeholder"
+    t = "t_placeholder"
+
+    combined=f"combined-{e}.yaml"
+    yamlpath = os.path.dirname(yamlfile)
+
+    # Combine model / experiment
+    quick_combine(yamlfile,p,t)
+
+    # Print experiment names
+    c = yaml_load(os.path.join(yamlpath,combined))
+
+    print("\nPlatforms available:")
+    for i in c.get("platforms"):
+        print(f'    - {i.get("name")}')
+    print("\n")
+    clean(combined)

fre/list_/tests/null_example/compile.yaml

@@ -0,0 +1,48 @@
+compile:
+     experiment: "null_model_full"
+     container_addlibs: 
+     baremetal_linkerflags:  
+     src:
+          - component: "FMS"
+            repo: "https://github.com/NOAA-GFDL/FMS.git"
+            cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML -DHAVE_GETTID"
+            otherFlags: "-fallow-argument-mismatch" # only needed for gcc
+            branch: *branch
+          - component: "atmos_null"
+            requires: ["FMS"]
+            repo: "https://github.com/NOAA-GFDL/atmos_null.git"
+            branch: *branch 
+            cppdefs: "-DINTERNAL_FILE_NML"
+            otherFlags: *FMSincludes
+          - component: "land_null"
+            requires: ["FMS"]
+            repo: "https://github.com/NOAA-GFDL/land_null.git"
+            branch: *branch 
+            cppdefs: "-DINTERNAL_FILE_NML"
+            otherFlags: *FMSincludes
+          - component: "ice_param"
+            requires: ["FMS"]
+            repo: "https://github.com/NOAA-GFDL/ice_param.git"
+            branch: *branch 
+            cppdefs: "-DINTERNAL_FILE_NML"
+            otherFlags: *FMSincludes
+          - component: "ocean_null"
+            requires: ["FMS"]
+            repo: "https://github.com/NOAA-GFDL/ocean_null.git"
+            branch: *branch
+            otherFlags: *FMSincludes
+          - component: "ice_null"
+            requires: ["FMS", "ice_param", "ocean_null"]
+            repo: "https://github.com/NOAA-GFDL/ice_null.git"
+            branch: *branch 
+            cppdefs: "-DINTERNAL_FILE_NML"
+            otherFlags: *FMSincludes
+          - component: "coupler"
+            requires: ["FMS", "ocean_null", "atmos_null", "land_null", "ice_param", "ocean_null", "ice_null"]
+            repo: "https://github.com/NOAA-GFDL/FMScoupler.git"
+            branch: *branch 
+            otherFlags: *FMSincludes
+            cppdefs: "-D_USE_LEGACY_LAND_ -Duse_AM3_physics -DINTERNAL_FILE_NML"
+            paths: [ "coupler/full",
+                     "coupler/shared" ]
+

fre/list_/tests/null_example/null_model.yaml

@@ -0,0 +1,57 @@
+# reusable variables
+fre_properties:
+  - &VERSION        "null"
+  - &FRE_STEM       !join [null_model/, *VERSION]
+
+  # amip
+  - &EXP_AMIP_START     "19790101T0000Z"
+  - &EXP_AMIP_END       "20200101T0000Z"
+  - &ANA_AMIP_START     "19800101T0000Z"
+  - &ANA_AMIP_END       "20200101T0000Z"
+
+  - &PP_AMIP_CHUNK96    "P1Y"
+  - &PP_AMIP_CHUNK384   "P1Y"
+  - &PP_XYINTERP96      "180,288"
+  - &PP_XYINTERP384     "720,1152"
+
+  # compile information
+  - &release             "f1a1r1"
+  - &INTEL               "intel-classic"
+  - &FMSincludes         "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include"
+  - &momIncludes         "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include"
+
+# compile information
+build:
+  compileYaml: "compile.yaml"
+  platformYaml: "platforms.yaml"
+
+shared:
+  # directories shared across tools
+  directories: &shared_directories
+    history_dir:  !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
+    pp_dir:       !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp]
+    analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
+    ptmp_dir:     "/xtmp/$USER/ptmp"
+
+  # shared pp settings
+  postprocess: 
+    settings: &shared_settings
+      site:               "ppan"
+    switches: &shared_switches
+      do_statics:         True
+      do_timeavgs:        True
+      clean_work:         True
+      do_refinediag:      False
+      do_atmos_plevel_masking: True
+      do_preanalysis:     False
+      do_analysis:        True
+
+experiments:
+  - name: "null_model_0"
+    pp:
+  - name: "null_model_1"
+    pp:
+  - name: "null_model_2"
+    pp:
+  - name: "null_model_3"
+    pp:

fre/list_/tests/null_example/platforms.yaml

@@ -0,0 +1,20 @@
+platforms:
+   - name: ncrc5.intel23
+     compiler: intel
+     modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+     modules: [!join [*INTEL, "/2023.2.0"],"fre/bronx-21",cray-hdf5/1.12.2.11, cray-netcdf/4.9.0.11]
+     mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"]
+     modelRoot: ${HOME}/fremake_canopy/test
+   - name: hpcme.2023
+     compiler: intel
+     RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load [email protected]", "spack load [email protected]"]
+     modelRoot: /apps
+     container: True
+     containerBuild: "podman"
+     containerRun: "apptainer" 
+     containerBase: "ecpe4s/noaa-intel-prototype:2023.09.25" 
+     mkTemplate: "/apps/mkmf/templates/hpcme-intel21.mk"
+   - name: ci.gnu
+     compiler: gnu
+     mkTemplate: /__w/fre-cli/fre-cli/mkmf/templates/linux-ubuntu-xenial-gnu.mk
+     modelRoot: ${TEST_BUILD_DIR}/fremake_canopy/test