-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into data_table_refactor
- Loading branch information
Showing
8 changed files
with
293 additions
and
473 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| [submodule "fms_yaml_tools/schema/gfdl_msd_schemas"] | ||
| path = fms_yaml_tools/schema/gfdl_msd_schemas | ||
| url = https://github.com/NOAA-GFDL/gfdl_msd_schemas.git |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,120 +1,45 @@ | ||
| #!/usr/bin/env python3 | ||
| """ | ||
| *********************************************************************** | ||
| * GNU Lesser General Public License | ||
| * | ||
| * This file is part of the GFDL Flexible Modeling System (FMS) YAML tools. | ||
| * | ||
| * FMS_yaml_tools is free software: you can redistribute it and/or modify it under | ||
| * the terms of the GNU Lesser General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or (at | ||
| * your option) any later version. | ||
| * | ||
| * FMS_yaml_tools is distributed in the hope that it will be useful, but WITHOUT | ||
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | ||
| * for more details. | ||
| * | ||
| * You should have received a copy of the GNU Lesser General Public | ||
| * License along with FMS. If not, see <http://www.gnu.org/licenses/>. | ||
| *********************************************************************** | ||
| """ | ||
| # *********************************************************************** | ||
| # * GNU Lesser General Public License | ||
| # * | ||
| # * This file is part of the GFDL Flexible Modeling System (FMS) YAML | ||
| # * tools. | ||
| # * | ||
| # * FMS_yaml_tools is free software: you can redistribute it and/or | ||
| # * modify it under the terms of the GNU Lesser General Public License | ||
| # * as published by the Free Software Foundation, either version 3 of the | ||
| # * License, or (at your option) any later version. | ||
| # * | ||
| # * FMS_yaml_tools is distributed in the hope that it will be useful, but | ||
| # * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
| # * General Public License for more details. | ||
| # * | ||
| # * You should have received a copy of the GNU Lesser General Public | ||
| # * License along with FMS. If not, see <http://www.gnu.org/licenses/>. | ||
| # *********************************************************************** | ||
|
|
||
| import click | ||
| import pkg_resources | ||
| from fms_yaml_tools.schema.validate_schema import validate_yaml | ||
|
|
||
|
|
||
| @click.command() | ||
| # Debug is used to print more information to the screen. | ||
| @click.option('--debug/--no-debug', type=click.BOOL, show_default=True, default=False, | ||
| help="Print steps in the validation") | ||
| # This option controls whether or not the program prints a success message | ||
| @click.option('--success/--no-show-success', type=click.BOOL, show_default=True, default=False, | ||
| help="Print success message") | ||
| # This is the path to the file to be validated | ||
| @click.argument("ypath") | ||
| def validate_data_yaml(ypath, debug, success): | ||
| """ Validates a data_table.yaml based on a schema to check for any errors. \n | ||
| YPATH - Path to the data_table.yaml file to be validated against the schema \n | ||
| """ | ||
| data_path = pkg_resources.resource_filename('fms_yaml_tools', '') | ||
| validate_yaml(ypath, data_path + "/schema/gfdl_msd_schemas/FMS/data_table.json", debug, success) | ||
|
|
||
| """ Determine if a yaml data_table is valid. | ||
| Run `python3 is_valid_data_table_yaml.py -h` for more details | ||
| Author: Uriel Ramirez 05/27/2022 | ||
| """ | ||
|
|
||
| import yaml | ||
| import sys | ||
| import argparse | ||
|
|
||
| def check_gridname(grid_name): | ||
| """Check if the input grid_name is valid. Crashes if it not.""" | ||
| valid = ["OCN", "LND", "ATM", "ICE"] | ||
| if (grid_name not in valid): raise Exception(grid_name+ ' is not a valid gridname. The only values allowed are "OCN", "LND", "ATM", "ICE"') | ||
|
|
||
| def check_fieldname_code(fieldname): | ||
| if (fieldname == ""): raise Exception("Fieldname can't be empty") | ||
|
|
||
| def check_filename_and_field(field, interp_method): | ||
| if (field =="" and interp_method != ""): raise Exception('If "fieldname_file" is empty, interp_method must be empty') | ||
|
|
||
| def check_interp_method(interp_method): | ||
| """Check if the interp method is valid. Crashes if it not. """ | ||
| valid = ["bilinear", "bicubic", "none"] | ||
| if (interp_method not in valid): raise Exception(interp_method + ' is not a valid interp_method. The only values allowed are "bilinear", "bicubic", and "none"') | ||
|
|
||
| def check_region_type(region_type): | ||
| """Check if the input region type is valid. Crashes if it is not.""" | ||
| valid = ["inside_region", "outside_region"] | ||
| if (region_type not in valid): raise Exception(region_type + 'is not a valid region_type. The only values allowed are "inside_region" and "outside_region"') | ||
|
|
||
| def check_if_bounds_present(entry): | ||
| """Check if the region bounds are valid, crashes if they are not """ | ||
| if ("lat_start" not in entry): raise Exception('lat_start must be present if region_type is set') | ||
| if ("lat_end" not in entry): raise Exception('lat_end must be present if region_type is set') | ||
| if ("lon_start" not in entry): raise Exception('lon_start must be present if region_type is set') | ||
| if ("lon_end" not in entry): raise Exception('lon_end must be present if region_type is set') | ||
|
|
||
| def check_region(my_type, start, end): | ||
| """Check if the region is defined correctly. Crashes if it not. """ | ||
| if (start > end): raise Exception(my_type+"_start is greater than "+my_type+"_end") | ||
|
|
||
| def main(): | ||
| parser = argparse.ArgumentParser(prog='is_valid_data_table_yaml', \ | ||
| description="Determines if a yaml data_table is valid. \ | ||
| Requires pyyaml (https://pyyaml.org/) \ | ||
| More details on the yaml format can be found in \ | ||
| https://github.com/NOAA-GFDL/FMS/tree/main/data_override") | ||
| parser.add_argument('-f', type=str, help='Name of the data_table yaml to check' ) | ||
| in_data_table = parser.parse_args().f | ||
|
|
||
| with open(in_data_table) as fl: | ||
| my_table = yaml.safe_load(fl) | ||
| for key, value in my_table.items(): | ||
| for i in range(0, len(value)): | ||
| entry = value[i] | ||
| if "gridname" not in entry: | ||
| raise Exception("gridname is a required key!") | ||
| gridname = entry["gridname"] | ||
| check_gridname(gridname) | ||
|
|
||
| if "fieldname_code" not in entry: | ||
| raise Exception("fieldname_code is a required key!") | ||
|
|
||
| fieldname_code = entry["fieldname_code"] | ||
| check_fieldname_code(fieldname_code) | ||
|
|
||
| if "fieldname_file" in entry: | ||
| fieldname_file = entry["fieldname_file"] | ||
|
|
||
| if "file_name" in entry: | ||
| file_name = entry["file_name"] | ||
|
|
||
| if "interpol_method" in entry: | ||
| interp_method = entry["interpol_method"] | ||
|
|
||
| check_interp_method(interp_method) | ||
| check_filename_and_field(fieldname_file, interp_method) | ||
|
|
||
| if "factor" not in entry: | ||
| raise Exception("factor is a required key") | ||
|
|
||
| factor = entry["factor"] | ||
|
|
||
| if "region_type" in entry: | ||
| region_type = entry["region_type"] | ||
| check_region_type(region_type) | ||
| check_if_bounds_present(entry) | ||
|
|
||
| lat_start = entry["lat_start"] | ||
| lat_end = entry["lat_end"] | ||
| check_region("lat", lat_start, lat_end) | ||
|
|
||
| lon_start = entry["lon_start"] | ||
| lon_end = entry["lon_end"] | ||
| check_region("lon", lon_start, lon_end) | ||
|
|
||
| if __name__ == "__main__": | ||
| main() | ||
| validate_data_yaml(prog_name="validate_data_yaml") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,114 +1,102 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| """ | ||
| *********************************************************************** | ||
| * GNU Lesser General Public License | ||
| * | ||
| * This file is part of the GFDL Flexible Modeling System (FMS) YAML tools. | ||
| * | ||
| * FMS_yaml_tools is free software: you can redistribute it and/or modify it under | ||
| * the terms of the GNU Lesser General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or (at | ||
| * your option) any later version. | ||
| * | ||
| * FMS_yaml_tools is distributed in the hope that it will be useful, but WITHOUT | ||
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | ||
| * for more details. | ||
| * | ||
| * You should have received a copy of the GNU Lesser General Public | ||
| * License along with FMS. If not, see <http://www.gnu.org/licenses/>. | ||
| *********************************************************************** | ||
| """ | ||
|
|
||
| """ Print the files and variables in a diag yaml | ||
| Author: Tom Robinson 12/7/2023 | ||
| """ | ||
| # *********************************************************************** | ||
| # * GNU Lesser General Public License | ||
| # * | ||
| # * This file is part of the GFDL Flexible Modeling System (FMS) YAML | ||
| # * tools. | ||
| # * | ||
| # * FMS_yaml_tools is free software: you can redistribute it and/or | ||
| # * modify it under the terms of the GNU Lesser General Public License | ||
| # * as published by the Free Software Foundation, either version 3 of the | ||
| # * License, or (at your option) any later version. | ||
| # * | ||
| # * FMS_yaml_tools is distributed in the hope that it will be useful, but | ||
| # * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
| # * General Public License for more details. | ||
| # * | ||
| # * You should have received a copy of the GNU Lesser General Public | ||
| # * License along with FMS. If not, see <http://www.gnu.org/licenses/>. | ||
| # *********************************************************************** | ||
|
|
||
| import click | ||
| import yaml | ||
| import sys | ||
| import copy | ||
|
|
||
|
|
||
| @click.command() | ||
| @click.option('--fileinfo/--no-fileinfo', type=click.BOOL, show_default=True, default=False, | ||
| help="Print all of the file information. Default only prints history file names") | ||
| help="Print all of the file information. Default only prints history file names") | ||
| @click.option('--varlist/--no-varlist', type=click.BOOL, show_default=True, default=False, | ||
| help="Prints the list of variables and table information below each history file") | ||
| help="Prints the list of variables and table information below each history file") | ||
| @click.option('--comma/--no-comma', type=click.BOOL, show_default=True, default=False, | ||
| help="With --varlist gives a comma separated list of the variables instead of a vertical listing. "\ | ||
| +"Only shows variable names (no other information). "\ | ||
| +"Does nothing if --varlist isn't used.") | ||
| help="With --varlist gives a comma separated list of the variables instead of a vertical listing. " | ||
| + "Only shows variable names (no other information). " | ||
| + "Does nothing if --varlist isn't used.") | ||
| @click.option('--varfiles/--no-varfiles', type=click.BOOL, show_default=True, default=False, | ||
| help="Prints variable names and a list of the files they appear in") | ||
| help="Prints variable names and a list of the files they appear in") | ||
| @click.argument("table") | ||
| def dyl(table, fileinfo, varlist, comma, varfiles): | ||
| """ Lists the history file names in a diag table YAML | ||
| """ Lists the history file names in a diag table YAML | ||
| TABLE is the path of the diag table YAML file | ||
| """ | ||
| with open(table) as fl: | ||
| my_table = yaml.safe_load(fl) | ||
| print_diag_file(my_table, fileinfo, print_vars=varlist, comma=comma) | ||
| if varfiles: | ||
| print_varstats(my_table) | ||
|
|
||
|
|
||
| TABLE is the path of the diag table YAML file | ||
| """ | ||
| with open(table) as fl: | ||
| my_table = yaml.safe_load(fl) | ||
| print_diag_file(my_table, fileinfo, print_vars=varlist, comma=comma) | ||
| if varfiles: | ||
| print_varstats(my_table) | ||
| """ Prints the varlist for the given diag_file """ | ||
| def print_diag_file_vars(my_table, diag_file, comma=False): | ||
| if "varlist" in my_table["diag_files"][diag_file]: | ||
| if not comma: ## The default is one variable per line | ||
| for var in my_table["diag_files"][diag_file]["varlist"]: | ||
| print(" - ",var) | ||
| """ Prints the varlist for the given diag_file """ | ||
| if "varlist" in my_table["diag_files"][diag_file]: | ||
| if not comma: # The default is one variable per line | ||
| for var in my_table["diag_files"][diag_file]["varlist"]: | ||
| print(" - ", var) | ||
| else: | ||
| varlist = [] | ||
| for v in my_table["diag_files"][diag_file]["varlist"]: | ||
| varlist.append(v["var_name"]) | ||
| print(" - ", varlist) | ||
| else: | ||
| varlist = [] | ||
| for v in my_table["diag_files"][diag_file]["varlist"]: | ||
| varlist.append(v["var_name"]) | ||
| print(" - ",varlist) | ||
| else: | ||
| print(" ("+my_table['diag_files'][diag_file]["file_name"]+" has no varlist)") | ||
| print(" ("+my_table['diag_files'][diag_file]["file_name"]+" has no varlist)") | ||
|
|
||
| """ Prints information for each variable """ | ||
| def print_varstats(my_table): | ||
| nvars = 0 | ||
| allvars = [] | ||
| filenames = [] | ||
| for f in my_table["diag_files"]: #loop through files | ||
| if "varlist" in f: | ||
| for v in f["varlist"]: | ||
| nvars = nvars + 1 | ||
| allvars.append(v["var_name"]) | ||
| filenames.append(f["file_name"]) | ||
| # print(v["var_name"],f["file_name"]) | ||
| for var in set(allvars): | ||
| filelist = [] | ||
| for f in my_table["diag_files"]: #loop through files | ||
| if "varlist" in f: | ||
| for v in f["varlist"]: | ||
| if v["var_name"] == var: | ||
| filelist.append(f["file_name"]) | ||
| print (var+' in ',filelist) | ||
| # allvarscopy = copy.deepcopy(allvars) | ||
| # for (v,f) in zip(allvarscopy,filenames): | ||
| # filelist = [] | ||
| # for vdup in allvars: | ||
| # if vdup == v: | ||
| # print (vdup,v,f) | ||
| # filelist.append(f) | ||
| # print(v+" in files ",filelist) | ||
|
|
||
| def print_varstats(my_table): | ||
| """ Prints information for each variable """ | ||
| nvars = 0 | ||
| allvars = [] | ||
| filenames = [] | ||
| for f in my_table["diag_files"]: # loop through files | ||
| if "varlist" in f: | ||
| for v in f["varlist"]: | ||
| nvars = nvars + 1 | ||
| allvars.append(v["var_name"]) | ||
| filenames.append(f["file_name"]) | ||
| for var in set(allvars): | ||
| filelist = [] | ||
| for f in my_table["diag_files"]: # loop through files | ||
| if "varlist" in f: | ||
| for v in f["varlist"]: | ||
| if v["var_name"] == var: | ||
| filelist.append(f["file_name"]) | ||
| print(var + ' in ', filelist) | ||
|
|
||
|
|
||
| """ Prints the diag_files in a diag_table.yaml """ | ||
| def print_diag_file(my_table, file_info, print_vars=False, comma=False): | ||
| for f in my_table["diag_files"]: #loop through files | ||
| if file_info: #Print all file info | ||
| fonly = copy.deepcopy(f) #Create a copy and pop off the varlist | ||
| fonly.pop("varlist", None) | ||
| print ("FILE: ",fonly) #Print file information without the varlist | ||
| else: #Print only the file name | ||
| print (f["file_name"]) | ||
| if print_vars: #Print variable list/info | ||
| i = my_table["diag_files"].index(f) | ||
| print_diag_file_vars(my_table, i, comma) | ||
| """ Prints the diag_files in a diag_table.yaml """ | ||
| for f in my_table["diag_files"]: # loop through files | ||
| if file_info: # Print all file info | ||
| fonly = copy.deepcopy(f) # Create a copy and pop off the varlist | ||
| fonly.pop("varlist", None) | ||
| print("FILE: ", fonly) # Print file information without the varlist | ||
| else: # Print only the file name | ||
| print(f["file_name"]) | ||
| if print_vars: # Print variable list/info | ||
| i = my_table["diag_files"].index(f) | ||
| print_diag_file_vars(my_table, i, comma) | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| dyl(prog_name="dyl") | ||
| dyl(prog_name="dyl") |
Oops, something went wrong.