Skip to content

Commit

Permalink
Add support GENERAL_OBSERVATIONs in fmuobs yaml
Browse files Browse the repository at this point in the history 
Also ensure that integer values are printed as integes in ert-obs
output
  • Loading branch information
@berland
berland committed Apr 28, 2021
1 parent 9501b7c commit 009ccd0
Show file tree
Hide file tree
Showing 5 changed files with 202 additions and 4 deletions.
22 changes: 21 additions & 1 deletion src/subscript/fmuobs/parsers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,7 @@ def smrydictlist2df(smrylist):

def blockdictlist2df(blocklist):
"""Parse a list structure (subpart of yaml syntax) of block observations
into dataframe format
into dataframe format
The internal dataframe format uses "LABEL" and "OBS" as unique keys
for individual observations. These are constructed if not present.
Expand Down Expand Up @@ -550,6 +550,24 @@ def blockdictlist2df(blocklist):
return dframe


def generaldictlist2df(generallist):
"""Parse a list structure (subpart of yaml syntax) of general observations
into dataframe format
Args:
generallist (list): List of dictionaries with GENERAL_OBSERVATIONs
Returns:
pd.DataFrame
"""
rows = []
for obs in filter(None, generallist):
rowdict = {"CLASS": "GENERAL_OBSERVATION"}
rowdict.update(uppercase_dictkeys(obs))
rows.append(rowdict)
return pd.DataFrame(rows)


def obsdict2df(obsdict):
"""Convert an observation dictionary (with YAML file format structure)
into the internal dataframe representation
Expand All @@ -569,4 +587,6 @@ def obsdict2df(obsdict):
dframes.append(smrydictlist2df(obsdict["smry"]))
if "rft" in obsdict:
dframes.append(blockdictlist2df(obsdict["rft"]))
if "general" in obsdict:
dframes.append(generaldictlist2df(obsdict["general"]))
return pd.concat(dframes, ignore_index=True)
31 changes: 30 additions & 1 deletion src/subscript/fmuobs/writers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,14 @@ def dfgeneral2ertobs(obs_df):
"ERROR_COVAR",
]:
if dataname in row and not pd.isnull(row[dataname]):
ertobs_str += " " + dataname + " = " + str(row[dataname]) + ";\n"
value = row[dataname]
try:
# Ensure integers are printed as integers
if int(value) == float(value):
value = int(value)
except ValueError:
pass
ertobs_str += " " + dataname + " = " + str(value) + ";\n"
ertobs_str += "};\n"

# Remove empty curly braces and return
Expand Down Expand Up @@ -350,6 +357,22 @@ def block_df2obsdict(block_df):
return block_obs_list


def general_df2obsdict(general_df):
"""Generate a dictionary structure suitable for yaml
for general observations in dataframe representation
Args:
general_df (pd.DataFrame)
Returns:
list: List of dictionaries
"""
general_obs_list = []
for _, row in general_df.iterrows():
general_obs_list.append(lowercase_dictkeys(row.dropna().to_dict()))
return general_obs_list


def df2obsdict(obs_df):
"""Generate a dictionary structure of all observations, this data structure
is designed to look good in yaml, and is supported by WebViz and
Expand Down Expand Up @@ -377,6 +400,12 @@ def df2obsdict(obs_df):
obs_df.set_index("CLASS").loc[["BLOCK_OBSERVATION"]]
)

# Process GENERAL_OBSERVATION:
if "GENERAL_OBSERVATION" in obs_df["CLASS"].values:
obsdict[CLASS_SHORTNAME["GENERAL_OBSERVATION"]] = general_df2obsdict(
obs_df.set_index("CLASS").loc[["GENERAL_OBSERVATION"]]
)

return obsdict


Expand Down
56 changes: 56 additions & 0 deletions tests/test_fmuobs_parsers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
split_by_sep_in_masked_string,
smrydictlist2df,
blockdictlist2df,
generaldictlist2df,
obsdict2df,
)

Expand Down Expand Up @@ -857,6 +858,61 @@ def test_blockdictlist2df(blocklist, expected_df):
)


# generaldictlist2df
@pytest.mark.parametrize(
"generallist, expected_df",
[
([{}], pd.DataFrame()),
(
[{"label": "GEN_OBS1"}],
pd.DataFrame([{"CLASS": "GENERAL_OBSERVATION", "LABEL": "GEN_OBS1"}]),
),
(
[{"label": "GEN_OBS1"}, {"label": "GEN_OBS2"}],
pd.DataFrame(
[
{"CLASS": "GENERAL_OBSERVATION", "LABEL": "GEN_OBS1"},
{"CLASS": "GENERAL_OBSERVATION", "LABEL": "GEN_OBS2"},
]
),
),
#################################################################
(
[
{
"CLASS": "GENERAL_OBSERVATION",
"LABEL": "GEN_OBS1",
"DATA": "SOME_FIELD",
"INDEX_LIST": "0,3,9",
"RESTART": 20,
"OBS_FILE": "some_file.txt",
}
],
pd.DataFrame(
[
{
"CLASS": "GENERAL_OBSERVATION",
"LABEL": "GEN_OBS1",
"DATA": "SOME_FIELD",
"INDEX_LIST": "0,3,9",
"RESTART": 20,
"OBS_FILE": "some_file.txt",
},
]
),
),
],
)
def test_generaldictlist2df(generallist, expected_df):
"""Test converting general observations in dict (yaml) format into
internal dataframe format"""
pd.testing.assert_frame_equal(
generaldictlist2df(generallist).sort_index(axis=1),
expected_df.sort_index(axis=1),
check_dtype=False,
)


# obsdictlist2df
@pytest.mark.parametrize(
"obsdict, expected_df",
Expand Down
87 changes: 85 additions & 2 deletions tests/test_fmuobs_writers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
summary_df2obsdict,
convert_dframe_date_to_str,
block_df2obsdict,
general_df2obsdict,
df2resinsight_df,
)
from subscript.fmuobs.parsers import ertobs2df
Expand Down Expand Up @@ -302,7 +303,7 @@ def test_dfhistory2ertobs(obs_df, expected_str):
"CLASS": "GENERAL_OBSERVATION",
"LABEL": "GEN_OBS1",
"DATA": "RFT_BH67",
"RESTART": 20,
"RESTART": 20.0,
"OBS_FILE": "some_file.txt",
"INDEX_LIST": "1,2,3,4",
"ERROR_COVAR": "e_covar.txt",
Expand Down Expand Up @@ -375,7 +376,7 @@ def test_dfgeneral2ertobs(obs_df, expected_str):
HISTORY_OBSERVATION WOPR:P1;
GENERAL_OBSERVATION GEN_OBS1 {
DATA = RFT_BH67;
RESTART = 20.0;
RESTART = 20;
};""",
),
],
Expand Down Expand Up @@ -498,6 +499,88 @@ def test_block_df2obsdict(obs_df, expected_dict):
assert block_df2obsdict(obs_df) == expected_dict


# test_general_df2obsdict()
@pytest.mark.parametrize(
"general_df, expected_listofdict",
[
(
pd.DataFrame(
[
{
"CLASS": "GENERAL_OBSERVATION",
"LABEL": "GEN_OBS1",
"DATA": "SOME_FIELD",
"RESTART": 20,
"OBS_FILE": "some_file.txt",
}
]
),
[
{
"class": "GENERAL_OBSERVATION",
"label": "GEN_OBS1",
"data": "SOME_FIELD",
"restart": 20,
"obs_file": "some_file.txt",
}
],
),
(
pd.DataFrame(
[
{
"CLASS": "GENERAL_OBSERVATION",
"LABEL": "GEN_OBS1",
"DATA": "SOME_FIELD",
"INDEX_LIST": "0,3,9",
"RESTART": 20,
"OBS_FILE": "some_file.txt",
},
]
),
[
{
"class": "GENERAL_OBSERVATION",
"label": "GEN_OBS1",
"data": "SOME_FIELD",
"index_list": "0,3,9",
"restart": 20,
"obs_file": "some_file.txt",
}
],
),
(
pd.DataFrame(
[
{
"CLASS": "GENERAL_OBSERVATION",
"LABEL": "GEN_OBS1",
},
{
"CLASS": "GENERAL_OBSERVATION",
"LABEL": "GEN_OBS2",
},
]
),
[
{
"class": "GENERAL_OBSERVATION",
"label": "GEN_OBS1",
},
{
"class": "GENERAL_OBSERVATION",
"label": "GEN_OBS2",
},
],
),
],
)
def test_general_df2obsdict(general_df, expected_listofdict):
"""Test converting from dataframe representation for GENERAL observations
to the dictionary representation designed for yaml output"""
assert general_df2obsdict(general_df) == expected_listofdict


@pytest.mark.parametrize(
"dframe, expected_dframe",
[
Expand Down
10 changes: 10 additions & 0 deletions tests/testdata_fmuobs/ert-doc.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,3 +63,13 @@ smry:
error: 10.0
label: SEP_TEST_2008
value: 213.0
general:
- label: GEN_OBS1
data: SOME_FIELD
restart: 20.0
obs_file: some_file.txt
- label: GEN_OBS2
data: SOME_FIELD
index_list: "0,3,9"
restart: 20.0
obs_file: some_file.txt

0 comments on commit 009ccd0

Please sign in to comment.