[fmuobs] Add naive support for GENERAL_OBSERVATIONs in yaml

src/subscript/fmuobs/parsers.py

@@ -504,7 +504,7 @@ def smrydictlist2df(smrylist):
 
 def blockdictlist2df(blocklist):
     """Parse a list structure (subpart of yaml syntax) of block observations
-    into  dataframe format
+    into dataframe format
 
     The internal dataframe format uses "LABEL" and "OBS" as unique keys
     for individual observations. These are constructed if not present.
@@ -550,6 +550,24 @@ def blockdictlist2df(blocklist):
     return dframe
 
 
+def generaldictlist2df(generallist):
+    """Parse a list structure (subpart of yaml syntax) of general observations
+    into dataframe format
+
+    Args:
+         generallist (list): List of dictionaries with GENERAL_OBSERVATIONs
+
+    Returns:
+        pd.DataFrame
+    """
+    rows = []
+    for obs in filter(None, generallist):
+        rowdict = {"CLASS": "GENERAL_OBSERVATION"}
+        rowdict.update(uppercase_dictkeys(obs))
+        rows.append(rowdict)
+    return pd.DataFrame(rows)
+
+
 def obsdict2df(obsdict):
     """Convert an observation dictionary (with YAML file format structure)
     into the internal dataframe representation
@@ -569,4 +587,6 @@ def obsdict2df(obsdict):
         dframes.append(smrydictlist2df(obsdict["smry"]))
     if "rft" in obsdict:
         dframes.append(blockdictlist2df(obsdict["rft"]))
+    if "general" in obsdict:
+        dframes.append(generaldictlist2df(obsdict["general"]))
     return pd.concat(dframes, ignore_index=True)

src/subscript/fmuobs/writers.py

@@ -183,7 +183,14 @@ def dfgeneral2ertobs(obs_df):
             "ERROR_COVAR",
         ]:
             if dataname in row and not pd.isnull(row[dataname]):
-                ertobs_str += "    " + dataname + " = " + str(row[dataname]) + ";\n"
+                value = row[dataname]
+                try:
+                    # Ensure integers are printed as integers
+                    if int(value) == float(value):
+                        value = int(value)
+                except ValueError:
+                    pass
+                ertobs_str += "    " + dataname + " = " + str(value) + ";\n"
         ertobs_str += "};\n"
 
     # Remove empty curly braces and return
@@ -350,6 +357,22 @@ def block_df2obsdict(block_df):
     return block_obs_list
 
 
+def general_df2obsdict(general_df):
+    """Generate a dictionary structure suitable for yaml
+    for general observations in dataframe representation
+
+    Args:
+        general_df (pd.DataFrame)
+
+    Returns:
+        list: List of dictionaries
+    """
+    general_obs_list = []
+    for _, row in general_df.iterrows():
+        general_obs_list.append(lowercase_dictkeys(row.dropna().to_dict()))
+    return general_obs_list
+
+
 def df2obsdict(obs_df):
     """Generate a dictionary structure of all observations, this data structure
     is designed to look good in yaml, and is supported by WebViz and
@@ -377,6 +400,12 @@ def df2obsdict(obs_df):
             obs_df.set_index("CLASS").loc[["BLOCK_OBSERVATION"]]
         )
 
+    # Process GENERAL_OBSERVATION:
+    if "GENERAL_OBSERVATION" in obs_df["CLASS"].values:
+        obsdict[CLASS_SHORTNAME["GENERAL_OBSERVATION"]] = general_df2obsdict(
+            obs_df.set_index("CLASS").loc[["GENERAL_OBSERVATION"]]
+        )
+
     return obsdict
 
 

tests/test_fmuobs_parsers.py

@@ -22,6 +22,7 @@
     split_by_sep_in_masked_string,
     smrydictlist2df,
     blockdictlist2df,
+    generaldictlist2df,
     obsdict2df,
 )
 
@@ -857,6 +858,61 @@ def test_blockdictlist2df(blocklist, expected_df):
     )
 
 
+# generaldictlist2df
+@pytest.mark.parametrize(
+    "generallist, expected_df",
+    [
+        ([{}], pd.DataFrame()),
+        (
+            [{"label": "GEN_OBS1"}],
+            pd.DataFrame([{"CLASS": "GENERAL_OBSERVATION", "LABEL": "GEN_OBS1"}]),
+        ),
+        (
+            [{"label": "GEN_OBS1"}, {"label": "GEN_OBS2"}],
+            pd.DataFrame(
+                [
+                    {"CLASS": "GENERAL_OBSERVATION", "LABEL": "GEN_OBS1"},
+                    {"CLASS": "GENERAL_OBSERVATION", "LABEL": "GEN_OBS2"},
+                ]
+            ),
+        ),
+        #################################################################
+        (
+            [
+                {
+                    "CLASS": "GENERAL_OBSERVATION",
+                    "LABEL": "GEN_OBS1",
+                    "DATA": "SOME_FIELD",
+                    "INDEX_LIST": "0,3,9",
+                    "RESTART": 20,
+                    "OBS_FILE": "some_file.txt",
+                }
+            ],
+            pd.DataFrame(
+                [
+                    {
+                        "CLASS": "GENERAL_OBSERVATION",
+                        "LABEL": "GEN_OBS1",
+                        "DATA": "SOME_FIELD",
+                        "INDEX_LIST": "0,3,9",
+                        "RESTART": 20,
+                        "OBS_FILE": "some_file.txt",
+                    },
+                ]
+            ),
+        ),
+    ],
+)
+def test_generaldictlist2df(generallist, expected_df):
+    """Test converting general observations in dict (yaml) format into
+    internal dataframe format"""
+    pd.testing.assert_frame_equal(
+        generaldictlist2df(generallist).sort_index(axis=1),
+        expected_df.sort_index(axis=1),
+        check_dtype=False,
+    )
+
+
 # obsdictlist2df
 @pytest.mark.parametrize(
     "obsdict, expected_df",

tests/test_fmuobs_writers.py

@@ -19,6 +19,7 @@
     summary_df2obsdict,
     convert_dframe_date_to_str,
     block_df2obsdict,
+    general_df2obsdict,
     df2resinsight_df,
 )
 from subscript.fmuobs.parsers import ertobs2df
@@ -302,7 +303,7 @@ def test_dfhistory2ertobs(obs_df, expected_str):
                         "CLASS": "GENERAL_OBSERVATION",
                         "LABEL": "GEN_OBS1",
                         "DATA": "RFT_BH67",
-                        "RESTART": 20,
+                        "RESTART": 20.0,
                         "OBS_FILE": "some_file.txt",
                         "INDEX_LIST": "1,2,3,4",
                         "ERROR_COVAR": "e_covar.txt",
@@ -375,7 +376,7 @@ def test_dfgeneral2ertobs(obs_df, expected_str):
 HISTORY_OBSERVATION WOPR:P1;
 GENERAL_OBSERVATION GEN_OBS1 {
     DATA = RFT_BH67;
-    RESTART = 20.0;
+    RESTART = 20;
 };""",
         ),
     ],
@@ -498,6 +499,88 @@ def test_block_df2obsdict(obs_df, expected_dict):
     assert block_df2obsdict(obs_df) == expected_dict
 
 
+# test_general_df2obsdict()
+@pytest.mark.parametrize(
+    "general_df, expected_listofdict",
+    [
+        (
+            pd.DataFrame(
+                [
+                    {
+                        "CLASS": "GENERAL_OBSERVATION",
+                        "LABEL": "GEN_OBS1",
+                        "DATA": "SOME_FIELD",
+                        "RESTART": 20,
+                        "OBS_FILE": "some_file.txt",
+                    }
+                ]
+            ),
+            [
+                {
+                    "class": "GENERAL_OBSERVATION",
+                    "label": "GEN_OBS1",
+                    "data": "SOME_FIELD",
+                    "restart": 20,
+                    "obs_file": "some_file.txt",
+                }
+            ],
+        ),
+        (
+            pd.DataFrame(
+                [
+                    {
+                        "CLASS": "GENERAL_OBSERVATION",
+                        "LABEL": "GEN_OBS1",
+                        "DATA": "SOME_FIELD",
+                        "INDEX_LIST": "0,3,9",
+                        "RESTART": 20,
+                        "OBS_FILE": "some_file.txt",
+                    },
+                ]
+            ),
+            [
+                {
+                    "class": "GENERAL_OBSERVATION",
+                    "label": "GEN_OBS1",
+                    "data": "SOME_FIELD",
+                    "index_list": "0,3,9",
+                    "restart": 20,
+                    "obs_file": "some_file.txt",
+                }
+            ],
+        ),
+        (
+            pd.DataFrame(
+                [
+                    {
+                        "CLASS": "GENERAL_OBSERVATION",
+                        "LABEL": "GEN_OBS1",
+                    },
+                    {
+                        "CLASS": "GENERAL_OBSERVATION",
+                        "LABEL": "GEN_OBS2",
+                    },
+                ]
+            ),
+            [
+                {
+                    "class": "GENERAL_OBSERVATION",
+                    "label": "GEN_OBS1",
+                },
+                {
+                    "class": "GENERAL_OBSERVATION",
+                    "label": "GEN_OBS2",
+                },
+            ],
+        ),
+    ],
+)
+def test_general_df2obsdict(general_df, expected_listofdict):
+    """Test converting from dataframe representation for GENERAL observations
+    to the dictionary representation designed for yaml output"""
+    assert general_df2obsdict(general_df) == expected_listofdict
+
+
 @pytest.mark.parametrize(
     "dframe, expected_dframe",
     [

tests/testdata_fmuobs/ert-doc.yml

@@ -63,3 +63,13 @@ smry:
     error: 10.0
     label: SEP_TEST_2008
     value: 213.0
+general:
+ - label: GEN_OBS1
+   data: SOME_FIELD
+   restart: 20.0
+   obs_file: some_file.txt
+ - label: GEN_OBS2
+   data: SOME_FIELD
+   index_list: "0,3,9"
+   restart: 20.0
+   obs_file: some_file.txt