Remove duplicate calls to load_smry (#505)

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+### Fixed
+- [#505](https://github.com/equinor/webviz-subsurface/pull/505) - Fixed recent performance regression issue for loading of UNSMRY data. Loading times when multiple plugins are using the same data is now significantly reduced. Note that all UNSMRY vectors are now stored in portable apps, independent of choice of column_keys in individual plugins.
+
 ## [0.1.5] - 2020-11-26
 ### Added
 - [#478](https://github.com/equinor/webviz-subsurface/pull/478) - New plugin `AssistedHistoryMatchingAnalysis`. This dashboard helps to analyze the update step performed during assisted history match. E.g. which observations are causing an update in a specific parameter. Based on Kolmogorov–Smirnov.

tests/conftest.py

@@ -1,6 +1,8 @@
 import pathlib
 
 import pytest
+import dash
+from webviz_config.common_cache import CACHE
 
 
 def pytest_addoption(parser):
@@ -15,3 +17,13 @@ def pytest_addoption(parser):
 @pytest.fixture
 def testdata_folder(request):
     return request.config.getoption("--testdata-folder")
+
+
+@pytest.fixture()
+def app():
+    dash_app = dash.Dash(__name__)
+    dash_app.css.config.serve_locally = True
+    dash_app.scripts.config.serve_locally = True
+    dash_app.config.suppress_callback_exceptions = True
+    CACHE.init_app(dash_app.server)
+    yield dash_app

tests/unit_tests/model_tests/test_ensemble_model.py

@@ -0,0 +1,187 @@
+from pathlib import Path
+import datetime
+
+import pytest
+import numpy as np
+
+from webviz_subsurface._models.ensemble_model import EnsembleModel
+
+
+@pytest.mark.usefixtures("app")
+def test_ensemble_set_init(testdata_folder):
+
+    emodel = EnsembleModel(
+        ensemble_name="iter-0",
+        ensemble_path=Path(testdata_folder)
+        / "reek_history_match"
+        / "realization-*"
+        / "iter-0",
+    )
+    assert emodel.ens_folder == {"iter-0": f"{testdata_folder}/reek_history_match/"}
+    assert emodel.webviz_store == []
+
+
+@pytest.mark.usefixtures("app")
+def test_bad_ensemble_path():
+    emodel = EnsembleModel(ensemble_name="iter-0", ensemble_path="some_path")
+    with pytest.raises(ValueError) as exception:
+        emodel.load_ensemble()
+    assert (
+        exception.value.args[0]
+        == "No realizations found for ensemble iter-0, located at 'some_path'. Aborting..."
+    )
+
+
+@pytest.mark.usefixtures("app")
+def test_smry_load(testdata_folder):
+
+    emodel = EnsembleModel(
+        ensemble_name="iter-0",
+        ensemble_path=Path(testdata_folder)
+        / "reek_history_match"
+        / "realization-*"
+        / "iter-0",
+    )
+    smry = emodel.load_smry()
+    assert len(smry.columns) == 475
+    assert len(smry["DATE"].unique()) == 480
+
+
+@pytest.mark.usefixtures("app")
+def test_smry_load_filter_and_dtypes(testdata_folder):
+
+    emodel = EnsembleModel(
+        ensemble_name="iter-0",
+        ensemble_path=Path(testdata_folder)
+        / "reek_history_match"
+        / "realization-*"
+        / "iter-0",
+    )
+    smry = emodel.load_smry(column_keys=["FO*"], time_index="yearly")
+    assert set(smry.columns) == set(
+        [
+            "DATE",
+            "REAL",
+            "FOPRF",
+            "FOPRS",
+            "FOPRH",
+            "FOPTH",
+            "FOPR",
+            "FOPTS",
+            "FOPTF",
+            "FOPT",
+            "FOIP",
+            "FOPP",
+        ]
+    )
+    assert set(smry["DATE"].unique()) == set(
+        [
+            datetime.date(2000, 1, 1),
+            datetime.date(2001, 1, 1),
+            datetime.date(2002, 1, 1),
+            datetime.date(2003, 1, 1),
+            datetime.date(2004, 1, 1),
+        ]
+    )
+    assert smry["DATE"].dtype == np.dtype("O")
+    assert smry["REAL"].dtype == np.dtype("int64")
+    assert all(
+        np.issubdtype(dtype, np.number)
+        for dtype in smry.drop(["REAL", "DATE"], axis=1).dtypes
+    )
+    smry = emodel.load_smry(
+        column_keys=["F[OGW]P?", "WWCT:OP*", "FOIP"], time_index="yearly"
+    )
+    assert set(smry.columns) == set(
+        [
+            "REAL",
+            "DATE",
+            "FGPP",
+            "FGPR",
+            "FGPT",
+            "FOPP",
+            "FOPR",
+            "FOPT",
+            "FWPP",
+            "FWPR",
+            "FWPT",
+            "WWCT:OP_1",
+            "WWCT:OP_2",
+            "WWCT:OP_3",
+            "WWCT:OP_4",
+            "WWCT:OP_5",
+            "FOIP",
+        ]
+    )
+
+
+@pytest.mark.usefixtures("app")
+def test_smry_meta(testdata_folder):
+    emodel = EnsembleModel(
+        ensemble_name="iter-0",
+        ensemble_path=Path(testdata_folder)
+        / "reek_history_match"
+        / "realization-*"
+        / "iter-0",
+    )
+    smeta = emodel.load_smry_meta()
+    assert set(smeta.columns) == set(
+        ["unit", "is_total", "is_rate", "is_historical", "keyword", "wgname", "get_num"]
+    )
+    assert len(smeta) == 473
+    assert "FOPT" in smeta.index
+
+
+@pytest.mark.usefixtures("app")
+def test_parameter_loading(testdata_folder):
+    emodel = EnsembleModel(
+        ensemble_name="iter-0",
+        ensemble_path=Path(testdata_folder)
+        / "reek_history_match"
+        / "realization-*"
+        / "iter-0",
+    )
+    parameters = emodel.load_parameters()
+    assert "REAL" in parameters.columns
+    assert parameters["REAL"].dtype == np.dtype("int64")
+    assert len(parameters.columns) == 27
+
+
+@pytest.mark.usefixtures("app")
+def test_load_csv(testdata_folder):
+    emodel = EnsembleModel(
+        ensemble_name="iter-0",
+        ensemble_path=Path(testdata_folder)
+        / "reek_history_match"
+        / "realization-*"
+        / "iter-0",
+    )
+    dframe = emodel.load_csv(Path("share") / "results" / "tables" / "rft.csv")
+    assert "REAL" in dframe.columns
+    assert dframe["REAL"].dtype == np.dtype("int64")
+    assert len(dframe.columns) == 14
+
+
+@pytest.mark.usefixtures("app")
+def test_webviz_store(testdata_folder):
+    emodel = EnsembleModel(
+        ensemble_name="iter-0",
+        ensemble_path=Path(testdata_folder)
+        / "reek_history_match"
+        / "realization-*"
+        / "iter-0",
+    )
+    emodel.load_parameters()
+    assert len(emodel.webviz_store) == 1
+    emodel.load_smry()
+    assert len(emodel.webviz_store) == 2
+    emodel.load_smry(column_keys=["FOIP"])
+    assert len(emodel.webviz_store) == 3
+    emodel.load_smry(time_index="raw")
+    assert len(emodel.webviz_store) == 4
+    emodel.load_smry_meta()
+    assert len(emodel.webviz_store) == 5
+    emodel.load_smry_meta(column_keys=["R*", "GW?T*"])
+    assert len(emodel.webviz_store) == 6
+    emodel.load_csv(Path("share") / "results" / "tables" / "rft.csv")
+    assert len(emodel.webviz_store) == 7

tests/unit_tests/model_tests/test_ensemble_set_model.py

@@ -0,0 +1,142 @@
+from pathlib import Path
+
+import pytest
+import numpy as np
+
+from webviz_subsurface._models.ensemble_set_model import EnsembleSetModel
+
+
+@pytest.mark.usefixtures("app")
+def test_single_ensemble(testdata_folder):
+
+    emodel = EnsembleSetModel(
+        ensemble_paths={
+            "iter-0": str(
+                Path(
+                    testdata_folder / "reek_history_match" / "realization-*" / "iter-0"
+                )
+            )
+        }
+    )
+    assert emodel.ens_folders == {
+        "iter-0": Path("webviz-subsurface-testdata") / "reek_history_match"
+    }
+    assert len(emodel.ensembles) == 1
+    smry = emodel.load_smry()
+    assert len(smry.columns) == 476
+    assert len(smry["DATE"].unique()) == 480
+    assert smry["ENSEMBLE"].unique() == ["iter-0"]
+    assert smry["ENSEMBLE"].dtype == np.dtype("O")
+    assert all(
+        np.issubdtype(dtype, np.number)
+        for dtype in smry.drop(["REAL", "ENSEMBLE", "DATE"], axis=1).dtypes
+    )
+
+    parameters = emodel.load_parameters()
+    assert all(col in parameters.columns for col in ["ENSEMBLE", "REAL"])
+    assert parameters["ENSEMBLE"].dtype == np.dtype("O")
+    assert parameters["REAL"].dtype == np.dtype("int64")
+    assert len(parameters.columns) == 28
+
+
+@pytest.mark.usefixtures("app")
+def test_smry_load_multiple_ensembles(testdata_folder):
+
+    emodel = EnsembleSetModel(
+        ensemble_paths={
+            "iter-0": str(
+                Path(
+                    testdata_folder / "reek_history_match" / "realization-*" / "iter-0"
+                )
+            ),
+            "iter-1": str(
+                Path(
+                    testdata_folder / "reek_history_match" / "realization-*" / "iter-1"
+                )
+            ),
+            "iter-2": str(
+                Path(
+                    testdata_folder / "reek_history_match" / "realization-*" / "iter-2"
+                )
+            ),
+            "iter-3": str(
+                Path(
+                    testdata_folder / "reek_history_match" / "realization-*" / "iter-3"
+                )
+            ),
+        }
+    )
+    smry = emodel.load_smry()
+    assert len(smry.columns) == 476
+    assert len(smry["DATE"].unique()) == 1141
+    assert set(smry["ENSEMBLE"].unique()) == set(
+        ["iter-0", "iter-1", "iter-2", "iter-3"]
+    )
+    assert smry["ENSEMBLE"].dtype == np.dtype("O")
+    # assert smry["DATE"].dtype == np.dtype("O") # Fails due to wrong input data?
+    assert smry["REAL"].dtype == np.dtype("int64")
+    assert all(
+        np.issubdtype(dtype, np.number)
+        for dtype in smry.drop(["REAL", "ENSEMBLE", "DATE"], axis=1).dtypes
+    )
+    smeta = emodel.load_smry_meta()
+    assert set(smeta.columns) == set(
+        ["unit", "is_total", "is_rate", "is_historical", "keyword", "wgname", "get_num"]
+    )
+    assert len(smeta) == 473
+    assert "FOPT" in smeta.index
+
+    parameters = emodel.load_parameters()
+    assert all(col in parameters.columns for col in ["ENSEMBLE", "REAL"])
+    assert parameters["ENSEMBLE"].dtype == np.dtype("O")
+    assert parameters["REAL"].dtype == np.dtype("int64")
+    assert len(parameters.columns) == 28
+
+    dframe = emodel.load_csv(Path("share") / "results" / "tables" / "rft.csv")
+    assert "ENSEMBLE" in dframe.columns
+    assert dframe["ENSEMBLE"].dtype == np.dtype("O")
+    assert len(dframe["ENSEMBLE"].unique()) == 4
+    assert len(dframe.columns) == 15
+
+    with pytest.raises(KeyError) as exc:
+        emodel.load_csv("some_path")
+    assert (
+        exc.value.args[0]
+        == "No data found for load_csv with arguments: {'csv_file': 'some_path'}"
+    )
+
+
+@pytest.mark.usefixtures("app")
+def test_webvizstore(testdata_folder):
+    emodel = EnsembleSetModel(
+        ensemble_paths={
+            "iter-0": str(
+                Path(
+                    testdata_folder / "reek_history_match" / "realization-*" / "iter-0"
+                )
+            ),
+            "iter-1": str(
+                Path(
+                    testdata_folder / "reek_history_match" / "realization-*" / "iter-1"
+                )
+            ),
+            "iter-2": str(
+                Path(
+                    testdata_folder / "reek_history_match" / "realization-*" / "iter-2"
+                )
+            ),
+            "iter-3": str(
+                Path(
+                    testdata_folder / "reek_history_match" / "realization-*" / "iter-3"
+                )
+            ),
+        }
+    )
+    emodel.load_parameters()
+    assert len(emodel.webvizstore) == 4
+    emodel.load_smry()
+    assert len(emodel.webvizstore) == 8
+    emodel.load_smry_meta()
+    assert len(emodel.webvizstore) == 12
+    emodel.load_csv(Path("share") / "results" / "tables" / "rft.csv")
+    assert len(emodel.webvizstore) == 16

webviz_subsurface/_models/__init__.py

@@ -0,0 +1,2 @@
+from .ensemble_model import EnsembleModel
+from .ensemble_set_model import EnsembleSetModel