From 33f2c3db01460ddc3092d9e7e9addede77162bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Thu, 16 Jan 2025 09:00:53 +0100 Subject: [PATCH] Fix bug where queue settings were not taken from site config --- src/everest/config/everest_config.py | 8 ++++-- src/everest/config/simulator_config.py | 12 ++++++--- tests/everest/test_detached.py | 31 ++++++++++++++++++++++-- tests/everest/test_res_initialization.py | 26 ++++++++++++++++++++ 4 files changed, 69 insertions(+), 8 deletions(-) diff --git a/src/everest/config/everest_config.py b/src/everest/config/everest_config.py index e8473a9f90e..34c7669f907 100644 --- a/src/everest/config/everest_config.py +++ b/src/everest/config/everest_config.py @@ -25,9 +25,10 @@ field_validator, model_validator, ) +from pydantic_core.core_schema import ValidationInfo from ruamel.yaml import YAML, YAMLError -from ert.config import ErtConfig +from ert.config import ErtConfig, QueueConfig from ert.config.parsing import BaseModelWithContextSupport from ert.config.parsing.base_model_context import init_context from ert.plugins import ErtPluginManager @@ -270,7 +271,7 @@ def validate_queue_system(self) -> Self: # pylint: disable=E0213 return self @model_validator(mode="after") - def validate_forward_model_job_name_installed(self) -> Self: # pylint: disable=E0213 + def validate_forward_model_job_name_installed(self, info: ValidationInfo) -> Self: # pylint: disable=E0213 install_jobs = self.install_jobs forward_model_jobs = self.forward_model if install_jobs is None: @@ -814,6 +815,9 @@ def load_file(config_file: str) -> "EverestConfig": def with_plugins(cls, config_dict): context = {} activate_script = ErtPluginManager().activate_script() + site_config = ErtConfig.read_site_config() + if site_config: + context["queue_system"] = QueueConfig.from_dict(site_config).queue_options if activate_script: context["activate_script"] = ErtPluginManager().activate_script() with init_context(context): diff --git a/src/everest/config/simulator_config.py b/src/everest/config/simulator_config.py index 1af76a8c471..776a1f3ff9c 100644 --- a/src/everest/config/simulator_config.py +++ b/src/everest/config/simulator_config.py @@ -1,14 +1,15 @@ from typing import Any from pydantic import ( - BaseModel, Field, NonNegativeInt, PositiveInt, field_validator, model_validator, ) +from pydantic_core.core_schema import ValidationInfo +from ert.config.parsing import BaseModelWithContextSupport from ert.config.queue_config import ( LocalQueueOptions, LsfQueueOptions, @@ -32,7 +33,7 @@ def check_removed_config(queue_system): ) -class SimulatorConfig(BaseModel, extra="forbid"): # type: ignore +class SimulatorConfig(BaseModelWithContextSupport, extra="forbid"): # type: ignore cores_per_node: PositiveInt | None = Field( default=None, description="""defines the number of CPUs when running @@ -93,9 +94,12 @@ class SimulatorConfig(BaseModel, extra="forbid"): # type: ignore @field_validator("queue_system", mode="before") @classmethod - def default_local_queue(cls, v): + def default_local_queue(cls, v, info: ValidationInfo): if v is None: - return LocalQueueOptions(max_running=8) + options = None + if info.context: + options = info.context.get(info.field_name) + return options or LocalQueueOptions(max_running=8) return v @model_validator(mode="before") diff --git a/tests/everest/test_detached.py b/tests/everest/test_detached.py index 91597894498..0d8f2616aa5 100644 --- a/tests/everest/test_detached.py +++ b/tests/everest/test_detached.py @@ -7,7 +7,7 @@ import requests import everest -from ert.config import ErtConfig +from ert.config import ErtConfig, QueueSystem from ert.config.queue_config import ( LocalQueueOptions, LsfQueueOptions, @@ -321,7 +321,34 @@ def test_queue_options_site_config(queue_options, use_plugin, monkeypatch, min_c config = EverestConfig.with_plugins( {"simulator": {"queue_system": queue_options}} | min_config ) - assert config.server.queue_system.activate_script == expected_result + assert config.simulator.queue_system.activate_script == expected_result + + +@pytest.mark.parametrize("use_plugin", (True, False)) +@pytest.mark.parametrize( + "queue_options", + [ + {"queue_system": {"name": "slurm"}}, + {}, + ], +) +def test_simulator_queue_system_site_config( + queue_options, use_plugin, monkeypatch, min_config +): + if queue_options: + expected_result = SlurmQueueOptions # User specified + elif use_plugin: + expected_result = LsfQueueOptions # Mock site config + else: + expected_result = LocalQueueOptions # Default value + if use_plugin: + monkeypatch.setattr( + everest.config.everest_config.ErtConfig, + "read_site_config", + MagicMock(return_value={"QUEUE_SYSTEM": QueueSystem.LSF}), + ) + config = EverestConfig.with_plugins({"simulator": queue_options} | min_config) + assert isinstance(config.simulator.queue_system, expected_result) @pytest.mark.timeout(5) # Simulation might not finish diff --git a/tests/everest/test_res_initialization.py b/tests/everest/test_res_initialization.py index 27fc2e5db81..e08c60c4e68 100644 --- a/tests/everest/test_res_initialization.py +++ b/tests/everest/test_res_initialization.py @@ -369,3 +369,29 @@ def test_user_config_jobs_precedence(tmp_path, monkeypatch): .executable == "echo" ) + + +def test_that_queue_settings_are_taken_from_site_config( + min_config, monkeypatch, tmp_path +): + monkeypatch.chdir(tmp_path) + assert "simulator" not in min_config # Double check + Path("site-config").write_text( + dedent(""" + QUEUE_SYSTEM LSF + QUEUE_OPTION LSF LSF_RESOURCE my_resource + QUEUE_OPTION LSF LSF_QUEUE my_queue + """), + encoding="utf-8", + ) + with open("config.yml", "w", encoding="utf-8") as f: + yaml.dump(min_config, f) + monkeypatch.setenv("ERT_SITE_CONFIG", "site-config") + config = EverestConfig.load_file("config.yml") + assert config.simulator.queue_system == LsfQueueOptions( + lsf_queue="my_queue", lsf_resource="my_resource" + ) + ert_config = everest_to_ert_config(config) + assert ert_config.queue_config.queue_options == LsfQueueOptions( + lsf_queue="my_queue", lsf_resource="my_resource" + )