Skip to content

Commit

Permalink
Poll first for exit status before sending running status in fm_model_…
Browse files Browse the repository at this point in the history
…step
  • Loading branch information
larsevj committed Jan 24, 2025
1 parent 1180fec commit 444d6d0
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 18 deletions.
28 changes: 15 additions & 13 deletions src/_ert/forward_model_runner/forward_model_step.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,21 @@ def _run(self) -> Generator[Start | Exited | Running | None]:

max_memory_usage = 0
fm_step_pids = {int(process.pid)}
while exit_code is None:
while True:
try:
exit_code = process.wait(timeout=self.MEMORY_POLL_PERIOD)
if exit_code is not None:
break
except TimeoutExpired:
potential_exited_msg = (
self.handle_process_timeout_and_create_exited_msg(
exit_code, proc, run_start_time
)
)
if isinstance(potential_exited_msg, Exited):
yield potential_exited_msg
return

(memory_rss, cpu_seconds, oom_score, pids) = _get_processtree_data(process)
fm_step_pids |= pids
max_memory_usage = max(memory_rss, max_memory_usage)
Expand All @@ -203,18 +217,6 @@ def _run(self) -> Generator[Start | Exited | Running | None]:
),
)

try:
exit_code = process.wait(timeout=self.MEMORY_POLL_PERIOD)
except TimeoutExpired:
potential_exited_msg = (
self.handle_process_timeout_and_create_exited_msg(
exit_code, proc, run_start_time
)
)
if isinstance(potential_exited_msg, Exited):
yield potential_exited_msg
return

ensure_file_handles_closed([stdin, stdout, stderr])
exited_message = self._create_exited_message_based_on_exit_code(
max_memory_usage, target_file_mtime, exit_code, fm_step_pids
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ def test_run_with_process_failing(mock_process, mock_popen, mock_check_executabl
run = fmstep.run()

assert isinstance(next(run), Start), "run did not yield Start message"
assert isinstance(next(run), Running), "run did not yield Running message"
exited = next(run)
assert isinstance(exited, Exited), "run did not yield Exited message"
assert exited.exit_code == 9, "Exited message had unexpected exit code"
Expand Down Expand Up @@ -209,9 +208,9 @@ def test_run_fails_using_exit_bash_builtin():

statuses = list(fmstep.run())

assert len(statuses) == 3, "Wrong statuses count"
assert statuses[2].exit_code == 1, "Exited status wrong exit_code"
assert statuses[2].error_message == "Process exited with status code 1", (
assert len(statuses) == 2, "Wrong statuses count"
assert statuses[1].exit_code == 1, "Exited status wrong exit_code"
assert statuses[1].error_message == "Process exited with status code 1", (
"Exited status wrong error_message"
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ def test_terminate_steps():
os.wait() # allow os to clean up zombie processes


@pytest.mark.integration_test
@pytest.mark.usefixtures("use_tmpdir")
def test_memory_profile_is_logged_as_csv():
"""This tests that a csv is produced and has basic validity.
Expand All @@ -126,7 +127,7 @@ def test_memory_profile_is_logged_as_csv():
with open(scriptname, "w", encoding="utf-8") as script:
script.write(
"""#!/bin/sh
exit 0
sleep 6
"""
)
os.chmod(scriptname, stat.S_IRWXU | stat.S_IRWXO | stat.S_IRWXG)
Expand Down

0 comments on commit 444d6d0

Please sign in to comment.