fix: stop reporting crashed cases as RUNNING forever

CHANGELOG.md

@@ -13,6 +13,7 @@ Make the dashboard follow what each solver can actually do: panels and action bu
 - `csauto doctor` reports the panels and capabilities derived for the configured solver
 
 ### Fixed
+- A case launched from the web UI stayed `RUNNING` forever when the run crashed before the solver started (a missing mesh, say). Two causes, both fixed: `is_process_alive` reported a zombie as alive, because `os.kill(pid, 0)` succeeds on a process that exited but was never reaped, which is what every run launched by the long-lived server becomes; and `detect_run_outcome` only read `run_solver.log` and `listing`, so a failure that produces neither went undetected even though code_saturne writes an explicit `run_status.failed` marker next to them. Zombies are now reported as dead (and reaped when we are the parent), and the status markers are read when the logs give no verdict
 - Opening the solver GUI on a case whose shared dirs are symlinks (the default since `mesh_mode = "symlink"` became the default in 0.4.1) left those symlinks dangling inside the container: `build_gui_command` (docker) and the singularity branch of `build_runtime_gui_command` mounted only the runs dir, unlike their `run` counterparts which also bind the symlink targets. Both now bind them the same way, `MESH` read-only and `POST` writable
 - Requesting a restart on a solver without restart support returned HTTP 500 "Launch error", a client error reported as a server fault; it now returns HTTP 400 naming the solver
 - Live control on a solver declaring no control action reported "Invalid action (expected one of [])"; both the API and the CLI now name the solver

csauto/runner.py

@@ -691,8 +691,29 @@ def _start_case_with_launch_slot(
         start_case()
 
 
+def _is_zombie(pid: int) -> bool:
+    """True when the process has exited but its parent has not reaped it yet.
+
+    Without procfs (non-Linux), report False and let the os.kill() answer stand.
+    """
+    try:
+        stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8", errors="ignore")
+    except OSError:
+        return False
+    # The comm field is parenthesized and may itself contain spaces or
+    # parentheses, so the state is the first token after the last ")".
+    fields = stat.rpartition(")")[2].split()
+    return bool(fields) and fields[0] == "Z"
+
+
 def is_process_alive(pid: int) -> bool:
-    """Check if a PID is alive (best effort, POSIX-oriented)."""
+    """Check if a PID is alive (best effort, POSIX-oriented).
+
+    A zombie counts as dead. `os.kill(pid, 0)` still succeeds for a process that
+    exited but was never reaped, which is what every run launched by the
+    long-lived web server becomes: `run_cases` spawns it with Popen and never
+    waits. Treating that as alive kept finished cases RUNNING forever.
+    """
     try:
         os.kill(pid, 0)
     except ProcessLookupError:
@@ -701,6 +722,12 @@ def is_process_alive(pid: int) -> bool:
         return True
     except OSError:
         return False
+    if _is_zombie(pid):
+        # Reap it when we are the parent, so the entry stops lingering in the
+        # process table for the lifetime of the server.
+        with contextlib.suppress(ChildProcessError, OSError):
+            os.waitpid(pid, os.WNOHANG)
+        return False
     return True
 
 

csauto/solvers/code_saturne.py

@@ -18,6 +18,8 @@
 )
 from ..logs import (
     ANOMALY_FILES_DEFAULT,
+    _is_recent,
+    _parse_start_time,
     detect_run_outcome,
     extract_last_iteration,
     extract_restart_origin,
@@ -29,6 +31,7 @@
     parse_performance_log,
 )
 from ..probes import list_probe_files, list_profile_files, locate_probe_files
+from ..registry import STATUS_FAILED
 from ..residuals import find_residuals_files, parse_residuals_from_log
 from ..template import find_run_cfg, find_setup_file
 from .base import CompareKind, PerfColumn, SolverAdapterBase
@@ -40,6 +43,12 @@
 CHECKPOINT_STATE_RE = re.compile(r"Checkpoint at iteration\s+(?P<iter>\d+),\s+physical time\s+(?P<time>[-+0-9.eE]+)")
 
 CONTROL_FILENAME = "control_file"
+# Status markers code_saturne leaves in RESU/<run>/ when a run stops abnormally.
+# Its own cs_case.py maps run_status.<name> to a case state; only these two mean
+# failure (FAILED and EXCEEDED_TIME_LIMIT). The others it writes are progress
+# states (preparing, prepared, preprocessing, ready, running, saving, finished),
+# and a normal completion removes the marker altogether.
+RUN_STATUS_FAILURE_NAMES = ("run_status.failed", "run_status.exceeded_time_limit")
 # code_saturne prints its configured iteration limit once at startup, into setup.log:
 #   "      nt_max:  500 (final time step)\n" (cs_time_step_log_setup, src/base/cs_time_step.cpp)
 NT_MAX_SETUP_RE = re.compile(r"nt_max:\s*(-?\d+)")
@@ -337,7 +346,20 @@ def _extract_restart_checkpoint_state(self, case_dir: Path, run_id: str) -> tupl
         return None, None
 
     def detect_outcome(self, case_dir: Path, start_time: str | None = None) -> str | None:
-        return detect_run_outcome(case_dir, start_time)
+        outcome = detect_run_outcome(case_dir, start_time)
+        if outcome:
+            return outcome
+        # The logs gave no verdict. A run that fails before the solver starts (a
+        # missing mesh, say) writes no run_solver.log at all, only a status
+        # marker beside it. Restricted to the current run, so a marker left by a
+        # previous run never overrides the log of this one.
+        start_ts = _parse_start_time(start_time)
+        for run_dir in self.list_run_dirs(case_dir):
+            for name in RUN_STATUS_FAILURE_NAMES:
+                marker = run_dir / name
+                if marker.is_file() and _is_recent(marker, start_ts):
+                    return STATUS_FAILED
+        return None
 
     def read_progress(self, case_dir: Path, start_time: str | None = None) -> int | None:
         run_status_path = locate_run_status_file(case_dir, start_time)

tests/runner/test_runner.py

@@ -1011,3 +1011,49 @@ def test_refresh_status_include_doe_returns_columns(
     assert "case_id" not in doe_columns
     assert rows[0]["doe"]["density"] == "1.2"
     assert rows[0]["doe"]["velocity"] == "3.5"
+
+
+def _wait_for_zombie(pid: int, timeout: float = 5.0) -> bool:
+    """Poll /proc until the child has exited but is not yet reaped."""
+    import time
+
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        try:
+            stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8", errors="ignore")
+        except OSError:
+            return False
+        if stat.rpartition(")")[2].split()[0] == "Z":
+            return True
+        time.sleep(0.01)
+    return False
+
+
+@pytest.mark.skipif(not Path("/proc/self/stat").is_file(), reason="requires procfs")
+def test_is_process_alive_reports_a_zombie_as_dead() -> None:
+    """The web server never reaps the runs it launches, so a finished run lingers as a zombie.
+
+    os.kill(pid, 0) succeeds on a zombie, which used to keep the case RUNNING forever.
+    """
+    import contextlib
+    import os
+
+    from csauto.runner import is_process_alive
+
+    pid = os.fork()
+    if pid == 0:  # pragma: no cover - child process
+        os._exit(0)
+    try:
+        assert _wait_for_zombie(pid), "child never became a zombie"
+        assert is_process_alive(pid) is False
+    finally:
+        with contextlib.suppress(ChildProcessError, OSError):
+            os.waitpid(pid, 0)
+
+
+def test_is_process_alive_reports_a_live_process_as_alive() -> None:
+    import os
+
+    from csauto.runner import is_process_alive
+
+    assert is_process_alive(os.getpid()) is True

tests/unit/test_solvers.py

@@ -394,3 +394,55 @@ def test_control_implementation_implies_declared_actions() -> None:
         adapter = get_solver_adapter(name)
         if adapter._provides("apply_control"):
             assert adapter.control_actions, f"{name} implements apply_control but declares no actions"
+
+
+def test_code_saturne_detect_outcome_reads_the_run_status_failure_marker(tmp_path) -> None:
+    """A crash before the solver starts (missing mesh) writes no solver log, only run_status.failed."""
+    adapter = get_solver_adapter("code_saturne")
+    case_dir = tmp_path / "case0001"
+    run_dir = case_dir / "RESU" / "20260911-1326"
+    run_dir.mkdir(parents=True)
+    (run_dir / "run_status.failed").write_text("", encoding="utf-8")
+
+    assert adapter.detect_outcome(case_dir) == STATUS_FAILED
+
+
+def test_code_saturne_detect_outcome_reads_the_time_limit_marker(tmp_path) -> None:
+    adapter = get_solver_adapter("code_saturne")
+    case_dir = tmp_path / "case0001"
+    run_dir = case_dir / "RESU" / "20260911-1326"
+    run_dir.mkdir(parents=True)
+    (run_dir / "run_status.exceeded_time_limit").write_text("", encoding="utf-8")
+
+    assert adapter.detect_outcome(case_dir) == STATUS_FAILED
+
+
+def test_code_saturne_detect_outcome_ignores_progress_markers(tmp_path) -> None:
+    """run_status.running and friends are progress states, not failures."""
+    adapter = get_solver_adapter("code_saturne")
+    case_dir = tmp_path / "case0001"
+    run_dir = case_dir / "RESU" / "20260911-1326"
+    run_dir.mkdir(parents=True)
+    for name in ("run_status.running", "run_status.preprocessing", "run_status.saving"):
+        (run_dir / name).write_text("", encoding="utf-8")
+
+    assert adapter.detect_outcome(case_dir) is None
+
+
+def test_code_saturne_detect_outcome_ignores_a_stale_failure_marker(tmp_path) -> None:
+    """A marker left by a previous run must not override the current run's log."""
+    import os
+    import time
+
+    adapter = get_solver_adapter("code_saturne")
+    case_dir = tmp_path / "case0001"
+    old_run = case_dir / "RESU" / "20260101-0000"
+    old_run.mkdir(parents=True)
+    (old_run / "run_status.failed").write_text("", encoding="utf-8")
+    os.utime(old_run / "run_status.failed", (1000, 1000))
+    new_run = case_dir / "RESU" / "20260911-1326"
+    new_run.mkdir(parents=True)
+    (new_run / "run_solver.log").write_text("END OF CALCULATION\n", encoding="utf-8")
+    start_time = datetime.fromtimestamp(time.time() - 60).isoformat(timespec="seconds")
+
+    assert adapter.detect_outcome(case_dir, start_time) == STATUS_DONE