diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index cadc4f9be0..1b75b43dc7 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -826,11 +826,11 @@ def complete_forest() -> bool: "resource process birth changed", ) observed[pid] = before - need( - self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], - "process RSS", - ) try: + need( + self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], + "process RSS", + ) children = self.owned.children(pid) except (FileNotFoundError, ProcessLookupError): need( diff --git a/scripts/gates/testsupport/cell_trigger/owned_children.py b/scripts/gates/testsupport/cell_trigger/owned_children.py index b0c5d950f8..afafd2ece4 100644 --- a/scripts/gates/testsupport/cell_trigger/owned_children.py +++ b/scripts/gates/testsupport/cell_trigger/owned_children.py @@ -38,10 +38,8 @@ def children(pid: int) -> list[int]: @staticmethod def rss(pid: int) -> int: - try: - status = Path("/proc", str(pid), "status").read_text() - except FileNotFoundError: - return 0 # Vanished sample, never proof that the process used no memory. + # The forest owner must verify disappearance after a failed status read. + status = Path("/proc", str(pid), "status").read_text() return next( (int(line.split()[1]) for line in status.splitlines() if line.startswith("VmRSS:")), 0 ) diff --git a/tests/unit/test_cell_trigger_resource_census.py b/tests/unit/test_cell_trigger_resource_census.py index fb3cf02058..75778534b7 100644 --- a/tests/unit/test_cell_trigger_resource_census.py +++ b/tests/unit/test_cell_trigger_resource_census.py @@ -7,6 +7,7 @@ import pytest from scripts.gates.testsupport.cell_trigger import fixture +from scripts.gates.testsupport.cell_trigger.owned_children import OwnedChildren @pytest.fixture @@ -82,6 +83,145 @@ def read(path, *args, **kwargs): return owner, owned, clock +@pytest.fixture +def rss_census(census, monkeypatch): + owner, owned, clock = census + original_read, original_identity = Path.read_text, owned.identify + gone, failures = set(), {} + + def read(path, *args, **kwargs): + if path.parent.parent == Path("/proc") and path.name == "status": + pid = int(path.parent.name) + if pid in failures: + return failures[pid]() + return "Name:\tfixture\nVmRSS:\t1024 kB\n" + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read) + owned.identify = lambda pid: None if pid in gone else original_identity(pid) + owned.measure = OwnedChildren.rss + return owner, owned, clock, gone, failures + + +@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError]) +def test_a_vanished_rss_read_requires_a_new_complete_forest(rss_census, error): + owner, owned, _, gone, failures = rss_census + + def vanish(): + gone.add(300) + raise error("gone during RSS read") + + failures[300] = vanish + owner.sample() + assert owner.first is None + assert owner.sampled == 10.0 + assert owned.round == 2 + assert owned.sampled_pids.count(200) == 2 + assert 400 in owned.sampled_pids + + +@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError]) +def test_a_live_process_with_unreadable_rss_never_refreshes_the_sample(rss_census, error): + owner, owned, _, _, failures = rss_census + + def unreadable(): + raise error("RSS unavailable while identity remains present") + + failures[300] = unreadable + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + assert owned.round == 1 + assert 400 not in owned.sampled_pids + + +def test_a_readable_rss_forest_keeps_the_existing_admission_path(rss_census): + owner, owned, _, _, _ = rss_census + owner.sample() + assert owner.first is None + assert owner.sampled == 10.0 + assert owned.round == 1 + assert set(owned.sampled_pids) == {100, 200, 300} + + +@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError]) +@pytest.mark.parametrize("pid", [100, 200]) +def test_rss_root_loss_does_not_retry_or_refresh(rss_census, error, pid): + owner, owned, _, gone, failures = rss_census + + def vanish(): + gone.add(pid) + raise error("root disappeared during RSS read") + + failures[pid] = vanish + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + assert owned.round <= 1 + + +@pytest.mark.parametrize("error", [PermissionError, OSError, ValueError]) +def test_other_rss_read_errors_remain_refusals(rss_census, error): + owner, owned, _, _, failures = rss_census + + def unreadable(): + raise error("RSS read failed without verified disappearance") + + failures[300] = unreadable + owner.sample() + assert isinstance(owner.first, error) + assert owner.sampled == 7.0 + assert owned.round == 1 + + +def test_new_forest_still_checks_the_next_child_rss(rss_census): + owner, owned, _, gone, failures = rss_census + + def vanish(): + gone.add(300) + raise ProcessLookupError("gone during RSS read") + + failures[300] = vanish + failures[400] = lambda: f"VmRSS:\t{fixture.POLICY['stop_process_rss_bytes'] // 1024 + 1} kB\n" + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 2 + assert 400 in owned.sampled_pids + + +def test_rss_disappearance_does_not_reset_the_census_deadline(rss_census): + owner, owned, clock, gone, failures = rss_census + + def vanish(): + gone.add(300) + clock.now = 13.0 + raise ProcessLookupError("gone at the census deadline") + + failures[300] = vanish + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 1 + assert 400 not in owned.sampled_pids + + +def test_repeated_rss_disappearance_keeps_the_three_forest_bound(rss_census): + owner, owned, _, gone, failures = rss_census + owned.trees = [[300], [400], [500], [600]] + + def vanish(pid): + gone.add(pid) + raise ProcessLookupError("another descendant vanished") + + failures.update({pid: lambda pid=pid: vanish(pid) for pid in (300, 400, 500)}) + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 3 + assert 600 not in owned.sampled_pids + + @pytest.mark.parametrize("stage", ["before", "after", "children"]) def test_a_vanished_descendant_restarts_the_whole_forest(census, stage): owner, owned, _ = census