Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions scripts/gates/testsupport/cell_trigger/fixture.py
Original file line number Diff line number Diff line change
Expand Up @@ -826,11 +826,11 @@ def complete_forest() -> bool:
"resource process birth changed",
)
observed[pid] = before
need(
self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"],
"process RSS",
)
try:
need(
self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"],
"process RSS",
)
children = self.owned.children(pid)
except (FileNotFoundError, ProcessLookupError):
need(
Expand Down
6 changes: 2 additions & 4 deletions scripts/gates/testsupport/cell_trigger/owned_children.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,8 @@ def children(pid: int) -> list[int]:

@staticmethod
def rss(pid: int) -> int:
try:
status = Path("/proc", str(pid), "status").read_text()
except FileNotFoundError:
return 0 # Vanished sample, never proof that the process used no memory.
# The forest owner must verify disappearance after a failed status read.
status = Path("/proc", str(pid), "status").read_text()
return next(
(int(line.split()[1]) for line in status.splitlines() if line.startswith("VmRSS:")), 0
)
Expand Down
140 changes: 140 additions & 0 deletions tests/unit/test_cell_trigger_resource_census.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import pytest

from scripts.gates.testsupport.cell_trigger import fixture
from scripts.gates.testsupport.cell_trigger.owned_children import OwnedChildren


@pytest.fixture
Expand Down Expand Up @@ -82,6 +83,145 @@ def read(path, *args, **kwargs):
return owner, owned, clock


@pytest.fixture
def rss_census(census, monkeypatch):
owner, owned, clock = census
original_read, original_identity = Path.read_text, owned.identify
gone, failures = set(), {}

def read(path, *args, **kwargs):
if path.parent.parent == Path("/proc") and path.name == "status":
pid = int(path.parent.name)
if pid in failures:
return failures[pid]()
return "Name:\tfixture\nVmRSS:\t1024 kB\n"
return original_read(path, *args, **kwargs)

monkeypatch.setattr(Path, "read_text", read)
owned.identify = lambda pid: None if pid in gone else original_identity(pid)
owned.measure = OwnedChildren.rss
return owner, owned, clock, gone, failures


@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError])
def test_a_vanished_rss_read_requires_a_new_complete_forest(rss_census, error):
owner, owned, _, gone, failures = rss_census

def vanish():
gone.add(300)
raise error("gone during RSS read")

failures[300] = vanish
owner.sample()
assert owner.first is None
assert owner.sampled == 10.0
assert owned.round == 2
assert owned.sampled_pids.count(200) == 2
assert 400 in owned.sampled_pids


@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError])
def test_a_live_process_with_unreadable_rss_never_refreshes_the_sample(rss_census, error):
owner, owned, _, _, failures = rss_census

def unreadable():
raise error("RSS unavailable while identity remains present")

failures[300] = unreadable
owner.sample()
assert owner.first is not None
assert owner.sampled == 7.0
assert owned.round == 1
assert 400 not in owned.sampled_pids


def test_a_readable_rss_forest_keeps_the_existing_admission_path(rss_census):
owner, owned, _, _, _ = rss_census
owner.sample()
assert owner.first is None
assert owner.sampled == 10.0
assert owned.round == 1
assert set(owned.sampled_pids) == {100, 200, 300}


@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError])
@pytest.mark.parametrize("pid", [100, 200])
def test_rss_root_loss_does_not_retry_or_refresh(rss_census, error, pid):
owner, owned, _, gone, failures = rss_census

def vanish():
gone.add(pid)
raise error("root disappeared during RSS read")

failures[pid] = vanish
owner.sample()
assert owner.first is not None
assert owner.sampled == 7.0
assert owned.round <= 1


@pytest.mark.parametrize("error", [PermissionError, OSError, ValueError])
def test_other_rss_read_errors_remain_refusals(rss_census, error):
owner, owned, _, _, failures = rss_census

def unreadable():
raise error("RSS read failed without verified disappearance")

failures[300] = unreadable
owner.sample()
assert isinstance(owner.first, error)
assert owner.sampled == 7.0
assert owned.round == 1


def test_new_forest_still_checks_the_next_child_rss(rss_census):
owner, owned, _, gone, failures = rss_census

def vanish():
gone.add(300)
raise ProcessLookupError("gone during RSS read")

failures[300] = vanish
failures[400] = lambda: f"VmRSS:\t{fixture.POLICY['stop_process_rss_bytes'] // 1024 + 1} kB\n"
owner.sample()
assert isinstance(owner.first, fixture.Refused)
assert owner.sampled == 7.0
assert owned.round == 2
assert 400 in owned.sampled_pids


def test_rss_disappearance_does_not_reset_the_census_deadline(rss_census):
owner, owned, clock, gone, failures = rss_census

def vanish():
gone.add(300)
clock.now = 13.0
raise ProcessLookupError("gone at the census deadline")

failures[300] = vanish
owner.sample()
assert isinstance(owner.first, fixture.Refused)
assert owner.sampled == 7.0
assert owned.round == 1
assert 400 not in owned.sampled_pids


def test_repeated_rss_disappearance_keeps_the_three_forest_bound(rss_census):
owner, owned, _, gone, failures = rss_census
owned.trees = [[300], [400], [500], [600]]

def vanish(pid):
gone.add(pid)
raise ProcessLookupError("another descendant vanished")

failures.update({pid: lambda pid=pid: vanish(pid) for pid in (300, 400, 500)})
owner.sample()
assert isinstance(owner.first, fixture.Refused)
assert owner.sampled == 7.0
assert owned.round == 3
assert 600 not in owned.sampled_pids


@pytest.mark.parametrize("stage", ["before", "after", "children"])
def test_a_vanished_descendant_restarts_the_whole_forest(census, stage):
owner, owned, _ = census
Expand Down