diff --git a/src/analysis/post_run/partition.py b/src/analysis/post_run/partition.py index 3d93cca1..252a000f 100644 --- a/src/analysis/post_run/partition.py +++ b/src/analysis/post_run/partition.py @@ -5,15 +5,19 @@ import logging from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, List, Optional, Sequence import pandas as pd from src.analysis.post_run.scenario_common import ( + MESH_EXPECTATION, POD_COLUMN, Row, event_time, latency_summary, + load_mesh_peers, + mesh_peers_row, + pod_names, prepare, write_table, ) @@ -25,6 +29,8 @@ logger = logging.getLogger(__name__) PHASES = ("before", "split", "healed") +MESH_MOVE = 0.1 +"""Share of its value at the cut a half's mean mesh degree must move by to count as a reaction.""" def load_publishes(dump_dir: Path) -> pd.DataFrame: @@ -151,6 +157,70 @@ def partition_table( return rows +def mesh_reaction(mesh: pd.DataFrame, pods: Sequence[str], cut: datetime, heal: datetime) -> str: + """When, under the split, the pods' mean mesh degree first left its value at the cut. + + A node whose mesh still counts peers it cannot reach reports the same degree as before + the cut, so the time this takes is how long the split went unnoticed. It is known only + to the scrape step: the move happened between the two snapshots named. + """ + columns = [p for p in pods if p in mesh.columns] + mean = mesh[columns].mean(axis=1).dropna() + before = mean[mean.index <= cut] + under = mean[(mean.index > cut) & (mean.index <= heal)] + if before.empty or under.empty: + return "no snapshot before the cut" if before.empty else "no snapshot under the split" + + base = before.iloc[-1] + moved = under[(under - base).abs() > MESH_MOVE * base] + if moved.empty: + held = (under.index[-1] - cut).total_seconds() + return f"held at {base:.1f} for the {held:.0f}s observed" + earlier = under[under.index < moved.index[0]] + low = (earlier.index[-1] - cut).total_seconds() if not earlier.empty else 0 + high = (moved.index[0] - cut).total_seconds() + return f"{base:.1f} to {moved.iloc[0]:.1f} between {low:.0f}s and {high:.0f}s after the cut" + + +def mesh_rows( + mesh: Optional[pd.DataFrame], + side_a: Sequence[str], + side_b: Sequence[str], + applied: datetime, + heal: datetime, +) -> List[Row]: + """Mesh degree of each half: when it reacted to the cut, and where it stood under the + split and after the heal.""" + under_a, under_b, after = ( + "mesh peers under the split, side a", + "mesh peers under the split, side b", + "mesh peers after the heal", + ) + if mesh is None or mesh.empty: + return [ + ("mesh reaction to the cut", "no bound yet", "no mesh-peers metrics in the run folder"), + *(mesh_peers_row(mesh, item) for item in (under_a, under_b, after)), + ] + + def degree(window: pd.DataFrame, item: str, pods: Sequence[str], missing: str) -> Row: + if window.empty: + return (item, MESH_EXPECTATION, missing) + return mesh_peers_row(window, item, pods=pods) + + split = mesh[(mesh.index > applied) & (mesh.index <= heal)] + healed = mesh[mesh.index > heal] + reaction = "; ".join( + f"side {name}: {mesh_reaction(mesh, pods, applied, heal)}" + for name, pods in (("a", side_a), ("b", side_b)) + ) + return [ + ("mesh reaction to the cut", "no bound yet", reaction), + degree(split, under_a, side_a, "no snapshot under the split"), + degree(split, under_b, side_b, "no snapshot under the split"), + degree(healed, after, [*side_a, *side_b], "no snapshot after the heal"), + ] + + def run_partition_analysis(experiment: "NimLibp2pExperiment") -> None: dump_dir, df = prepare(experiment) cfg = experiment.config @@ -172,8 +242,12 @@ def run_partition_analysis(experiment: "NimLibp2pExperiment") -> None: split = applied_events[-1]["side_a"] per_msg = per_message_reach(publishes, df, split, cfg.num_relay_nodes, applied, heal) + pods = pod_names(df[POD_COLUMN].iloc[0], cfg.num_relay_nodes) write_table( dump_dir, "partition_result", - partition_table(per_msg, df, applied, heal, cfg.num_relay_nodes), + partition_table(per_msg, df, applied, heal, cfg.num_relay_nodes) + + mesh_rows( + load_mesh_peers(experiment.output_folder), pods[:split], pods[split:], applied, heal + ), ) diff --git a/src/analysis/post_run/tests/test_partition.py b/src/analysis/post_run/tests/test_partition.py index 0aa32dca..550ef983 100644 --- a/src/analysis/post_run/tests/test_partition.py +++ b/src/analysis/post_run/tests/test_partition.py @@ -2,7 +2,7 @@ import pandas as pd -from src.analysis.post_run.partition import partition_table, per_message_reach +from src.analysis.post_run.partition import mesh_rows, partition_table, per_message_reach from src.analysis.post_run.tests.conftest import T0 CUT = T0 + timedelta(seconds=50) @@ -100,3 +100,75 @@ def test_counts_each_node_once(deliveries, fanout): assert rows["delivery under the split, own half"].startswith( "0 of 1 messages reached their whole half" ) + + +SIDE_A, SIDE_B = ["pod-0", "pod-1"], ["pod-2", "pod-3"] + + +def mesh(snapshots): + """{seconds_after_T0: [degree of pod-0..pod-3]}, as the scraped mesh-peers frame.""" + return pd.DataFrame( + list(snapshots.values()), + index=pd.to_datetime([T0 + timedelta(seconds=s) for s in snapshots]), + columns=SIDE_A + SIDE_B, + ) + + +def mesh_table(snapshots): + return dict( + (item, result) for item, _, result in mesh_rows(snapshots, SIDE_A, SIDE_B, CUT, HEAL) + ) + + +def test_mesh_reaction_is_bracketed_by_the_snapshots_around_it(): + """Side a still reports 6 ten seconds into the cut and 3 thirty seconds in; side b never moves.""" + rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [6, 6, 6, 6], 80: [3, 3, 6, 6]})) + assert rows["mesh reaction to the cut"] == ( + "side a: 6.0 to 3.0 between 10s and 30s after the cut; " + "side b: held at 6.0 for the 30s observed" + ) + + +def test_mesh_reaction_in_the_first_snapshot_starts_at_the_cut(): + rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [2, 2, 2, 2]})) + assert rows["mesh reaction to the cut"].startswith( + "side a: 6.0 to 2.0 between 0s and 10s after the cut" + ) + + +def test_mesh_reaction_ignores_a_move_smaller_than_a_tenth(): + rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [6, 5, 6, 6]})) + assert rows["mesh reaction to the cut"].startswith("side a: held at 6.0 for the 10s observed") + + +def test_mesh_reaction_ignores_what_happens_after_the_heal(): + rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [6, 6, 6, 6], 130: [9, 9, 9, 9]})) + assert rows["mesh reaction to the cut"].startswith("side a: held at 6.0 for the 10s observed") + + +def test_mesh_reaction_says_which_snapshot_is_missing(): + assert mesh_table(mesh({60: [6, 6, 6, 6]}))["mesh reaction to the cut"].startswith( + "side a: no snapshot before the cut" + ) + assert mesh_table(mesh({40: [6, 6, 6, 6], 130: [6, 6, 6, 6]}))[ + "mesh reaction to the cut" + ].startswith("side a: no snapshot under the split") + + +def test_mesh_degree_is_reported_per_half_under_the_split_and_whole_after_it(): + rows = mesh_table(mesh({40: [6, 6, 6, 6], 80: [3, 5, 7, 9], 130: [4, 6, 6, 8]})) + assert rows["mesh peers under the split, side a"] == "median 4, range 3 to 5" + assert rows["mesh peers under the split, side b"] == "median 8, range 7 to 9" + assert rows["mesh peers after the heal"] == "median 6, range 4 to 8" + + +def test_mesh_rows_say_when_the_run_has_no_metrics(): + rows = mesh_table(None) + assert set(rows.values()) == {"no mesh-peers metrics in the run folder"} + assert len(rows) == 4 + + +def test_mesh_degree_says_when_a_phase_has_no_snapshot(): + rows = mesh_table(mesh({40: [6, 6, 6, 6]})) + assert rows["mesh peers under the split, side a"] == "no snapshot under the split" + assert rows["mesh peers after the heal"] == "no snapshot after the heal"