From 68c06e50df46993140ae21da0aefdfdac8cbaa6f Mon Sep 17 00:00:00 2001 From: shanurwan Date: Sat, 14 Nov 2026 15:18:24 +0800 Subject: [PATCH] Partition analysis: when each half's mesh reacted to the cut The churn and degraded tables end with a mesh-peers row; the partition table had none. A node whose mesh still counts peers it cannot reach reports the same degree as before the cut, so delivery alone does not show how long a split goes unnoticed. Adds four rows from the mesh-peers gauge the run already scrapes: when each half's mean mesh degree first moved by more than a tenth from its value at the cut, bracketed by the two snapshots around the move, and the degree of each half under the split and of the whole network after the heal. --- src/analysis/post_run/partition.py | 78 ++++++++++++++++++- src/analysis/post_run/tests/test_partition.py | 74 +++++++++++++++++- 2 files changed, 149 insertions(+), 3 deletions(-) diff --git a/src/analysis/post_run/partition.py b/src/analysis/post_run/partition.py index 3d93cca1..252a000f 100644 --- a/src/analysis/post_run/partition.py +++ b/src/analysis/post_run/partition.py @@ -5,15 +5,19 @@ import logging from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, List, Optional, Sequence import pandas as pd from src.analysis.post_run.scenario_common import ( + MESH_EXPECTATION, POD_COLUMN, Row, event_time, latency_summary, + load_mesh_peers, + mesh_peers_row, + pod_names, prepare, write_table, ) @@ -25,6 +29,8 @@ logger = logging.getLogger(__name__) PHASES = ("before", "split", "healed") +MESH_MOVE = 0.1 +"""Share of its value at the cut a half's mean mesh degree must move by to count as a reaction.""" def load_publishes(dump_dir: Path) -> pd.DataFrame: @@ -151,6 +157,70 @@ def partition_table( return rows +def mesh_reaction(mesh: pd.DataFrame, pods: Sequence[str], cut: datetime, heal: datetime) -> str: + """When, under the split, the pods' mean mesh degree first left its value at the cut. + + A node whose mesh still counts peers it cannot reach reports the same degree as before + the cut, so the time this takes is how long the split went unnoticed. It is known only + to the scrape step: the move happened between the two snapshots named. + """ + columns = [p for p in pods if p in mesh.columns] + mean = mesh[columns].mean(axis=1).dropna() + before = mean[mean.index <= cut] + under = mean[(mean.index > cut) & (mean.index <= heal)] + if before.empty or under.empty: + return "no snapshot before the cut" if before.empty else "no snapshot under the split" + + base = before.iloc[-1] + moved = under[(under - base).abs() > MESH_MOVE * base] + if moved.empty: + held = (under.index[-1] - cut).total_seconds() + return f"held at {base:.1f} for the {held:.0f}s observed" + earlier = under[under.index < moved.index[0]] + low = (earlier.index[-1] - cut).total_seconds() if not earlier.empty else 0 + high = (moved.index[0] - cut).total_seconds() + return f"{base:.1f} to {moved.iloc[0]:.1f} between {low:.0f}s and {high:.0f}s after the cut" + + +def mesh_rows( + mesh: Optional[pd.DataFrame], + side_a: Sequence[str], + side_b: Sequence[str], + applied: datetime, + heal: datetime, +) -> List[Row]: + """Mesh degree of each half: when it reacted to the cut, and where it stood under the + split and after the heal.""" + under_a, under_b, after = ( + "mesh peers under the split, side a", + "mesh peers under the split, side b", + "mesh peers after the heal", + ) + if mesh is None or mesh.empty: + return [ + ("mesh reaction to the cut", "no bound yet", "no mesh-peers metrics in the run folder"), + *(mesh_peers_row(mesh, item) for item in (under_a, under_b, after)), + ] + + def degree(window: pd.DataFrame, item: str, pods: Sequence[str], missing: str) -> Row: + if window.empty: + return (item, MESH_EXPECTATION, missing) + return mesh_peers_row(window, item, pods=pods) + + split = mesh[(mesh.index > applied) & (mesh.index <= heal)] + healed = mesh[mesh.index > heal] + reaction = "; ".join( + f"side {name}: {mesh_reaction(mesh, pods, applied, heal)}" + for name, pods in (("a", side_a), ("b", side_b)) + ) + return [ + ("mesh reaction to the cut", "no bound yet", reaction), + degree(split, under_a, side_a, "no snapshot under the split"), + degree(split, under_b, side_b, "no snapshot under the split"), + degree(healed, after, [*side_a, *side_b], "no snapshot after the heal"), + ] + + def run_partition_analysis(experiment: "NimLibp2pExperiment") -> None: dump_dir, df = prepare(experiment) cfg = experiment.config @@ -172,8 +242,12 @@ def run_partition_analysis(experiment: "NimLibp2pExperiment") -> None: split = applied_events[-1]["side_a"] per_msg = per_message_reach(publishes, df, split, cfg.num_relay_nodes, applied, heal) + pods = pod_names(df[POD_COLUMN].iloc[0], cfg.num_relay_nodes) write_table( dump_dir, "partition_result", - partition_table(per_msg, df, applied, heal, cfg.num_relay_nodes), + partition_table(per_msg, df, applied, heal, cfg.num_relay_nodes) + + mesh_rows( + load_mesh_peers(experiment.output_folder), pods[:split], pods[split:], applied, heal + ), ) diff --git a/src/analysis/post_run/tests/test_partition.py b/src/analysis/post_run/tests/test_partition.py index 0aa32dca..550ef983 100644 --- a/src/analysis/post_run/tests/test_partition.py +++ b/src/analysis/post_run/tests/test_partition.py @@ -2,7 +2,7 @@ import pandas as pd -from src.analysis.post_run.partition import partition_table, per_message_reach +from src.analysis.post_run.partition import mesh_rows, partition_table, per_message_reach from src.analysis.post_run.tests.conftest import T0 CUT = T0 + timedelta(seconds=50) @@ -100,3 +100,75 @@ def test_counts_each_node_once(deliveries, fanout): assert rows["delivery under the split, own half"].startswith( "0 of 1 messages reached their whole half" ) + + +SIDE_A, SIDE_B = ["pod-0", "pod-1"], ["pod-2", "pod-3"] + + +def mesh(snapshots): + """{seconds_after_T0: [degree of pod-0..pod-3]}, as the scraped mesh-peers frame.""" + return pd.DataFrame( + list(snapshots.values()), + index=pd.to_datetime([T0 + timedelta(seconds=s) for s in snapshots]), + columns=SIDE_A + SIDE_B, + ) + + +def mesh_table(snapshots): + return dict( + (item, result) for item, _, result in mesh_rows(snapshots, SIDE_A, SIDE_B, CUT, HEAL) + ) + + +def test_mesh_reaction_is_bracketed_by_the_snapshots_around_it(): + """Side a still reports 6 ten seconds into the cut and 3 thirty seconds in; side b never moves.""" + rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [6, 6, 6, 6], 80: [3, 3, 6, 6]})) + assert rows["mesh reaction to the cut"] == ( + "side a: 6.0 to 3.0 between 10s and 30s after the cut; " + "side b: held at 6.0 for the 30s observed" + ) + + +def test_mesh_reaction_in_the_first_snapshot_starts_at_the_cut(): + rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [2, 2, 2, 2]})) + assert rows["mesh reaction to the cut"].startswith( + "side a: 6.0 to 2.0 between 0s and 10s after the cut" + ) + + +def test_mesh_reaction_ignores_a_move_smaller_than_a_tenth(): + rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [6, 5, 6, 6]})) + assert rows["mesh reaction to the cut"].startswith("side a: held at 6.0 for the 10s observed") + + +def test_mesh_reaction_ignores_what_happens_after_the_heal(): + rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [6, 6, 6, 6], 130: [9, 9, 9, 9]})) + assert rows["mesh reaction to the cut"].startswith("side a: held at 6.0 for the 10s observed") + + +def test_mesh_reaction_says_which_snapshot_is_missing(): + assert mesh_table(mesh({60: [6, 6, 6, 6]}))["mesh reaction to the cut"].startswith( + "side a: no snapshot before the cut" + ) + assert mesh_table(mesh({40: [6, 6, 6, 6], 130: [6, 6, 6, 6]}))[ + "mesh reaction to the cut" + ].startswith("side a: no snapshot under the split") + + +def test_mesh_degree_is_reported_per_half_under_the_split_and_whole_after_it(): + rows = mesh_table(mesh({40: [6, 6, 6, 6], 80: [3, 5, 7, 9], 130: [4, 6, 6, 8]})) + assert rows["mesh peers under the split, side a"] == "median 4, range 3 to 5" + assert rows["mesh peers under the split, side b"] == "median 8, range 7 to 9" + assert rows["mesh peers after the heal"] == "median 6, range 4 to 8" + + +def test_mesh_rows_say_when_the_run_has_no_metrics(): + rows = mesh_table(None) + assert set(rows.values()) == {"no mesh-peers metrics in the run folder"} + assert len(rows) == 4 + + +def test_mesh_degree_says_when_a_phase_has_no_snapshot(): + rows = mesh_table(mesh({40: [6, 6, 6, 6]})) + assert rows["mesh peers under the split, side a"] == "no snapshot under the split" + assert rows["mesh peers after the heal"] == "no snapshot after the heal"