Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 76 additions & 2 deletions src/analysis/post_run/partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,19 @@
import logging
from datetime import datetime
from pathlib import Path
from typing import TYPE_CHECKING, List
from typing import TYPE_CHECKING, List, Optional, Sequence

import pandas as pd

from src.analysis.post_run.scenario_common import (
MESH_EXPECTATION,
POD_COLUMN,
Row,
event_time,
latency_summary,
load_mesh_peers,
mesh_peers_row,
pod_names,
prepare,
write_table,
)
Expand All @@ -25,6 +29,8 @@
logger = logging.getLogger(__name__)

PHASES = ("before", "split", "healed")
MESH_MOVE = 0.1
"""Share of its value at the cut a half's mean mesh degree must move by to count as a reaction."""


def load_publishes(dump_dir: Path) -> pd.DataFrame:
Expand Down Expand Up @@ -151,6 +157,70 @@ def partition_table(
return rows


def mesh_reaction(mesh: pd.DataFrame, pods: Sequence[str], cut: datetime, heal: datetime) -> str:
"""When, under the split, the pods' mean mesh degree first left its value at the cut.

A node whose mesh still counts peers it cannot reach reports the same degree as before
the cut, so the time this takes is how long the split went unnoticed. It is known only
to the scrape step: the move happened between the two snapshots named.
"""
columns = [p for p in pods if p in mesh.columns]
mean = mesh[columns].mean(axis=1).dropna()
before = mean[mean.index <= cut]
under = mean[(mean.index > cut) & (mean.index <= heal)]
if before.empty or under.empty:
return "no snapshot before the cut" if before.empty else "no snapshot under the split"

base = before.iloc[-1]
moved = under[(under - base).abs() > MESH_MOVE * base]
if moved.empty:
held = (under.index[-1] - cut).total_seconds()
return f"held at {base:.1f} for the {held:.0f}s observed"
earlier = under[under.index < moved.index[0]]
low = (earlier.index[-1] - cut).total_seconds() if not earlier.empty else 0
high = (moved.index[0] - cut).total_seconds()
return f"{base:.1f} to {moved.iloc[0]:.1f} between {low:.0f}s and {high:.0f}s after the cut"


def mesh_rows(
mesh: Optional[pd.DataFrame],
side_a: Sequence[str],
side_b: Sequence[str],
applied: datetime,
heal: datetime,
) -> List[Row]:
"""Mesh degree of each half: when it reacted to the cut, and where it stood under the
split and after the heal."""
under_a, under_b, after = (
"mesh peers under the split, side a",
"mesh peers under the split, side b",
"mesh peers after the heal",
)
if mesh is None or mesh.empty:
return [
("mesh reaction to the cut", "no bound yet", "no mesh-peers metrics in the run folder"),
*(mesh_peers_row(mesh, item) for item in (under_a, under_b, after)),
]

def degree(window: pd.DataFrame, item: str, pods: Sequence[str], missing: str) -> Row:
if window.empty:
return (item, MESH_EXPECTATION, missing)
return mesh_peers_row(window, item, pods=pods)

split = mesh[(mesh.index > applied) & (mesh.index <= heal)]
healed = mesh[mesh.index > heal]
reaction = "; ".join(
f"side {name}: {mesh_reaction(mesh, pods, applied, heal)}"
for name, pods in (("a", side_a), ("b", side_b))
)
return [
("mesh reaction to the cut", "no bound yet", reaction),
degree(split, under_a, side_a, "no snapshot under the split"),
degree(split, under_b, side_b, "no snapshot under the split"),
degree(healed, after, [*side_a, *side_b], "no snapshot after the heal"),
]


def run_partition_analysis(experiment: "NimLibp2pExperiment") -> None:
dump_dir, df = prepare(experiment)
cfg = experiment.config
Expand All @@ -172,8 +242,12 @@ def run_partition_analysis(experiment: "NimLibp2pExperiment") -> None:

split = applied_events[-1]["side_a"]
per_msg = per_message_reach(publishes, df, split, cfg.num_relay_nodes, applied, heal)
pods = pod_names(df[POD_COLUMN].iloc[0], cfg.num_relay_nodes)
write_table(
dump_dir,
"partition_result",
partition_table(per_msg, df, applied, heal, cfg.num_relay_nodes),
partition_table(per_msg, df, applied, heal, cfg.num_relay_nodes)
+ mesh_rows(
load_mesh_peers(experiment.output_folder), pods[:split], pods[split:], applied, heal
),
)
74 changes: 73 additions & 1 deletion src/analysis/post_run/tests/test_partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import pandas as pd

from src.analysis.post_run.partition import partition_table, per_message_reach
from src.analysis.post_run.partition import mesh_rows, partition_table, per_message_reach
from src.analysis.post_run.tests.conftest import T0

CUT = T0 + timedelta(seconds=50)
Expand Down Expand Up @@ -100,3 +100,75 @@ def test_counts_each_node_once(deliveries, fanout):
assert rows["delivery under the split, own half"].startswith(
"0 of 1 messages reached their whole half"
)


SIDE_A, SIDE_B = ["pod-0", "pod-1"], ["pod-2", "pod-3"]


def mesh(snapshots):
"""{seconds_after_T0: [degree of pod-0..pod-3]}, as the scraped mesh-peers frame."""
return pd.DataFrame(
list(snapshots.values()),
index=pd.to_datetime([T0 + timedelta(seconds=s) for s in snapshots]),
columns=SIDE_A + SIDE_B,
)


def mesh_table(snapshots):
return dict(
(item, result) for item, _, result in mesh_rows(snapshots, SIDE_A, SIDE_B, CUT, HEAL)
)


def test_mesh_reaction_is_bracketed_by_the_snapshots_around_it():
"""Side a still reports 6 ten seconds into the cut and 3 thirty seconds in; side b never moves."""
rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [6, 6, 6, 6], 80: [3, 3, 6, 6]}))
assert rows["mesh reaction to the cut"] == (
"side a: 6.0 to 3.0 between 10s and 30s after the cut; "
"side b: held at 6.0 for the 30s observed"
)


def test_mesh_reaction_in_the_first_snapshot_starts_at_the_cut():
rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [2, 2, 2, 2]}))
assert rows["mesh reaction to the cut"].startswith(
"side a: 6.0 to 2.0 between 0s and 10s after the cut"
)


def test_mesh_reaction_ignores_a_move_smaller_than_a_tenth():
rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [6, 5, 6, 6]}))
assert rows["mesh reaction to the cut"].startswith("side a: held at 6.0 for the 10s observed")


def test_mesh_reaction_ignores_what_happens_after_the_heal():
rows = mesh_table(mesh({40: [6, 6, 6, 6], 60: [6, 6, 6, 6], 130: [9, 9, 9, 9]}))
assert rows["mesh reaction to the cut"].startswith("side a: held at 6.0 for the 10s observed")


def test_mesh_reaction_says_which_snapshot_is_missing():
assert mesh_table(mesh({60: [6, 6, 6, 6]}))["mesh reaction to the cut"].startswith(
"side a: no snapshot before the cut"
)
assert mesh_table(mesh({40: [6, 6, 6, 6], 130: [6, 6, 6, 6]}))[
"mesh reaction to the cut"
].startswith("side a: no snapshot under the split")


def test_mesh_degree_is_reported_per_half_under_the_split_and_whole_after_it():
rows = mesh_table(mesh({40: [6, 6, 6, 6], 80: [3, 5, 7, 9], 130: [4, 6, 6, 8]}))
assert rows["mesh peers under the split, side a"] == "median 4, range 3 to 5"
assert rows["mesh peers under the split, side b"] == "median 8, range 7 to 9"
assert rows["mesh peers after the heal"] == "median 6, range 4 to 8"


def test_mesh_rows_say_when_the_run_has_no_metrics():
rows = mesh_table(None)
assert set(rows.values()) == {"no mesh-peers metrics in the run folder"}
assert len(rows) == 4


def test_mesh_degree_says_when_a_phase_has_no_snapshot():
rows = mesh_table(mesh({40: [6, 6, 6, 6]}))
assert rows["mesh peers under the split, side a"] == "no snapshot under the split"
assert rows["mesh peers after the heal"] == "no snapshot after the heal"