diff --git a/results/stall.json b/results/stall.json index daa2955..5f1000c 100644 --- a/results/stall.json +++ b/results/stall.json @@ -3,7 +3,8 @@ "summary": { "runs": 3, "fast_burn_firing_after_s_median": 60.549, - "fast_burn_resolved_after_resume_s_median": 27.77, + "fast_burn_resolved_after_resume_s_median": 45.33, + "fast_burn_resolved_after_jobs_running_s_median": 27.77, "jobs_running_after_resume_s_median": 18.724, "all_exact": false }, @@ -18,7 +19,8 @@ "fast_burn_pending_after_s": 59.188616037368774, "fast_burn_firing_after_s": 59.388949155807495, "jobs_running_after_resume_s": 34.94162583351135, - "fast_burn_resolved_after_resume_s": 9.206732034683228, + "fast_burn_resolved_after_jobs_running_s": 9.206732034683228, + "fast_burn_resolved_after_resume_s": 44.15, "error_ratio_5m_at_resolve": 0.5423728813559322, "slow_burn_state_at_end": "inactive", "exact": false, @@ -123,7 +125,8 @@ "fast_burn_pending_after_s": 61.62715005874634, "fast_burn_firing_after_s": 61.79443311691284, "jobs_running_after_resume_s": 18.72392702102661, - "fast_burn_resolved_after_resume_s": 27.769928216934204, + "fast_burn_resolved_after_jobs_running_s": 27.769928216934204, + "fast_burn_resolved_after_resume_s": 46.49, "error_ratio_5m_at_resolve": 0.5423728813559322, "slow_burn_state_at_end": "inactive", "exact": false, @@ -228,7 +231,8 @@ "fast_burn_pending_after_s": 60.38098216056824, "fast_burn_firing_after_s": 60.54856204986572, "jobs_running_after_resume_s": 11.509905338287354, - "fast_burn_resolved_after_resume_s": 33.82449293136597, + "fast_burn_resolved_after_jobs_running_s": 33.82449293136597, + "fast_burn_resolved_after_resume_s": 45.33, "error_ratio_5m_at_resolve": 0.5423728813559322, "slow_burn_state_at_end": "inactive", "exact": false, diff --git a/scripts/measure.py b/scripts/measure.py index 7806b22..f33a746 100644 --- a/scripts/measure.py +++ b/scripts/measure.py @@ -396,7 +396,9 @@ def exp_stall(runs: int, rate: int, hold: int) -> None: t_resume = time.time() sh(*COMPOSE, "unpause", *tms) recovered = wait_until(jobs_healthy, 300, 0.5) + # Starts when the jobs are healthy again, not at the unpause. resolved = wait_until(lambda: "FreshnessBudgetFastBurn" not in alerts(), 600, 1) + since_resume = round(time.time() - t_resume, 2) if resolved is not None else None burn5m = prom("sp:freshness_error_ratio:rate5m") g.join() rec = reconcile(run_id, wait=300) @@ -404,7 +406,8 @@ def exp_stall(runs: int, rate: int, hold: int) -> None: "paused_s": round(t_resume - t_pause, 1), "fast_burn_pending_after_s": pending, "fast_burn_firing_after_s": fired_at, "jobs_running_after_resume_s": recovered, - "fast_burn_resolved_after_resume_s": resolved, + "fast_burn_resolved_after_jobs_running_s": resolved, + "fast_burn_resolved_after_resume_s": since_resume, "error_ratio_5m_at_resolve": burn5m, "slow_burn_state_at_end": alerts().get("FreshnessBudgetSlowBurn", "inactive"), "exact": rec["exact"], "layers": layers_exact(rec), "reconcile": rec, @@ -416,6 +419,9 @@ def exp_stall(runs: int, rate: int, hold: int) -> None: "fast_burn_resolved_after_resume_s_median": med( [r["fast_burn_resolved_after_resume_s"] for r in out] ), + "fast_burn_resolved_after_jobs_running_s_median": med( + [r["fast_burn_resolved_after_jobs_running_s"] for r in out] + ), "jobs_running_after_resume_s_median": med([r["jobs_running_after_resume_s"] for r in out]), "all_exact": all(r["exact"] for r in out), }