From a131845f3b0b6cd24b69f04461e93eeb262f8c3a Mon Sep 17 00:00:00 2001 From: sharareh-y Date: Fri, 23 Oct 2026 15:22:24 +0000 Subject: [PATCH] Use a linear default score curve (SCORE_POWER 0.5 -> 1.0) The square-root curve has an unbounded slope at the 1.05x threshold, so the score of a kernel just above the cutoff swings with timing noise and can dominate a run's sensitivity. The linear curve keeps a finite slope there (t/x^2), scores the fraction of runtime saved beyond the threshold, and leaves more room to separate large speedups. The 1.05x threshold is unchanged. Update the README and API reference scoring sections, the release notes, and the expected scores in tests (2x: 0.475, 1.1x: 1/22, 1.5x: 0.3); pin the default constants in a test. --- README.md | 20 ++++++++++++-------- docs/reference/api-reference.md | 18 +++++++++++------- docs/reference/release-notes.md | 6 ++++++ src/score.py | 7 ++++--- tests/test_candidate_verification.py | 2 +- tests/test_held_out_v2.py | 2 +- tests/test_postprocessing_outcomes_v2.py | 2 +- tests/test_score.py | 17 ++++++++++++----- tests/test_task_run_v2.py | 4 ++-- 9 files changed, 50 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 04de02c42..8c2cb4ca4 100755 --- a/README.md +++ b/README.md @@ -364,7 +364,7 @@ tasks: Every normal optimization task produces `task_result.yaml` with compilation and correctness status, baseline and optimized times, speedup, timing-method metadata, and the final score. -The score uses the power family with **`p = 0.5`** (square root): +The score uses the power family with **`p = 1`** (linear): ```text score = max(0, 1 - speedup_threshold / speedup_ratio) ** p @@ -378,13 +378,17 @@ must be more than 5% faster to earn a positive score: | Speedup | Score | | --- | ---: | | At or below 1.05× | 0 | -| 1.1× | 0.2132 | -| 1.2× | 0.3536 | -| 2× | 0.6892 | - -Scores are bounded between `0` and `1`. The concave curve gives modest qualifying -improvements more credit than a linear curve. A zero score does not imply a -correctness failure; compilation, correctness, and acceptance have separate fields. +| 1.1× | 0.0455 | +| 1.2× | 0.1250 | +| 2× | 0.4750 | + +Scores are bounded between `0` and `1`. With the linear curve, the score is the +fraction of runtime saved beyond the threshold, `1 - candidate_time / +(baseline_time / 1.05)`, and its slope stays finite just above the cutoff, so +timing noise moves near-threshold scores only slightly. Over a set of tasks that +all score above zero, the mean score is `1 - 1.05 / H`, where `H` is their +harmonic-mean speedup. A zero score does not imply a correctness failure; +compilation, correctness, and acceptance have separate fields. To adjust the curve, edit `SCORE_POWER` in [`src/score.py`](src/score.py): `0.5` is concave, `1.0` is linear, and `2.0` is convex. Edit diff --git a/docs/reference/api-reference.md b/docs/reference/api-reference.md index 6dde3a946..b4bd3fa62 100644 --- a/docs/reference/api-reference.md +++ b/docs/reference/api-reference.md @@ -342,7 +342,7 @@ the ordinary correctness run. ## Scoring The existing `score` field uses a bounded power-family reward. The default is -the concave square-root curve (`p = 0.5`) with a `1.05×` speedup cutoff: +the linear curve (`p = 1`) with a `1.05×` speedup cutoff: ```text s = speedup_ratio @@ -375,19 +375,23 @@ run-configuration section: | Constant | Default | Effect | | --- | ---: | --- | -| `SCORE_POWER` | `0.5` | Exponent `p`: `0.5` is concave, `1.0` linear, `2.0` convex | +| `SCORE_POWER` | `1.0` | Exponent `p`: `1.0` linear, `0.5` concave, `2.0` convex | | `SCORE_SPEEDUP_THRESHOLD` | `1.05` | Speedup at or below which the score is zero | -Both constants must be finite and positive. `p = 0.5` raises scores for modest -qualifying improvements; `p = 2.0` emphasizes large improvements. All choices -preserve the ranking above the cutoff. Changing `p` does not provide a signal +Both constants must be finite and positive. With the default `p = 1`, the score +is the fraction of runtime saved beyond the threshold, and its slope +`d score / d s = speedup_threshold / s**2` stays finite at the cutoff. Any +`p < 1` has an unbounded slope as `s` approaches the cutoff from above, so +scores of candidates just above `1.05×` swing with timing noise. `p = 0.5` +raises scores for modest qualifying improvements; `p = 2.0` emphasizes large +improvements. All choices preserve the ranking above the cutoff. Changing `p` does not provide a signal below the cutoff. Setting the threshold to `1.0` removes the 5% margin but still gives zero to baseline-equivalent and slower candidates. **Example**: For a correct candidate taking 80 microseconds against a 100-microsecond baseline, `s = 1.25`. The default score is -`sqrt(1 - 1.05 / 1.25) = 0.4`. The same measurement scores `0.16` with `p = 1` -and `0.0256` with `p = 2`. +`1 - 1.05 / 1.25 = 0.16`. The same measurement scores `0.4` with `p = 0.5` +(`sqrt(0.16)`) and `0.0256` with `p = 2`. #### Result interpretation and historical reports diff --git a/docs/reference/release-notes.md b/docs/reference/release-notes.md index 84d3075d0..3963e6449 100644 --- a/docs/reference/release-notes.md +++ b/docs/reference/release-notes.md @@ -11,6 +11,12 @@ This topic summarizes the features available in each AgentKernelArena release. F ## Unreleased +- Changed the default per-task score curve from square root to linear + (`SCORE_POWER` 0.5 → 1.0); the 1.05× threshold is unchanged. The linear + curve has a finite slope at the threshold, so scores just above it no longer + swing with timing noise. Positive scores decrease (for example, 2× now scores + 0.475 instead of 0.689), so scores from earlier revisions are not directly + comparable. See [Scoring](api-reference.md#scoring). - Added the opt-in `deepseek_harness` agent with a pinned headless CLI, API-key authentication, isolated per-task sessions, and Docker/Slurm quickstarts. See the [integration guide](../../agents/deepseek_harness/README.md) diff --git a/src/score.py b/src/score.py index eedbdfb6f..053f9a215 100755 --- a/src/score.py +++ b/src/score.py @@ -5,7 +5,7 @@ # Power-family scoring; see docs/reference/api-reference.md#scoring. -SCORE_POWER = 0.5 +SCORE_POWER = 1.0 SCORE_SPEEDUP_THRESHOLD = 1.05 @@ -56,8 +56,9 @@ def score( Calculate a bounded power-family score for a correct, comparable candidate. The score is ``max(0, 1 - SCORE_SPEEDUP_THRESHOLD / speedup) ** SCORE_POWER``. - Defaults use the concave square-root curve and require more than 1.05x - speedup for a positive score. Compilation/correctness failures and invalid + Defaults use the linear curve (power 1), which scores the fraction of + runtime saved beyond the threshold, and require more than 1.05x speedup + for a positive score. Compilation/correctness failures and invalid performance comparisons receive zero, with no partial-credit bonuses. Args: diff --git a/tests/test_candidate_verification.py b/tests/test_candidate_verification.py index 90b0b2847..f6f596318 100644 --- a/tests/test_candidate_verification.py +++ b/tests/test_candidate_verification.py @@ -139,7 +139,7 @@ def test_review_gate_controls_final_acceptance_and_exports(tmp_path, monkeypatch assert saved['candidate_accepted'] is accepted assert bool(saved['exports']) is accepted assert saved['speedup_ratio'] == (2 if accepted else 0) - assert saved['score'] == (pytest.approx(0.689202437604511) if accepted else 0) + assert saved['score'] == (pytest.approx(0.475) if accepted else 0) def test_backend_uses_read_only_tools_and_never_exposes_shell(monkeypatch, tmp_path): diff --git a/tests/test_held_out_v2.py b/tests/test_held_out_v2.py index a77109467..eb480318b 100644 --- a/tests/test_held_out_v2.py +++ b/tests/test_held_out_v2.py @@ -69,7 +69,7 @@ def test_uses_saved_baseline_and_shared_actions_instead_of_current_task_checkout result = evaluate(workspace, task_dir) assert result["generalization_status"] == "both_pass", result assert result["speedup_ratio"] == 2.0 - assert result["score"] == pytest.approx(0.689202437604511) + assert result["score"] == pytest.approx(0.475) orig = tmp_path / "heldout" / "orig" / "kernel.py" assert "return 3" in orig.read_text() assert "return 1 + 2" in (workspace / "kernel.py").read_text() diff --git a/tests/test_postprocessing_outcomes_v2.py b/tests/test_postprocessing_outcomes_v2.py index e73b783a4..4e43462c0 100644 --- a/tests/test_postprocessing_outcomes_v2.py +++ b/tests/test_postprocessing_outcomes_v2.py @@ -15,7 +15,7 @@ def test_final_outcomes_survive_all_report_formats_without_changing_scores( tmp_path, monkeypatch, recover_from_text, ): # A positive score exercises the four-decimal report and dashboard formats. - speedup, expected_score = 1.1, 0.21320071635561044 + speedup, expected_score = 1.1, 1 / 22 # 1 - 1.05/1.1, reported as 0.0455 run = tmp_path / "workspace_MI355X_codex" / "run_20260916_120000" # Legacy results have numerical evidence, but no evidence of final acceptance. fixtures = [ diff --git a/tests/test_score.py b/tests/test_score.py index 340b31465..87b72fdf9 100644 --- a/tests/test_score.py +++ b/tests/test_score.py @@ -3,6 +3,7 @@ import pytest import yaml +import src.score as score_module from src.score import resolve_speedup_ratio, score, task_result_scoring @@ -67,10 +68,16 @@ def test_explicit_speedup_without_method_metadata_gets_no_performance_points(): ) == 0.0 +def test_default_scoring_constants_are_linear_with_5_percent_cutoff(): + assert score_module.SCORE_POWER == 1.0 + assert score_module.SCORE_SPEEDUP_THRESHOLD == 1.05 + + @pytest.mark.parametrize("speedup,expected", [ - (1.0, 0.0), (1.05, 0.0), (1.25, 0.4), (2.0, 0.689202437604511), + (1.0, 0.0), (1.05, 0.0), (1.1, 1 / 22), (1.2, 0.125), (1.25, 0.16), (2.0, 0.475), ]) -def test_default_square_root_score(speedup, expected): +def test_default_linear_score(speedup, expected): + # 1 - 1.05/1.1 = 1/22 ~ 0.0455; 1 - 1.05/1.2 = 0.125; 1 - 1.05/2 = 0.475. assert score(True, True, 100.0, 100.0 / speedup, speedup, True) == pytest.approx(expected) @@ -89,7 +96,7 @@ def test_score_requires_boolean_method_consistency(consistent): assert score(True, True, 100.0, 50.0, 2.0, consistent) == 0.0 -@pytest.mark.parametrize("power,expected", [(1.0, 0.16), (2.0, 0.0256)]) +@pytest.mark.parametrize("power,expected", [(0.5, 0.4), (1.0, 0.16), (2.0, 0.0256)]) def test_power_constant_selects_curve(monkeypatch, power, expected): monkeypatch.setattr("src.score.SCORE_POWER", power) assert score(True, True, 100.0, 80.0, 1.25, True) == pytest.approx(expected) @@ -99,7 +106,7 @@ def test_speedup_threshold_constant_changes_cutoff(monkeypatch): monkeypatch.setattr("src.score.SCORE_SPEEDUP_THRESHOLD", 1.2) assert score(True, True, 100.0, 90.0, 1.1, True) == 0.0 assert score(True, True, 100.0, 100.0 / 1.2, 1.2, True) == 0.0 - assert score(True, True, 100.0, 80.0, 1.25, True) == pytest.approx(0.2) + assert score(True, True, 100.0, 80.0, 1.25, True) == pytest.approx(0.04) @pytest.mark.parametrize("name", ["SCORE_POWER", "SCORE_SPEEDUP_THRESHOLD"]) @@ -132,5 +139,5 @@ def test_task_result_scores_explicit_mean_speedup_and_preserves_measurements(tmp calculated = task_result_scoring(str(tmp_path)) - assert calculated == pytest.approx(0.5477225575051661) + assert calculated == pytest.approx(0.3) assert yaml.safe_load(result_file.read_text()) == {**result, "score": calculated} diff --git a/tests/test_task_run_v2.py b/tests/test_task_run_v2.py index 35275dc03..e3184cdd9 100644 --- a/tests/test_task_run_v2.py +++ b/tests/test_task_run_v2.py @@ -84,7 +84,7 @@ def observe(self, action, *args, **kwargs): assert report["candidate_accepted"] is not blocked assert ("performance" in actions) is not blocked assert report["speedup_ratio"] == (0 if blocked else 2) - assert report["score"] == (0 if blocked else pytest.approx(0.689202437604511)) + assert report["score"] == (0 if blocked else pytest.approx(0.475)) assert report["benchmark_method_consistent"] is not blocked if blocked: assert "Process sweep" in report["speedup_calculation_error_message"] @@ -93,7 +93,7 @@ def observe(self, action, *args, **kwargs): summary = json.loads((workspace.parent / "reports/task_type_breakdown.json").read_text())["overall"] assert summary["valid_speedup_count"] == (0 if blocked else 1) assert summary["speedup_gt_1_count"] == (0 if blocked else 1) - assert read_report(workspace)["score"] == (0 if blocked else pytest.approx(0.689202437604511)) + assert read_report(workspace)["score"] == (0 if blocked else pytest.approx(0.475)) def test_complete_pipeline_keeps_baseline_and_evaluates_after_agent_error(tmp_path):