From e7d7edb0742ebba0b3cd4e313de2d5eba892d9f4 Mon Sep 17 00:00:00 2001 From: Momin Date: Tue, 11 Aug 2026 17:12:10 +0500 Subject: [PATCH] fix: correct expected_rule in gradient/tangent_line benchmarksbenchmark_gradient.json and benchmark_tangent_line.json had expected_ruleset to the literal operation name ('gradient', 'tangent_line'), which doesnot exist anywhere in the 13 real RULE: tokens in vocab.json. This does notaffect today's exact-match accuracy (run_eval.py's accuracy calc is purelyexpr-comparison based, expected_rule is currently unread there), but itwould silently produce a permanent 0% the moment any rule-level accuracymetric is implemented -- unrelated to model quality.problem_generator.py's generate_gradient_diff() and generate_tangent_line_diff()already assign rule_id=7 (partial_derivative) and rule_id=0 (power_rule)respectively (see inline comments at those return statements). This alignsexpected_rule with what the model is actually trained to predict.Verified: all 50/50 rows in each file updated; cross-checked againstvocab.json's actual rule_tokens list -- zero invalid labels remain. --- eval/benchmarks/benchmark_gradient.json | 100 ++++++++++---------- eval/benchmarks/benchmark_tangent_line.json | 100 ++++++++++---------- 2 files changed, 100 insertions(+), 100 deletions(-) diff --git a/eval/benchmarks/benchmark_gradient.json b/eval/benchmarks/benchmark_gradient.json index 4a2b88a..0eb6a52 100644 --- a/eval/benchmarks/benchmark_gradient.json +++ b/eval/benchmarks/benchmark_gradient.json @@ -51,7 +51,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -108,7 +108,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -162,7 +162,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -219,7 +219,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -270,7 +270,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -321,7 +321,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -375,7 +375,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -429,7 +429,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -486,7 +486,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -537,7 +537,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -594,7 +594,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -648,7 +648,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -699,7 +699,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -753,7 +753,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -810,7 +810,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -864,7 +864,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -918,7 +918,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -975,7 +975,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1026,7 +1026,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1077,7 +1077,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1134,7 +1134,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1188,7 +1188,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1245,7 +1245,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1296,7 +1296,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1347,7 +1347,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1404,7 +1404,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1461,7 +1461,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1518,7 +1518,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1569,7 +1569,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1620,7 +1620,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1671,7 +1671,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1728,7 +1728,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1782,7 +1782,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1836,7 +1836,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1890,7 +1890,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1944,7 +1944,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -1998,7 +1998,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2055,7 +2055,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2109,7 +2109,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2163,7 +2163,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2217,7 +2217,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2268,7 +2268,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2322,7 +2322,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2379,7 +2379,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2433,7 +2433,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2484,7 +2484,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2541,7 +2541,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2595,7 +2595,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2652,7 +2652,7 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" }, { "expr": { @@ -2706,6 +2706,6 @@ } }, "operation": "gradient", - "expected_rule": "gradient" + "expected_rule": "partial_derivative" } ] \ No newline at end of file diff --git a/eval/benchmarks/benchmark_tangent_line.json b/eval/benchmarks/benchmark_tangent_line.json index d4a1b5c..99ab845 100644 --- a/eval/benchmarks/benchmark_tangent_line.json +++ b/eval/benchmarks/benchmark_tangent_line.json @@ -37,7 +37,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -77,7 +77,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -117,7 +117,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -157,7 +157,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -197,7 +197,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -237,7 +237,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -277,7 +277,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -317,7 +317,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -357,7 +357,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -397,7 +397,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -437,7 +437,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -477,7 +477,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -517,7 +517,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -557,7 +557,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -597,7 +597,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -637,7 +637,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -677,7 +677,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -717,7 +717,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -757,7 +757,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -797,7 +797,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -837,7 +837,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -877,7 +877,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -917,7 +917,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -957,7 +957,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -997,7 +997,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1037,7 +1037,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1077,7 +1077,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1117,7 +1117,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1157,7 +1157,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1197,7 +1197,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1237,7 +1237,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1277,7 +1277,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1317,7 +1317,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1357,7 +1357,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1397,7 +1397,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1437,7 +1437,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1477,7 +1477,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1517,7 +1517,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1557,7 +1557,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1597,7 +1597,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1637,7 +1637,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1677,7 +1677,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1717,7 +1717,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1757,7 +1757,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1797,7 +1797,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1837,7 +1837,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1877,7 +1877,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1917,7 +1917,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1957,7 +1957,7 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" }, { "expr": { @@ -1997,6 +1997,6 @@ "deno": 1 }, "operation": "tangent_line", - "expected_rule": "tangent_line" + "expected_rule": "power_rule" } ] \ No newline at end of file