From 3e8a3e7bbe98fdd2483ba0467c77d2c0a76703e9 Mon Sep 17 00:00:00 2001 From: Morten Punnerud-Engelstad Date: Thu, 13 Aug 2026 18:05:14 +0200 Subject: [PATCH] A derivative field in the step schema, like calc -- with its usage measured at zero The field, the parser, the loop wiring, the UI chip, the graphdb fact and the bench capture are all in place and tested: 'd/dx at x=' is settled by mpeqs.calculus, whose two independent derivative paths must agree, and what cannot be settled -- x**x, sin(x), a missing point -- is silence rather than a guess. What the measurement says, and it is not the story the field was built for: re-run on the calculus domain the model filled the field ZERO times in eight questions -- and scored 8/8 anyway, up from 5/8. The visible mechanism is the CALC field: the model applies the chain rule itself and hands the resulting arithmetic over, "3*(4*(-2)^2 + 6)^2 * 8*(-2)" evaluated exactly. The earlier misses were arithmetic slips in exactly that step, so the existing machinery carries derivative questions once the model writes the expression down. 8/8 against 5/8 at n=8 is inside the measured noise floor and is NOT claimed as the field's effect; nothing that never fired can have caused anything. The field stays because it is passive -- an empty string costs nothing, silence is the failure mode -- and a model that does reach for it gets an answer that cannot be a matching mistake. The same shape as the convert field's early history, where required-but-unused preceded used-and-decisive by one model generation. Found and fixed on the way: the UI edit split the calc handler and orphaned its chip code inside the new branch, killing every sums chip -- caught by the existing render tests, both handlers now share one exactChip helper. 9 tests here, 461 in all. --- dev/tests/conftest.py | 4 ++- dev/tests/test_selection.py | 59 ++++++++++++++++++++++++++++++++ src/mpe_lkg/arithmetic.py | 38 ++++++++++++++++++++ src/mpe_lkg/backends/_shared.py | 15 +++++++- src/mpe_lkg/battery/bench.py | 2 ++ src/mpe_lkg/graphdb.py | 4 +++ src/mpe_lkg/reasoning.py | 31 ++++++++++++++++- src/mpe_lkg/templates/index.html | 30 ++++++++++------ 8 files changed, 169 insertions(+), 14 deletions(-) diff --git a/dev/tests/conftest.py b/dev/tests/conftest.py index e0ac3e5..5580dd3 100644 --- a/dev/tests/conftest.py +++ b/dev/tests/conftest.py @@ -16,7 +16,7 @@ def step(title: str, content: str, next_action: str = "continue", - calc: str = "", convert: str = "") -> str: + calc: str = "", convert: str = "", derivative: str = "") -> str: # calc and convert are omitted unless asked for, so the scripts that predate # them stay honest about what a model without the fields sends. body = {"title": title, "content": content, "next_action": next_action} @@ -24,6 +24,8 @@ def step(title: str, content: str, next_action: str = "continue", body["calc"] = calc if convert: body["convert"] = convert + if derivative: + body["derivative"] = derivative return json.dumps(body) diff --git a/dev/tests/test_selection.py b/dev/tests/test_selection.py index ec21475..28e73c5 100644 --- a/dev/tests/test_selection.py +++ b/dev/tests/test_selection.py @@ -556,3 +556,62 @@ def test_prose_without_claims_is_untouched(self): text = "The capital of France is Paris, established over 2000 years ago." assert repair_sums(text) == (text, []) + + +class TestDerivativeField: + """A derivative the model NAMES rather than works out, like calc and convert. + + Measured need, from the six-domain battery: the model dropped a chain + factor (6912 where the answer is 235824) and read a tangent at the wrong + point. mpeqs.calculus computes the derivative two independent ways whose + tests demand agreement, so what comes back cannot be a matching mistake. + """ + + def test_the_field_is_required_like_its_siblings(self): + from mpe_lkg.backends import STEP_SCHEMA + + assert "derivative" in STEP_SCHEMA["required"] + + @pytest.mark.parametrize(("request_text", "value"), [ + ("d/dx (4*x**2 + 8)**4 at x=3/2", "235824"), + ("derivative of 3*x**2 + 7*x at 4", "31"), + ("d/dx x**3 at x=2", "12"), + ]) + def test_requests_settle_exactly(self, request_text, value): + from mpe_lkg.arithmetic import derivative_request + + text, exact, label = derivative_request(request_text) + assert str(exact) == value + assert label.startswith("derivative of") + + @pytest.mark.parametrize("refused", [ + "d/dx x**x at 2", # not a polynomial; calculus refuses + "d/dx sin(x) at 0", # calls are refused + "derivative of x**2", # no point given + "just words", + ]) + def test_what_cannot_be_settled_is_silence(self, refused): + from mpe_lkg.arithmetic import derivative_request + + assert derivative_request(refused) is None + + def test_the_loop_carries_it_to_the_synthesis(self): + import json as _json + + from mpe_lkg.backends import DeterministicEmbedding, ScriptedChat + from mpe_lkg.reasoning import reason + + script = [ + _json.dumps({"title": "Slope", "content": "Ask for it.", + "calc": "", "calc_of": "", "convert": "", + "derivative": "d/dx 3*x**2 + 7*x at x=4", + "next_action": "final_answer"}), + "The slope is 31.", + ] + events = list(reason("What is the slope at x=4?", + chat=ScriptedChat(script), + embedder=DeterministicEmbedding(24))) + settled = next(e for e in events if e["type"] == "derivative") + assert settled["result"].endswith("= 31") + done = next(e for e in events if e["type"] == "done") + assert done["derivatives"] == 1 diff --git a/src/mpe_lkg/arithmetic.py b/src/mpe_lkg/arithmetic.py index 04fc9ad..f8d4c31 100644 --- a/src/mpe_lkg/arithmetic.py +++ b/src/mpe_lkg/arithmetic.py @@ -370,3 +370,41 @@ def question_conversion(question: str): return None return convert(f"{match.group('value')} {match.group('source')} " f"to {match.group('target')}") + + +# "d/dx (3*x**2+5)**4 at x=1/2", "derivative of x**3 - 4*x at 2". +DERIVATIVE = re.compile( + r"(?:d/dx|derivative\s+of)\s*(?P.+?)\s+at\s*(?:x\s*=\s*)?" + r"(?P-?\d+(?:\.\d+)?(?:\s*/\s*\d+)?)\s*$", + re.IGNORECASE, +) + + +def derivative_request(request: str): + """A derivative the model asked for, settled exactly, or None. + + The same contract as ``convert`` one shelf over: the model NAMES the + derivative and the value comes from mpeqs.calculus, which computes it two + independent ways and whose tests demand they agree. Measured need: on the + calculus battery the model dropped a chain factor (6912 where the answer + is 235824) and read a tangent at the wrong point -- the same + guessed-instead-of-asked failure the calc field closed for arithmetic. + + Returns ``(text, value, label)`` or None; a refusal from the calculus + module -- x**x, an unknown name -- is silence rather than a guess. + """ + match = DERIVATIVE.search(_normalise(str(request or ""))) + if not match: + return None + try: + from mpeqs import calculus + except ImportError: + return None + try: + at = Fraction(match.group("at").replace(" ", "")) + expression = match.group("expr").strip() + value = calculus.derivative_at(expression, at=at) + except Exception: + return None + return (f"d/dx {expression} at x={at} = {readable(value)}", value, + f"derivative of {expression} at x={at}") diff --git a/src/mpe_lkg/backends/_shared.py b/src/mpe_lkg/backends/_shared.py index 6040a11..7709364 100644 --- a/src/mpe_lkg/backends/_shared.py +++ b/src/mpe_lkg/backends/_shared.py @@ -71,9 +71,22 @@ "different values identifies neither. Use only wording " "drawn from this question. Empty string if no calculation.", }, + # Same contract as calc and convert: name it, never work it out. The + # calculus battery measured the model dropping a chain factor and + # reading a tangent at the wrong point -- the guessed-instead-of-asked + # failure this field closes for derivatives. + "derivative": { + "type": "string", + "description": "A derivative this step needs, in one line with the " + "point included: 'd/dx at x='. It " + "is computed exactly and given back to you -- never " + "apply the chain rule yourself. Empty string if the " + "step needs no derivative.", + }, "next_action": {"type": "string", "enum": ["continue", "final_answer"]}, }, - "required": ["title", "content", "calc", "calc_of", "convert", "next_action"], + "required": ["title", "content", "calc", "calc_of", "convert", "derivative", + "next_action"], } diff --git a/src/mpe_lkg/battery/bench.py b/src/mpe_lkg/battery/bench.py index a3257ab..3802fde 100644 --- a/src/mpe_lkg/battery/bench.py +++ b/src/mpe_lkg/battery/bench.py @@ -64,6 +64,8 @@ def run( ): if event["type"] == "calc": calcs.append(f"{event['expression']} = {event['value']}") + elif event["type"] == "derivative": + calcs.append(event["result"]) elif event["type"] == "step": steps += 1 elif event["type"] == "final": diff --git a/src/mpe_lkg/graphdb.py b/src/mpe_lkg/graphdb.py index 3f35c67..4f154ed 100644 --- a/src/mpe_lkg/graphdb.py +++ b/src/mpe_lkg/graphdb.py @@ -92,6 +92,10 @@ def record(self, job) -> None: self.conn.execute( "INSERT INTO facts (run, kind, statement) VALUES (?, 'conversion', ?)", (job.id, str(event.get("result", "")))) + for event in job.of_type("derivative"): + self.conn.execute( + "INSERT INTO facts (run, kind, statement) VALUES (?, 'derivative', ?)", + (job.id, str(event.get("result", "")))) for event in job.of_type("calc"): self.conn.execute( "INSERT INTO facts (run, kind, statement) VALUES (?, 'calculation', ?)", diff --git a/src/mpe_lkg/reasoning.py b/src/mpe_lkg/reasoning.py index 9e13e83..0205303 100644 --- a/src/mpe_lkg/reasoning.py +++ b/src/mpe_lkg/reasoning.py @@ -19,6 +19,7 @@ as_text, convert, correction, + derivative_request, evaluate, product_unit, question_conversion, @@ -78,6 +79,9 @@ "a conversion into steps and never multiply conversion factors together yourself -- " "that is the single most common way this goes wrong. Leave it empty if this question " "involves no units. " + "If a step needs a derivative, put the WHOLE request in a 'derivative' field in one " + "line with the point included, as 'd/dx at x='. It is computed " + "exactly and given back to you -- never apply the chain rule yourself. " "If a step relies on a calculation, ALSO put that calculation in a 'calc' field as a " "bare arithmetic expression with no words and no equals sign, built ONLY from numbers " "that appear in this question. It is evaluated exactly and the result is given back to " @@ -640,7 +644,7 @@ def _synthesise( # was computed exactly -- but an answer of 10080 written beside it. units_block = "" if converted: - units_block = "\n\nThese conversions were done exactly:\n" + "\n".join( + units_block = "\n\nThese were computed exactly:\n" + "\n".join( f" {c}" for c in converted ) + "\nIf one of them answers the question directly, give that number." try: @@ -733,6 +737,7 @@ def reason( # index and title of its step, and only those on the strongest path are shown. settled: list[tuple[int, str, str]] = [] converted: list[str] = [] + derivatives = 0 # Facts that can be OFFERED as the answer: a label saying what the value is, # and the exact value itself. Kept apart from the display strings because the # answer comes from the Fraction, never from text parsed back out of a model. @@ -880,6 +885,29 @@ def graph_payload() -> tuple[dict, dict | None]: "result": text, } + wanted = str(step_json.get("derivative", "")).strip() + if check_arithmetic and wanted: + done = derivative_request(wanted) + if done is not None: + text, exact, label = done + # Like conversions, unfiltered to the synthesis: an exact + # derivative cannot be wrong -- x**x and unknown names + # refuse rather than answer -- so every one is a fact + # worth carrying. + if text not in converted: + converted.append(text) + labelled.append((label, exact)) + derivatives += 1 + if as_text(exact) not in content.replace(",", ""): + content = f"{content} ({text})" + step_json["content"] = content + yield { + "type": "derivative", + "step": step_number, + "request": wanted, + "result": text, + } + calc = str(step_json.get("calc", "")).strip() if check_arithmetic and calc: exact = evaluate(calc) @@ -1081,6 +1109,7 @@ def graph_payload() -> tuple[dict, dict | None]: "sums_checked": sums_checked, "sums_corrected": sums_corrected, "conversions": conversions, + "derivatives": derivatives, # Long numbers in the answer that no tool computed. Zero on the # selection path by construction; this is the synthesis path's score. "unsupported_numbers": unsupported, diff --git a/src/mpe_lkg/templates/index.html b/src/mpe_lkg/templates/index.html index 21d7999..98f09ba 100644 --- a/src/mpe_lkg/templates/index.html +++ b/src/mpe_lkg/templates/index.html @@ -319,6 +319,16 @@

Local Llama Knowledge Graph

network.fit(); } + function exactChip(text, title) { + if (seenSums.has(text)) return; + seenSums.add(text); + if (sums.hidden) { sums.hidden = false; sums.innerHTML = 'Exact:'; } + const chip = document.createElement('span'); + chip.textContent = text; + chip.title = title; + sums.appendChild(chip); + } + function showHints(items) { const box = document.getElementById('hints-box'); box.innerHTML = ''; @@ -670,18 +680,16 @@

Local Llama Knowledge Graph

addExactConversion(data); } else if (data.type === 'calc') { + // Shown once per distinct expression: a repeated sum means + // the model re-derived it, but the same text twice adds + // nothing to the strip. + exactChip(`${data.expression} = ${data.value}`, + `Evaluated exactly at step ${data.step}`); addExactSum(data); - // Deliberately not deduplicated away entirely: a repeated sum - // means the model re-derived it, which is worth seeing. But the - // same expression twice adds nothing, so it is shown once. - if (!seenSums.has(data.expression)) { - seenSums.add(data.expression); - if (sums.hidden) { sums.hidden = false; sums.innerHTML = 'Exact:'; } - const chip = document.createElement('span'); - chip.textContent = `${data.expression} = ${data.value}`; - chip.title = `Evaluated exactly at step ${data.step}`; - sums.appendChild(chip); - } + + } else if (data.type === 'derivative') { + // Exact like a sum: into the strip a reader can check. + exactChip(data.result, `Computed exactly at step ${data.step}`); } else if (data.type === 'hints') { // From OTHER sessions, opt-in, and drawn apart: a hint must