Conversation-level judging and joint-vs-decomposed analyzer

humanebench/joint_scorer.py

@@ -115,11 +115,28 @@ def principle_order_for(
 """
 
 
-def build_joint_prompt(prompt: str, response: str, patterns=None) -> str:
+def build_joint_prompt(prompt: str, response: str, patterns=None,
+                       template: str | None = None) -> str:
     """Build the joint prompt. `patterns` sets the principle ORDER (and is what
     the per-turn randomization varies); None uses the canonical order. Both the
     numbered principle list and the JSON output template follow this order, so
-    the model never sees a canonical ordering leak back in through the template."""
+    the model never sees a canonical ordering leak back in through the template.
+    `template` overrides JOINT_TEMPLATE_HEADER (same format slots) — used by the
+    conversation-level variant, whose only edit is the framing sentence."""
+    if template is None:
+        template = JOINT_TEMPLATE_HEADER
+    # A slot-less template would not error in str.format — it would silently
+    # drop the principle list, output contract, or conversation and produce
+    # plausible but meaningless judge output.
+    missing = [
+        slot
+        for slot in ("{principles_block}", "{json_block}", "{prompt}", "{response}")
+        if slot not in template
+    ]
+    if missing:
+        raise ValueError(
+            f"joint template is missing required format slot(s): {missing}"
+        )
     if patterns is None:
         patterns = humanebench.humane_patterns.resolve_humane_patterns(None)
     principles_block = "\n\n".join(
@@ -131,7 +148,7 @@ def build_joint_prompt(prompt: str, response: str, patterns=None) -> str:
         f'"reasoning": "<str> your reasoning"}},'
         for p in patterns
     )
-    return JOINT_TEMPLATE_HEADER.format(
+    return template.format(
         principles_block=principles_block,
         json_block=json_block,
         prompt=prompt,
@@ -244,6 +261,7 @@ def joint_overseer(
     system_message: str = OVERSEER_SYSTEM_MESSAGE,
     score_attempts: int = 3,
     principle_order_salt: str = PRINCIPLE_ORDER_SALT,
+    template: str | None = None,
 ):
     slugs = list(humanebench.humane_patterns.HUMANE_PATTERNS.keys())
 
@@ -256,7 +274,7 @@ async def score(state: TaskState, target: Target):
         order = principle_order_for(state.sample_id, principle_order_salt)
         principle_order = [p.id for p in order]
         score_prompt = build_joint_prompt(
-            state.input_text, state.output.completion, order
+            state.input_text, state.output.completion, order, template=template
         )
 
         model_names: list[str] = []

humanebench/scorer.py

@@ -151,7 +151,22 @@ def overseer(
     temperature: float = 0.0,
     system_message: str = OVERSEER_SYSTEM_MESSAGE,
     score_attempts: int = 3,
+    template: str = OVERSEER_TEMPLATE,
 ):
+    # Fail fast (at task construction, before any judge spend): a template
+    # missing a slot would not error in str.format — it would silently render
+    # a prompt without the principle or the response and produce plausible
+    # but meaningless scores.
+    missing = [
+        slot
+        for slot in ("{issue_display_name}", "{issue_description}",
+                     "{prompt}", "{response}")
+        if slot not in template
+    ]
+    if missing:
+        raise ValueError(
+            f"overseer template is missing required format slot(s): {missing}"
+        )
     issues = {
         humane_pattern.id: humane_pattern
         for humane_pattern in humanebench.humane_patterns.resolve_humane_patterns(humane_patterns)
@@ -165,7 +180,7 @@ async def score(state: TaskState, target: Target):
         models = [inspect_ai.model.get_model(m) for m in models]
 
         issue = issues[target.text]
-        score_prompt = OVERSEER_TEMPLATE.format(
+        score_prompt = template.format(
             issue_key=issue.id,
             issue_display_name=issue.display_name,
             issue_description=issue.description,