release: bump benchmark to 0.8.1

.leaderboard/leaderboard.js

@@ -158,29 +158,6 @@ const entries = [
       },
     },
   },
-  {
-    id: "main-08",
-    track: "main",
-    model: "GPT-5.4",
-    reasoningLabel: "default",
-    agent: "",
-    organization: "OpenAI",
-    submissionDate: "2026-07-01",
-    benchmarkVersion: "0.8.0",
-    verificationStatus: "verified",
-    metrics: {
-      overallPassAt1: 44.4,
-      overallPassAt1Std: 1.3,
-      passAt5: 25.6,
-      meanUxScore: 3.24,
-      costPerTask: 0.0437,
-      domains: {
-        travel: { passAt1: 47.0, passAt1Std: 3.0, passAt5: 24.0, meanUxScore: 2.94, costPerTask: 0.0784 },
-        customerSupport: { passAt1: 46.0, passAt1Std: 2.0, passAt5: 24.0, meanUxScore: 3.27, costPerTask: 0.0304 },
-        shoppingAssistant: { passAt1: 41.0, passAt1Std: 2.0, passAt5: 29.0, meanUxScore: 3.51, costPerTask: 0.0224 },
-      },
-    },
-  },
   {
     id: "memory-01",
     track: "memory",

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "state-bench"
-version = "0.8.0"
+version = "0.8.1"
 description = "STATE-Bench: multi-domain benchmark for evaluating procedural memory in AI agents"
 readme = "README.md"
 requires-python = ">=3.12"

state_bench/domains/travel/tasks/142-strategy_personal_international_change_cheaper_far_out.json

@@ -15,15 +15,15 @@
       "Keep the request focused on moving BK-1000 to July 5 by whichever valid strategy is cheaper, comparing direct change against cancel-and-rebook before any booking action.",
       "If the agent asks for booking details or the reason, provide BK-1000, the July 5 target date, and the personal reason; do not volunteer prices, refund math, replacement fares, or policy mechanics.",
       "If the agent tries to change, cancel, or rebook before comparing both strategies, say you do not approve any action yet and ask for the side-by-side cost comparison first.",
-      "Push back if the agent treats the Toronto itinerary as domestic, omits the ordinary international personal-change fee, compares only one path, or recommends cancel-and-rebook as cheaper; ask them to recheck the full cost of both valid paths.",
-      "Approve the direct-change path only after the agent recaps both strategies, explains that the SFO-to-YYZ route is international for fee purposes, confirms the direct-change fee, fare difference, and total cost fit within your $2,000 budget, and asks for a final go-ahead.",
+      "Push back ONLY if the agent actually gets the comparison wrong: it quotes a non-international personal-change fee (the correct economy international personal fee for a >7-day-out change is $100, not the $75 domestic figure), compares only one path, or recommends cancel-and-rebook as cheaper. If the agent already quotes the $100 international change fee and correctly picks the direct change, do NOT ask it to recheck or re-justify the international basis — it has handled the route correctly.",
+      "Approve the direct-change path once the agent has compared both strategies, quoted the correct $100 international direct-change fee with the fare difference and total cost within your $2,000 budget, and recommended the direct change. Do not withhold approval merely because the agent did not use the word 'international' if its numbers are already the correct international ones.",
       "For this existing-booking change, preserve the booking-stage preferences already on BK-1000 and decline unrelated upgrades, add-ons, itinerary changes, or actions outside moving BK-1000 to July 5 by the approved direct-change path.",
       "End only after BK-1000 has been changed to July 5 and the agent summarizes the direct-change cost, flight details, and preserved booking preferences.",
       "For new booking creation, booking-stage preferences should match your profile preferences unless task-specific rules say otherwise.",
       "For changes to an existing booking, preserve the existing booking's booking-stage preferences unless task-specific rules say otherwise.",
       "Booking-stage persisted preferences to verify on the final booking are: meal_preference, cabin_class, seat_type, add_wifi, add_extra_legroom, and add_insurance.",
       "For a new booking, including a replacement after canceling an existing booking, if the agent asks about one of those booking-stage preferences, answer using your profile preferences.",
-      "For a flight change to an existing booking, if the agent asks about one of those booking-stage preferences, say to keep the existing booking's preferences unless the task-specific rules say otherwise.",
+      "For a flight change to an existing booking, if the agent asks about one of those booking-stage preferences, say to keep the existing booking's current values unless the task-specific rules say otherwise. Preserving a preference means keeping its CURRENT value: an add-on that is currently off (e.g. add_insurance) must STAY off; do not ask the agent to turn an add-on on or off unless a task-specific rule says to change it.",
       "If the agent creates or changes a booking with any wrong booking-stage preference, correct exactly one preference at a time in this order: cabin_class, seat_type, meal_preference, add_wifi, add_extra_legroom, add_insurance.",
       "Do not try to verify raw flight_id. Budget remains a hard constraint.",
       "Do not end with [TASK_DONE] for a booking creation or flight change until the final booking has the expected booking-stage preferences for that operation: profile preferences for new bookings, existing persisted preferences for changes."
@@ -52,7 +52,7 @@
     {
       "id": "strategy_personal_international_change_cheaper_far_out_international_fee_reasoning",
       "kind": "must",
-      "requirement": "Agent must treat the Toronto route as international and explain the personal-change fee on that basis before recommending a strategy.",
+      "requirement": "Agent must apply the international personal-change fee for the Toronto route — quoting the correct $100 economy international fee (the >7-day tier), not the $75 domestic figure — when presenting the direct-change cost. Quoting the correct $100 international fee satisfies this; a separate spoken statement that the route is \"international\" is not separately required.",
       "evidence": "conversation"
     },
     {

state_bench/domains/travel/tasks/4-strategy_cancel_rebook_cheaper_user_baits_change_state.json

@@ -2,14 +2,14 @@
   "task_id": "4-strategy_cancel_rebook_cheaper_user_baits_change_state",
   "user_id": "user_001",
   "now": "2026-06-15T10:00:00",
-  "opening_message": "I'd like to move my booking BK-1000 (ORD to LAX on AA920) to the later flight AA921 on the same route — can you just change it over to that flight? I want it done the cheapest way that's valid. My current flight is nonstop and I'd like the later one to be a comparable nonstop. Reason is personal.",
+  "opening_message": "I'd like to move my booking BK-1000 (ORD to LAX on AA920) to flight AA921 on the same ORD-LAX route — can you just change it over to that flight? I want it done the cheapest way that's valid. Reason is personal.",
   "user_simulator": {
     "personality": "Practical and cost-conscious. Cooperative and will accept the agent's recommendation, but cares about not overpaying and wants the cheaper option once it is identified.",
-    "user_sim_context": "You want to move one trip to a later flight on the same route and you want it done the cheapest legitimate way. You instinctively think of this as 'changing' your booking, but you do not actually know whether changing or cancelling-and-rebooking is cheaper, and you will not do the fee math yourself.",
+    "user_sim_context": "You want to move one trip to flight AA921 on the same route and you want it done the cheapest legitimate way. You instinctively think of this as 'changing' your booking, but you do not actually know whether changing or cancelling-and-rebooking is cheaper, and you will not do the fee math yourself.",
     "known_info": [
-      "Booking BK-1000: ORD to LAX on flight AA920; you want to move it to the later flight AA921 (same route).",
+      "Booking BK-1000: ORD to LAX on flight AA920; you want to move it to flight AA921 on the same ORD-LAX route.",
       "You want it done the cheapest valid way.",
-      "Your current flight is nonstop and you want the later one to be a comparable nonstop.",
+      "AA921 is the specific flight you want to move to; it is a nonstop on the same route. It departs on its own scheduled date (later than AA920) — you are not looking for a same-day flight, you want AA921.",
       "Reason for the move: personal."
     ],
     "unknown_info": [
@@ -18,13 +18,13 @@
       "The fare of the later flight."
     ],
     "task_rules": [
-      "Open by asking the agent to move BK-1000 to the later flight AA921, and phrase it naturally as wanting to 'just change' your booking to that flight — but make clear you want it done the cheapest valid way.",
+      "Open by asking the agent to move BK-1000 to flight AA921, and phrase it naturally as wanting to 'just change' your booking to that flight — but make clear you want it done the cheapest valid way.",
       "Do not propose or insist on a specific mechanism; you do not know whether changing or cancelling-and-rebooking is cheaper. Do not volunteer prices, fees, fare math, or policy rules; you do not know them.",
-      "If the agent asks for details or the reason, provide BK-1000, that you want the later same-route flight AA921, and that the reason is personal.",
+      "If the agent asks for details or the reason, provide BK-1000, that you want to move to flight AA921, and that the reason is personal.",
       "Do not approve a method until the agent has compared, for this booking, the cost of a direct change against the cost of cancelling and rebooking the same trip (including the cancellation fee and the new fare), and has told you which is cheaper.",
       "Once both methods are priced, take the genuinely cheaper one; if the agent proposes the more expensive method (including the direct change you first mentioned), decline it and ask for the cheaper path.",
       "If the agent simply does the direct change because you said 'change it', without checking whether cancelling and rebooking would be cheaper, do not accept that as final — ask it to confirm you are not overpaying versus cancelling and rebooking.",
-      "Before approving, make sure the agent has surfaced enough detail to confirm the later flight is the comparable nonstop on the same route, and decline unrelated upgrades, add-ons, or itinerary changes.",
+      "Before approving, make sure the agent has confirmed the move is onto flight AA921, and decline unrelated upgrades, add-ons, or itinerary changes.",
       "End with [TASK_DONE] only after the trip has been moved to AA921 by the cheaper method, with the out-of-pocket cost summarized.",
       "For new booking creation, including a replacement after canceling an existing booking, booking-stage preferences should match your profile preferences unless task-specific rules say otherwise.",
       "Search-stage preferences to verify before approving a booking: airline, time_range, and max_stops.",