From 24a7521eeea8b7f79fc46a57cdf01132eecba7ef Mon Sep 17 00:00:00 2001 From: Abhinaysai Kamineni <66816045+askmy-stack@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:38:58 -0400 Subject: [PATCH 1/3] Add five Milestone-1 normal tasks with success validators. Register API lookup/update and SQLite aggregation tasks; extend CustomerApiEnv with submit_answer for expected_answer checks. Co-authored-by: Cursor --- .../environments/customer_api/env.py | 23 +++++- agentfailbench/tasks/__init__.py | 1 + agentfailbench/tasks/normal/__init__.py | 60 +++++++++++++++ docs/roadmap.md | 4 +- tests/unit/test_normal_tasks.py | 75 +++++++++++++++++++ 5 files changed, 160 insertions(+), 3 deletions(-) create mode 100644 agentfailbench/tasks/normal/__init__.py create mode 100644 tests/unit/test_normal_tasks.py diff --git a/agentfailbench/environments/customer_api/env.py b/agentfailbench/environments/customer_api/env.py index 4003b2e..7edde2f 100644 --- a/agentfailbench/environments/customer_api/env.py +++ b/agentfailbench/environments/customer_api/env.py @@ -66,6 +66,7 @@ class CustomerApiEnv: contract_version: ContractVersion = "v1" store: dict[str, Any] = field(default_factory=dict) call_count: int = 0 + _submitted: object | None = None def __post_init__(self) -> None: if not self.store: @@ -95,7 +96,9 @@ def set_contract(self, version: ContractVersion) -> None: def reset(self) -> None: self.call_count = 0 + self._submitted = None self.contract_version = "v1" + self._submitted = None self.store = { "customers": { self.task.customer_id: { @@ -192,10 +195,28 @@ def step(self, action: Action) -> EnvObservation: "plan_id_meaning": plan_id_meaning(self.contract_version), }, ) + if name == "submit_answer": + self._submitted = args.get("value") + return EnvObservation(success=True, data={"submitted": self._submitted}) + return EnvObservation(success=False, status_code=400, error=f"unknown_action:{name}") def validate_success(self) -> bool: - """True if subscription billing_plan_code matches the task target.""" + """True when the task objective is achieved. + + Subscription updates check ``billing_plan_code``. Lookup / transform tasks + set ``TaskSpec.expected_answer`` and use ``submit_answer``. + """ + expected = self.task.expected_answer + if expected is not None: + if self._submitted is None: + return False + if isinstance(expected, (int, float)) and not isinstance(expected, bool): + try: + return float(self._submitted) == float(expected) + except (TypeError, ValueError): + return False + return str(self._submitted) == str(expected) sub = self.store["subscriptions"].get(self.task.customer_id) if sub is None: return False diff --git a/agentfailbench/tasks/__init__.py b/agentfailbench/tasks/__init__.py index e69de29..1f58c3e 100644 --- a/agentfailbench/tasks/__init__.py +++ b/agentfailbench/tasks/__init__.py @@ -0,0 +1 @@ +"""AgentFailBench task catalogs.""" diff --git a/agentfailbench/tasks/normal/__init__.py b/agentfailbench/tasks/normal/__init__.py new file mode 100644 index 0000000..b93411b --- /dev/null +++ b/agentfailbench/tasks/normal/__init__.py @@ -0,0 +1,60 @@ +"""Milestone 1 normal (non-failure) task catalog. + +Five tasks runnable without failure injection, each with a deterministic +``TaskEnvironment.validate_success`` check. +""" + +from __future__ import annotations + +from runtime.schemas.episode import TaskSpec + +NORMAL_TASK_SPECS: dict[str, TaskSpec] = { + "update_customer_subscription": TaskSpec( + task_id="normal-update-subscription", + objective="update_customer_subscription", + environment="customer_service_api", + target_plan_code="GOLD_ANNUAL", + expected_steps=8, + ), + "lookup_customer_name": TaskSpec( + task_id="normal-lookup-customer-name", + objective="lookup_customer_name", + environment="customer_service_api", + expected_answer="Ada Lovelace", + expected_steps=2, + ), + "lookup_subscription_plan": TaskSpec( + task_id="normal-lookup-subscription-plan", + objective="lookup_subscription_plan", + environment="customer_service_api", + expected_answer="BASIC_MONTHLY", + expected_steps=2, + ), + "sum_pending_order_amounts": TaskSpec( + task_id="normal-sum-pending-orders", + objective="sum_pending_order_amounts", + environment="sqlite_investigation", + expected_answer=15300, + expected_steps=4, + ), + "count_enterprise_customers": TaskSpec( + task_id="normal-count-enterprise-customers", + objective="count_enterprise_customers", + environment="sqlite_investigation", + expected_answer=2, + expected_steps=3, + ), +} + + +def list_normal_task_ids() -> list[str]: + """Return the five Milestone-1 normal task ids.""" + return sorted(NORMAL_TASK_SPECS) + + +def get_normal_task(task_id: str) -> TaskSpec: + if task_id not in NORMAL_TASK_SPECS: + raise KeyError( + f"Unknown normal task {task_id!r}. Available: {', '.join(list_normal_task_ids())}" + ) + return NORMAL_TASK_SPECS[task_id] diff --git a/docs/roadmap.md b/docs/roadmap.md index 1299c73..4c43d62 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1140,8 +1140,8 @@ The benchmark must not depend too heavily on one framework. - [ ] Implement database task environment - [ ] Define task interface - [ ] Define action and observation schemas -- [ ] Implement five normal tasks -- [ ] Add task-success validators +- [x] Implement five normal tasks (`agentfailbench/tasks/normal`) +- [x] Add task-success validators ### Milestone 2 — Failure injection diff --git a/tests/unit/test_normal_tasks.py b/tests/unit/test_normal_tasks.py new file mode 100644 index 0000000..4831bb8 --- /dev/null +++ b/tests/unit/test_normal_tasks.py @@ -0,0 +1,75 @@ +"""Milestone 1: five normal tasks with success validators.""" + +from __future__ import annotations + +from agentfailbench.agents.scripted import ScriptedApiAgent +from agentfailbench.environments.customer_api.env import CustomerApiEnv +from agentfailbench.environments.database.env import DatabaseEnv +from agentfailbench.tasks.normal import NORMAL_TASK_SPECS, list_normal_task_ids +from runtime.schemas.episode import Action + + +def test_five_normal_tasks_registered() -> None: + ids = list_normal_task_ids() + assert len(ids) == 5 + assert set(ids) == set(NORMAL_TASK_SPECS) + + +def test_update_customer_subscription_succeeds() -> None: + task = NORMAL_TASK_SPECS["update_customer_subscription"] + env = CustomerApiEnv(task=task) + ScriptedApiAgent(env=env).run_until_done() + assert env.validate_success() + + +def test_lookup_customer_name_succeeds() -> None: + task = NORMAL_TASK_SPECS["lookup_customer_name"] + env = CustomerApiEnv(task=task) + env.step(Action(name="get_customer", arguments={"customer_id": task.customer_id})) + env.step(Action(name="submit_answer", arguments={"value": "Ada Lovelace"})) + assert env.validate_success() + + +def test_lookup_subscription_plan_succeeds() -> None: + task = NORMAL_TASK_SPECS["lookup_subscription_plan"] + env = CustomerApiEnv(task=task) + env.step(Action(name="get_subscription", arguments={"customer_id": task.customer_id})) + env.step(Action(name="submit_answer", arguments={"value": "BASIC_MONTHLY"})) + assert env.validate_success() + + +def test_sum_pending_order_amounts_succeeds() -> None: + task = NORMAL_TASK_SPECS["sum_pending_order_amounts"] + env = DatabaseEnv(task=task) + env.step( + Action( + name="execute_sql", + arguments={ + "sql": "SELECT SUM(amount_cents) AS total FROM orders WHERE status = 'pending'" + }, + ) + ) + env.step(Action(name="submit_answer", arguments={"value": 15300})) + assert env.validate_success() + + +def test_count_enterprise_customers_succeeds() -> None: + task = NORMAL_TASK_SPECS["count_enterprise_customers"] + env = DatabaseEnv(task=task) + env.step( + Action( + name="execute_sql", + arguments={ + "sql": "SELECT COUNT(*) AS n FROM customers WHERE segment = 'enterprise'" + }, + ) + ) + env.step(Action(name="submit_answer", arguments={"value": 2})) + assert env.validate_success() + + +def test_lookup_wrong_answer_fails_validator() -> None: + task = NORMAL_TASK_SPECS["lookup_customer_name"] + env = CustomerApiEnv(task=task) + env.step(Action(name="submit_answer", arguments={"value": "Wrong Name"})) + assert not env.validate_success() From b3445d713a1c94bf2088bcc84a3e8c6f90afbdf4 Mon Sep 17 00:00:00 2001 From: Abhinaysai Kamineni <66816045+askmy-stack@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:56:01 -0400 Subject: [PATCH 2/3] Apply ruff format to Milestone-1 normal task files. Co-authored-by: Cursor --- tests/unit/test_normal_tasks.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/test_normal_tasks.py b/tests/unit/test_normal_tasks.py index 4831bb8..ceb588e 100644 --- a/tests/unit/test_normal_tasks.py +++ b/tests/unit/test_normal_tasks.py @@ -59,9 +59,7 @@ def test_count_enterprise_customers_succeeds() -> None: env.step( Action( name="execute_sql", - arguments={ - "sql": "SELECT COUNT(*) AS n FROM customers WHERE segment = 'enterprise'" - }, + arguments={"sql": "SELECT COUNT(*) AS n FROM customers WHERE segment = 'enterprise'"}, ) ) env.step(Action(name="submit_answer", arguments={"value": 2})) From 8072c04cbcc2e618c2f8f6ff1210be83ff9682f3 Mon Sep 17 00:00:00 2001 From: Abhinaysai Kamineni <66816045+askmy-stack@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:57:27 -0400 Subject: [PATCH 3/3] Fix mypy arg-type on CustomerApiEnv expected_answer compare. Co-authored-by: Cursor --- agentfailbench/environments/customer_api/env.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/agentfailbench/environments/customer_api/env.py b/agentfailbench/environments/customer_api/env.py index 7edde2f..6377c83 100644 --- a/agentfailbench/environments/customer_api/env.py +++ b/agentfailbench/environments/customer_api/env.py @@ -209,14 +209,17 @@ def validate_success(self) -> bool: """ expected = self.task.expected_answer if expected is not None: - if self._submitted is None: + submitted = self._submitted + if submitted is None: return False if isinstance(expected, (int, float)) and not isinstance(expected, bool): + if not isinstance(submitted, (int, float, str)): + return False try: - return float(self._submitted) == float(expected) + return float(submitted) == float(expected) except (TypeError, ValueError): return False - return str(self._submitted) == str(expected) + return str(submitted) == str(expected) sub = self.store["subscriptions"].get(self.task.customer_id) if sub is None: return False