diff --git a/agentfailbench/environments/customer_api/env.py b/agentfailbench/environments/customer_api/env.py index 4003b2e..6377c83 100644 --- a/agentfailbench/environments/customer_api/env.py +++ b/agentfailbench/environments/customer_api/env.py @@ -66,6 +66,7 @@ class CustomerApiEnv: contract_version: ContractVersion = "v1" store: dict[str, Any] = field(default_factory=dict) call_count: int = 0 + _submitted: object | None = None def __post_init__(self) -> None: if not self.store: @@ -95,7 +96,9 @@ def set_contract(self, version: ContractVersion) -> None: def reset(self) -> None: self.call_count = 0 + self._submitted = None self.contract_version = "v1" + self._submitted = None self.store = { "customers": { self.task.customer_id: { @@ -192,10 +195,31 @@ def step(self, action: Action) -> EnvObservation: "plan_id_meaning": plan_id_meaning(self.contract_version), }, ) + if name == "submit_answer": + self._submitted = args.get("value") + return EnvObservation(success=True, data={"submitted": self._submitted}) + return EnvObservation(success=False, status_code=400, error=f"unknown_action:{name}") def validate_success(self) -> bool: - """True if subscription billing_plan_code matches the task target.""" + """True when the task objective is achieved. + + Subscription updates check ``billing_plan_code``. Lookup / transform tasks + set ``TaskSpec.expected_answer`` and use ``submit_answer``. + """ + expected = self.task.expected_answer + if expected is not None: + submitted = self._submitted + if submitted is None: + return False + if isinstance(expected, (int, float)) and not isinstance(expected, bool): + if not isinstance(submitted, (int, float, str)): + return False + try: + return float(submitted) == float(expected) + except (TypeError, ValueError): + return False + return str(submitted) == str(expected) sub = self.store["subscriptions"].get(self.task.customer_id) if sub is None: return False diff --git a/agentfailbench/tasks/__init__.py b/agentfailbench/tasks/__init__.py index e69de29..1f58c3e 100644 --- a/agentfailbench/tasks/__init__.py +++ b/agentfailbench/tasks/__init__.py @@ -0,0 +1 @@ +"""AgentFailBench task catalogs.""" diff --git a/agentfailbench/tasks/normal/__init__.py b/agentfailbench/tasks/normal/__init__.py new file mode 100644 index 0000000..b93411b --- /dev/null +++ b/agentfailbench/tasks/normal/__init__.py @@ -0,0 +1,60 @@ +"""Milestone 1 normal (non-failure) task catalog. + +Five tasks runnable without failure injection, each with a deterministic +``TaskEnvironment.validate_success`` check. +""" + +from __future__ import annotations + +from runtime.schemas.episode import TaskSpec + +NORMAL_TASK_SPECS: dict[str, TaskSpec] = { + "update_customer_subscription": TaskSpec( + task_id="normal-update-subscription", + objective="update_customer_subscription", + environment="customer_service_api", + target_plan_code="GOLD_ANNUAL", + expected_steps=8, + ), + "lookup_customer_name": TaskSpec( + task_id="normal-lookup-customer-name", + objective="lookup_customer_name", + environment="customer_service_api", + expected_answer="Ada Lovelace", + expected_steps=2, + ), + "lookup_subscription_plan": TaskSpec( + task_id="normal-lookup-subscription-plan", + objective="lookup_subscription_plan", + environment="customer_service_api", + expected_answer="BASIC_MONTHLY", + expected_steps=2, + ), + "sum_pending_order_amounts": TaskSpec( + task_id="normal-sum-pending-orders", + objective="sum_pending_order_amounts", + environment="sqlite_investigation", + expected_answer=15300, + expected_steps=4, + ), + "count_enterprise_customers": TaskSpec( + task_id="normal-count-enterprise-customers", + objective="count_enterprise_customers", + environment="sqlite_investigation", + expected_answer=2, + expected_steps=3, + ), +} + + +def list_normal_task_ids() -> list[str]: + """Return the five Milestone-1 normal task ids.""" + return sorted(NORMAL_TASK_SPECS) + + +def get_normal_task(task_id: str) -> TaskSpec: + if task_id not in NORMAL_TASK_SPECS: + raise KeyError( + f"Unknown normal task {task_id!r}. Available: {', '.join(list_normal_task_ids())}" + ) + return NORMAL_TASK_SPECS[task_id] diff --git a/docs/roadmap.md b/docs/roadmap.md index 1299c73..4c43d62 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1140,8 +1140,8 @@ The benchmark must not depend too heavily on one framework. - [ ] Implement database task environment - [ ] Define task interface - [ ] Define action and observation schemas -- [ ] Implement five normal tasks -- [ ] Add task-success validators +- [x] Implement five normal tasks (`agentfailbench/tasks/normal`) +- [x] Add task-success validators ### Milestone 2 — Failure injection diff --git a/tests/unit/test_normal_tasks.py b/tests/unit/test_normal_tasks.py new file mode 100644 index 0000000..ceb588e --- /dev/null +++ b/tests/unit/test_normal_tasks.py @@ -0,0 +1,73 @@ +"""Milestone 1: five normal tasks with success validators.""" + +from __future__ import annotations + +from agentfailbench.agents.scripted import ScriptedApiAgent +from agentfailbench.environments.customer_api.env import CustomerApiEnv +from agentfailbench.environments.database.env import DatabaseEnv +from agentfailbench.tasks.normal import NORMAL_TASK_SPECS, list_normal_task_ids +from runtime.schemas.episode import Action + + +def test_five_normal_tasks_registered() -> None: + ids = list_normal_task_ids() + assert len(ids) == 5 + assert set(ids) == set(NORMAL_TASK_SPECS) + + +def test_update_customer_subscription_succeeds() -> None: + task = NORMAL_TASK_SPECS["update_customer_subscription"] + env = CustomerApiEnv(task=task) + ScriptedApiAgent(env=env).run_until_done() + assert env.validate_success() + + +def test_lookup_customer_name_succeeds() -> None: + task = NORMAL_TASK_SPECS["lookup_customer_name"] + env = CustomerApiEnv(task=task) + env.step(Action(name="get_customer", arguments={"customer_id": task.customer_id})) + env.step(Action(name="submit_answer", arguments={"value": "Ada Lovelace"})) + assert env.validate_success() + + +def test_lookup_subscription_plan_succeeds() -> None: + task = NORMAL_TASK_SPECS["lookup_subscription_plan"] + env = CustomerApiEnv(task=task) + env.step(Action(name="get_subscription", arguments={"customer_id": task.customer_id})) + env.step(Action(name="submit_answer", arguments={"value": "BASIC_MONTHLY"})) + assert env.validate_success() + + +def test_sum_pending_order_amounts_succeeds() -> None: + task = NORMAL_TASK_SPECS["sum_pending_order_amounts"] + env = DatabaseEnv(task=task) + env.step( + Action( + name="execute_sql", + arguments={ + "sql": "SELECT SUM(amount_cents) AS total FROM orders WHERE status = 'pending'" + }, + ) + ) + env.step(Action(name="submit_answer", arguments={"value": 15300})) + assert env.validate_success() + + +def test_count_enterprise_customers_succeeds() -> None: + task = NORMAL_TASK_SPECS["count_enterprise_customers"] + env = DatabaseEnv(task=task) + env.step( + Action( + name="execute_sql", + arguments={"sql": "SELECT COUNT(*) AS n FROM customers WHERE segment = 'enterprise'"}, + ) + ) + env.step(Action(name="submit_answer", arguments={"value": 2})) + assert env.validate_success() + + +def test_lookup_wrong_answer_fails_validator() -> None: + task = NORMAL_TASK_SPECS["lookup_customer_name"] + env = CustomerApiEnv(task=task) + env.step(Action(name="submit_answer", arguments={"value": "Wrong Name"})) + assert not env.validate_success()