Milestone 1: five normal tasks with success validators

agentfailbench/environments/customer_api/env.py

@@ -66,6 +66,7 @@ class CustomerApiEnv:
     contract_version: ContractVersion = "v1"
     store: dict[str, Any] = field(default_factory=dict)
     call_count: int = 0
+    _submitted: object | None = None
 
     def __post_init__(self) -> None:
         if not self.store:
@@ -95,7 +96,9 @@ def set_contract(self, version: ContractVersion) -> None:
 
     def reset(self) -> None:
         self.call_count = 0
+        self._submitted = None
         self.contract_version = "v1"
+        self._submitted = None
         self.store = {
             "customers": {
                 self.task.customer_id: {
@@ -192,10 +195,31 @@ def step(self, action: Action) -> EnvObservation:
                     "plan_id_meaning": plan_id_meaning(self.contract_version),
                 },
             )
+        if name == "submit_answer":
+            self._submitted = args.get("value")
+            return EnvObservation(success=True, data={"submitted": self._submitted})
+
         return EnvObservation(success=False, status_code=400, error=f"unknown_action:{name}")
 
     def validate_success(self) -> bool:
-        """True if subscription billing_plan_code matches the task target."""
+        """True when the task objective is achieved.
+
+        Subscription updates check ``billing_plan_code``. Lookup / transform tasks
+        set ``TaskSpec.expected_answer`` and use ``submit_answer``.
+        """
+        expected = self.task.expected_answer
+        if expected is not None:
+            submitted = self._submitted
+            if submitted is None:
+                return False
+            if isinstance(expected, (int, float)) and not isinstance(expected, bool):
+                if not isinstance(submitted, (int, float, str)):
+                    return False
+                try:
+                    return float(submitted) == float(expected)
+                except (TypeError, ValueError):
+                    return False
+            return str(submitted) == str(expected)
         sub = self.store["subscriptions"].get(self.task.customer_id)
         if sub is None:
             return False

agentfailbench/tasks/__init__.py

@@ -0,0 +1 @@
+"""AgentFailBench task catalogs."""

agentfailbench/tasks/normal/__init__.py

@@ -0,0 +1,60 @@
+"""Milestone 1 normal (non-failure) task catalog.
+
+Five tasks runnable without failure injection, each with a deterministic
+``TaskEnvironment.validate_success`` check.
+"""
+
+from __future__ import annotations
+
+from runtime.schemas.episode import TaskSpec
+
+NORMAL_TASK_SPECS: dict[str, TaskSpec] = {
+    "update_customer_subscription": TaskSpec(
+        task_id="normal-update-subscription",
+        objective="update_customer_subscription",
+        environment="customer_service_api",
+        target_plan_code="GOLD_ANNUAL",
+        expected_steps=8,
+    ),
+    "lookup_customer_name": TaskSpec(
+        task_id="normal-lookup-customer-name",
+        objective="lookup_customer_name",
+        environment="customer_service_api",
+        expected_answer="Ada Lovelace",
+        expected_steps=2,
+    ),
+    "lookup_subscription_plan": TaskSpec(
+        task_id="normal-lookup-subscription-plan",
+        objective="lookup_subscription_plan",
+        environment="customer_service_api",
+        expected_answer="BASIC_MONTHLY",
+        expected_steps=2,
+    ),
+    "sum_pending_order_amounts": TaskSpec(
+        task_id="normal-sum-pending-orders",
+        objective="sum_pending_order_amounts",
+        environment="sqlite_investigation",
+        expected_answer=15300,
+        expected_steps=4,
+    ),
+    "count_enterprise_customers": TaskSpec(
+        task_id="normal-count-enterprise-customers",
+        objective="count_enterprise_customers",
+        environment="sqlite_investigation",
+        expected_answer=2,
+        expected_steps=3,
+    ),
+}
+
+
+def list_normal_task_ids() -> list[str]:
+    """Return the five Milestone-1 normal task ids."""
+    return sorted(NORMAL_TASK_SPECS)
+
+
+def get_normal_task(task_id: str) -> TaskSpec:
+    if task_id not in NORMAL_TASK_SPECS:
+        raise KeyError(
+            f"Unknown normal task {task_id!r}. Available: {', '.join(list_normal_task_ids())}"
+        )
+    return NORMAL_TASK_SPECS[task_id]

docs/roadmap.md

@@ -1140,8 +1140,8 @@ The benchmark must not depend too heavily on one framework.
 - [ ] Implement database task environment
 - [ ] Define task interface
 - [ ] Define action and observation schemas
-- [ ] Implement five normal tasks
-- [ ] Add task-success validators
+- [x] Implement five normal tasks (`agentfailbench/tasks/normal`)
+- [x] Add task-success validators
 
 ### Milestone 2 — Failure injection
 

tests/unit/test_normal_tasks.py

@@ -0,0 +1,73 @@
+"""Milestone 1: five normal tasks with success validators."""
+
+from __future__ import annotations
+
+from agentfailbench.agents.scripted import ScriptedApiAgent
+from agentfailbench.environments.customer_api.env import CustomerApiEnv
+from agentfailbench.environments.database.env import DatabaseEnv
+from agentfailbench.tasks.normal import NORMAL_TASK_SPECS, list_normal_task_ids
+from runtime.schemas.episode import Action
+
+
+def test_five_normal_tasks_registered() -> None:
+    ids = list_normal_task_ids()
+    assert len(ids) == 5
+    assert set(ids) == set(NORMAL_TASK_SPECS)
+
+
+def test_update_customer_subscription_succeeds() -> None:
+    task = NORMAL_TASK_SPECS["update_customer_subscription"]
+    env = CustomerApiEnv(task=task)
+    ScriptedApiAgent(env=env).run_until_done()
+    assert env.validate_success()
+
+
+def test_lookup_customer_name_succeeds() -> None:
+    task = NORMAL_TASK_SPECS["lookup_customer_name"]
+    env = CustomerApiEnv(task=task)
+    env.step(Action(name="get_customer", arguments={"customer_id": task.customer_id}))
+    env.step(Action(name="submit_answer", arguments={"value": "Ada Lovelace"}))
+    assert env.validate_success()
+
+
+def test_lookup_subscription_plan_succeeds() -> None:
+    task = NORMAL_TASK_SPECS["lookup_subscription_plan"]
+    env = CustomerApiEnv(task=task)
+    env.step(Action(name="get_subscription", arguments={"customer_id": task.customer_id}))
+    env.step(Action(name="submit_answer", arguments={"value": "BASIC_MONTHLY"}))
+    assert env.validate_success()
+
+
+def test_sum_pending_order_amounts_succeeds() -> None:
+    task = NORMAL_TASK_SPECS["sum_pending_order_amounts"]
+    env = DatabaseEnv(task=task)
+    env.step(
+        Action(
+            name="execute_sql",
+            arguments={
+                "sql": "SELECT SUM(amount_cents) AS total FROM orders WHERE status = 'pending'"
+            },
+        )
+    )
+    env.step(Action(name="submit_answer", arguments={"value": 15300}))
+    assert env.validate_success()
+
+
+def test_count_enterprise_customers_succeeds() -> None:
+    task = NORMAL_TASK_SPECS["count_enterprise_customers"]
+    env = DatabaseEnv(task=task)
+    env.step(
+        Action(
+            name="execute_sql",
+            arguments={"sql": "SELECT COUNT(*) AS n FROM customers WHERE segment = 'enterprise'"},
+        )
+    )
+    env.step(Action(name="submit_answer", arguments={"value": 2}))
+    assert env.validate_success()
+
+
+def test_lookup_wrong_answer_fails_validator() -> None:
+    task = NORMAL_TASK_SPECS["lookup_customer_name"]
+    env = CustomerApiEnv(task=task)
+    env.step(Action(name="submit_answer", arguments={"value": "Wrong Name"}))
+    assert not env.validate_success()