diff --git a/src/benchmarks/benchmark-config.ts b/src/benchmarks/benchmark-config.ts index bbf2c897..e1cf9a50 100644 --- a/src/benchmarks/benchmark-config.ts +++ b/src/benchmarks/benchmark-config.ts @@ -10,13 +10,9 @@ import { ProviderSort } from "../internal/enums"; import type { ValueOf } from "../internal/guards"; import { z, zDefaultedText } from "../internal/zod"; import { SWITCHYARD_MODEL } from "../providers/switchyard-router-plugin"; -import { - TAU3_BENCH_BANKING_META, - TAU_BENCH_AIRLINE_META, -} from "./benchmark-meta"; +import { TAU_BENCH_AIRLINE_META } from "./benchmark-meta"; import { DracoPanelConfigSchema } from "./draco/schemas"; import { SearchLaneConfigSchema } from "./search/core/config"; -import { BankingRetrievalConfigSchema } from "./tau3-bench-banking/retrieval-config"; export const GEMINI_MEDIA_RESOLUTIONS = [ "MEDIA_RESOLUTION_UNSPECIFIED", @@ -103,22 +99,6 @@ export const TauBenchAirlineConfigSchema = z.object({ export type TauBenchAirlineConfig = z.infer; -export const Tau3BenchBankingOptionsSchema = z.object({ - userModel: zDefaultedText(TAU3_BENCH_BANKING_META.userModel), - userReasoningEffort: z.enum(REASONING_EFFORTS).default("medium"), - retrievalConfig: BankingRetrievalConfigSchema, -}); - -export const Tau3BenchBankingConfigSchema = z.object({ - benchmarkId: z.literal("tau3_bench_banking"), - ...FixedTemperatureBenchmarkBaseSchema.shape, - ...Tau3BenchBankingOptionsSchema.shape, -}); - -export type Tau3BenchBankingConfig = z.infer< - typeof Tau3BenchBankingConfigSchema ->; - export const MmmuProVisionOptionsSchema = z.object({ imageDetail: z.enum(IMAGE_DETAIL_VALUES).optional(), mediaResolution: z.enum(GEMINI_MEDIA_RESOLUTIONS).optional(), @@ -240,7 +220,6 @@ export const KeplerBenchmarkRunConfigSchema = z.discriminatedUnion( GpqaBenchmarkConfigSchema, MmluProBenchmarkConfigSchema, TauBenchAirlineConfigSchema, - Tau3BenchBankingConfigSchema, MmmuProVisionBenchmarkConfigSchema, DracoBenchmarkConfigSchema, IfStructBenchmarkConfigSchema, @@ -270,7 +249,6 @@ export const BENCHMARK_OPTIONS_SCHEMAS = { gpqa_diamond: GpqaOptionsSchema, mmlu_pro: MmluProOptionsSchema, tau_bench_verified_airline: TauBenchOptionsSchema, - tau3_bench_banking: Tau3BenchBankingOptionsSchema, mmmu_pro_vision: MmmuProVisionOptionsSchema, ifstruct: IfStructOptionsSchema, toolcall_formats: ToolCallFormatsOptionsSchema, diff --git a/src/benchmarks/benchmark-meta.ts b/src/benchmarks/benchmark-meta.ts index 0d9d437c..4f13e479 100644 --- a/src/benchmarks/benchmark-meta.ts +++ b/src/benchmarks/benchmark-meta.ts @@ -28,12 +28,6 @@ export const TAU_BENCH_AIRLINE_META = { userModel: "google/gemini-2.5-flash", } as const satisfies BenchmarkMeta; -export const TAU3_BENCH_BANKING_META = { - id: "tau3_bench_banking", - defaultEpochs: 5, - userModel: "openai/gpt-5.4-mini", -} as const satisfies BenchmarkMeta; - export const DRACO_META = { id: "draco", defaultEpochs: 1, @@ -80,7 +74,6 @@ const BENCHMARK_META: Readonly> = { [MMLU_PRO_META.id]: MMLU_PRO_META, [MMMU_PRO_VISION_META.id]: MMMU_PRO_VISION_META, [TAU_BENCH_AIRLINE_META.id]: TAU_BENCH_AIRLINE_META, - [TAU3_BENCH_BANKING_META.id]: TAU3_BENCH_BANKING_META, [DRACO_META.id]: DRACO_META, [IFSTRUCT_META.id]: IFSTRUCT_META, [TOOLCALL_FORMATS_META.id]: TOOLCALL_FORMATS_META, diff --git a/src/benchmarks/native-eval-migration.test.ts b/src/benchmarks/native-eval-migration.test.ts new file mode 100644 index 00000000..9111a9a5 --- /dev/null +++ b/src/benchmarks/native-eval-migration.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "bun:test"; + +import { assertLeft, assertRight } from "../internal/testing"; +import { parseSchema } from "../internal/zod"; +import { + BENCHMARK_OPTIONS_SCHEMAS, + KeplerBenchmarkRunConfigSchema, +} from "./benchmark-config"; +import { getBenchmarkMeta } from "./benchmark-meta"; +import { getBenchmark } from "./registry"; + +describe("native eval migration", () => { + it("does not expose Tau3 Banking through Kepler", () => { + const benchmarkId = "tau3_bench_banking"; + expect(getBenchmark(benchmarkId)).toBeUndefined(); + expect(getBenchmarkMeta(benchmarkId)).toBeUndefined(); + expect(Object.keys(BENCHMARK_OPTIONS_SCHEMAS)).not.toContain(benchmarkId); + assertLeft( + parseSchema(KeplerBenchmarkRunConfigSchema, { + benchmarkId, + model: "openai/gpt-5", + reasoningEffort: "high", + }) + ); + }); + + it.each([ + "gpqa_diamond", + "mmlu_pro", + "mmmu_pro_vision", + "tau_bench_verified_airline", + "vgi_bench", + ])("retains %s", (benchmarkId) => { + expect(getBenchmark(benchmarkId)).toBeDefined(); + expect(getBenchmarkMeta(benchmarkId)).toBeDefined(); + assertRight( + parseSchema(KeplerBenchmarkRunConfigSchema, { + benchmarkId, + model: "openai/gpt-5", + reasoningEffort: "high", + }) + ); + }); +}); diff --git a/src/benchmarks/registry.ts b/src/benchmarks/registry.ts index 37a0d9a3..bfcd07b1 100644 --- a/src/benchmarks/registry.ts +++ b/src/benchmarks/registry.ts @@ -8,7 +8,6 @@ import { DSQA_BENCHMARK } from "./search/dsqa/benchmark"; import { HLE_BENCHMARK } from "./search/hle/benchmark"; import { WIDESEARCH_BENCHMARK } from "./search/widesearch/benchmark"; import { TAU_BENCH_AIRLINE_BENCHMARK } from "./tau-bench-airline/benchmark"; -import { TAU3_BENCH_BANKING_BENCHMARK } from "./tau3-bench-banking/benchmark"; import { TOOLCALL_FORMATS_BENCHMARK } from "./toolcall-formats/benchmark"; import type { Benchmark } from "./types"; import { VGI_BENCH_BENCHMARK } from "./vgi-bench/benchmark"; @@ -17,7 +16,6 @@ const BENCHMARKS: Record = { [GPQA_BENCHMARK.id]: GPQA_BENCHMARK, [MMLU_PRO_BENCHMARK.id]: MMLU_PRO_BENCHMARK, [TAU_BENCH_AIRLINE_BENCHMARK.id]: TAU_BENCH_AIRLINE_BENCHMARK, - [TAU3_BENCH_BANKING_BENCHMARK.id]: TAU3_BENCH_BANKING_BENCHMARK, [MMMU_PRO_VISION_BENCHMARK.id]: MMMU_PRO_VISION_BENCHMARK, [DRACO_BENCHMARK.id]: DRACO_BENCHMARK, [IFSTRUCT_BENCHMARK.id]: IFSTRUCT_BENCHMARK, diff --git a/src/benchmarks/tau3-bench-banking/action-match.test.ts b/src/benchmarks/tau3-bench-banking/action-match.test.ts deleted file mode 100644 index 251f10e5..00000000 --- a/src/benchmarks/tau3-bench-banking/action-match.test.ts +++ /dev/null @@ -1,245 +0,0 @@ -import { describe, expect, it } from "bun:test"; - -import { compareActionWithToolCall } from "./action-match"; -import type { Tau3Action } from "./types"; -describe("compareActionWithToolCall", () => { - it("matches a simple tool call by name and args", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "get_user_information_by_id", - arguments: { user_id: "user_123" }, - requestor: "assistant", - }; - const result = compareActionWithToolCall( - action, - "get_user_information_by_id", - { - user_id: "user_123", - } - ); - expect(result).toBe(true); - }); - it("rejects when tool names do not match", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "get_user_information_by_id", - arguments: { user_id: "user_123" }, - requestor: "assistant", - }; - const result = compareActionWithToolCall( - action, - "get_user_information_by_name", - { - user_id: "user_123", - } - ); - expect(result).toBe(false); - }); - it("rejects when arguments do not match", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "get_user_information_by_id", - arguments: { user_id: "user_123" }, - requestor: "assistant", - }; - const result = compareActionWithToolCall( - action, - "get_user_information_by_id", - { - user_id: "user_456", - } - ); - expect(result).toBe(false); - }); - it("matches inner-tool call for call_discoverable_agent_tool", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "call_discoverable_agent_tool", - arguments: { - agent_tool_name: "get_bank_account_transactions_9173", - arguments: { account_id: "acc_001", limit: 10 }, - }, - requestor: "assistant", - }; - const result = compareActionWithToolCall( - action, - "call_discoverable_agent_tool", - { - agent_tool_name: "get_bank_account_transactions_9173", - arguments: { account_id: "acc_001", limit: 10 }, - } - ); - expect(result).toBe(true); - }); - it("rejects inner-tool call when inner tool name differs", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "call_discoverable_agent_tool", - arguments: { - agent_tool_name: "get_bank_account_transactions_9173", - arguments: { account_id: "acc_001" }, - }, - requestor: "assistant", - }; - const result = compareActionWithToolCall( - action, - "call_discoverable_agent_tool", - { - agent_tool_name: "get_bank_account_transactions_9174", - arguments: { account_id: "acc_001" }, - } - ); - expect(result).toBe(false); - }); - it("rejects inner-tool call when inner arguments differ", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "call_discoverable_agent_tool", - arguments: { - agent_tool_name: "get_bank_account_transactions_9173", - arguments: { account_id: "acc_001", limit: 10 }, - }, - requestor: "assistant", - }; - const result = compareActionWithToolCall( - action, - "call_discoverable_agent_tool", - { - agent_tool_name: "get_bank_account_transactions_9173", - arguments: { account_id: "acc_001", limit: 20 }, - } - ); - expect(result).toBe(false); - }); - it("matches inner-tool call for call_discoverable_user_tool", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "call_discoverable_user_tool", - arguments: { - discoverable_tool_name: "apply_for_credit_card", - arguments: { card_type: "Gold Rewards Card" }, - }, - requestor: "user", - }; - const result = compareActionWithToolCall( - action, - "call_discoverable_user_tool", - { - discoverable_tool_name: "apply_for_credit_card", - arguments: { card_type: "Gold Rewards Card" }, - } - ); - expect(result).toBe(true); - }); - it("compares only compare_args keys when compare_args is set", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "transfer_to_human_agents", - arguments: { reason: "user_request", summary: "golden summary" }, - compare_args: ["reason"], - requestor: "assistant", - }; - const result = compareActionWithToolCall( - action, - "transfer_to_human_agents", - { - reason: "user_request", - summary: "a completely different summary", - } - ); - expect(result).toBe(true); - }); - it("rejects when a compare_args key differs", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "transfer_to_human_agents", - arguments: { reason: "user_request" }, - compare_args: ["reason"], - requestor: "assistant", - }; - const result = compareActionWithToolCall( - action, - "transfer_to_human_agents", - { - reason: "out_of_scope", - } - ); - expect(result).toBe(false); - }); - it("matches on name alone when compare_args is empty", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "request_human_agent_transfer", - arguments: {}, - compare_args: [], - requestor: "user", - }; - const result = compareActionWithToolCall( - action, - "request_human_agent_transfer", - { - anything: "goes", - } - ); - expect(result).toBe(true); - }); - it("matches discoverable call on inner name only when compare_args selects it", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "call_discoverable_agent_tool", - arguments: { - agent_tool_name: "get_bank_account_transactions_9173", - arguments: '{"account_id": "acc_001"}', - }, - compare_args: ["agent_tool_name"], - requestor: "assistant", - }; - const result = compareActionWithToolCall( - action, - "call_discoverable_agent_tool", - { - agent_tool_name: "get_bank_account_transactions_9173", - arguments: '{"account_id": "acc_999", "limit": 5}', - } - ); - expect(result).toBe(true); - }); - it("rejects when a compare_args key is present on only one side", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "change_user_email", - arguments: { user_id: "user_123" }, - compare_args: ["user_id", "new_email"], - requestor: "assistant", - }; - const result = compareActionWithToolCall(action, "change_user_email", { - user_id: "user_123", - new_email: "a@b.com", - }); - expect(result).toBe(false); - }); - it("handles nested object arguments correctly", () => { - const action: Tau3Action = { - action_id: "act_001", - name: "call_discoverable_agent_tool", - arguments: { - agent_tool_name: "some_tool", - arguments: { - nested: { key1: "value1", key2: 2 }, - }, - }, - requestor: "assistant", - }; - const result = compareActionWithToolCall( - action, - "call_discoverable_agent_tool", - { - agent_tool_name: "some_tool", - arguments: { - nested: { key2: 2, key1: "value1" }, - }, - } - ); - expect(result).toBe(true); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/action-match.ts b/src/benchmarks/tau3-bench-banking/action-match.ts deleted file mode 100644 index 6f69878f..00000000 --- a/src/benchmarks/tau3-bench-banking/action-match.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { isRecord } from "../../internal/guards"; -import type { Tau3Action } from "./types"; - -function stableStringify(value: unknown): string { - if (Array.isArray(value)) { - return `[${value.map(stableStringify).join(",")}]`; - } - if (isRecord(value)) { - const entries = Object.keys(value) - .toSorted() - .map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`); - return `{${entries.join(",")}}`; - } - return JSON.stringify(value) ?? "null"; -} - -function deepEqual(a: unknown, b: unknown): boolean { - return stableStringify(a) === stableStringify(b); -} - -export function compareActionWithToolCall( - action: Tau3Action, - toolName: string, - toolArgs: Readonly> -): boolean { - if (action.name !== toolName) { - return false; - } - const compareArgs = action.compare_args ?? Object.keys(toolArgs); - return compareArgs.every( - (key) => - key in toolArgs === key in action.arguments && - deepEqual(toolArgs[key], action.arguments[key]) - ); -} diff --git a/src/benchmarks/tau3-bench-banking/agent-prompts.test.ts b/src/benchmarks/tau3-bench-banking/agent-prompts.test.ts deleted file mode 100644 index 8d78b501..00000000 --- a/src/benchmarks/tau3-bench-banking/agent-prompts.test.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { describe, it, expect, beforeEach } from "bun:test"; -import { createHash } from "node:crypto"; - -import { - buildBankingAgentSystemPrompt, - DEFAULT_FIRST_AGENT_MESSAGE, -} from "./agent-prompts"; -import { seedBankingDocumentsCache } from "./documents"; - -const FIXTURE_DOCS = [ - { - id: "doc_test_accounts_001", - title: "Accounts Overview", - content: "Information about bank accounts.", - }, - { - id: "doc_test_cards_002", - title: "Debit Cards Guide", - content: "How to use debit cards.", - }, - { - id: "doc_test_disputes_003", - title: "Dispute Resolution Process", - content: "Steps to file a dispute.", - }, -]; -describe("Banking Agent Prompts", () => { - beforeEach(() => { - seedBankingDocumentsCache(FIXTURE_DOCS); - }); - describe("DEFAULT_FIRST_AGENT_MESSAGE", () => { - it("is a greeting string", () => { - expect(typeof DEFAULT_FIRST_AGENT_MESSAGE).toBe("string"); - expect(DEFAULT_FIRST_AGENT_MESSAGE.length).toBeGreaterThan(0); - expect(DEFAULT_FIRST_AGENT_MESSAGE).toContain("help"); - }); - }); - describe("buildBankingAgentSystemPrompt", () => { - it("builds prompt with single required document", () => { - const prompt = buildBankingAgentSystemPrompt({ - requiredDocIds: ["doc_test_accounts_001"], - retrievalConfig: "required_docs", - }); - expect(prompt).toContain(""); - expect(prompt).toContain(""); - expect(prompt).toContain(""); - expect(prompt).toContain(""); - expect(prompt).toContain("# Rho-Bank Customer Service Policy"); - expect(prompt).toContain("## Accounts Overview"); - expect(prompt).toContain("Information about bank accounts."); - expect(prompt).toContain(""); - }); - it("renders multiple required documents in order", () => { - const prompt = buildBankingAgentSystemPrompt({ - requiredDocIds: [ - "doc_test_accounts_001", - "doc_test_cards_002", - "doc_test_disputes_003", - ], - retrievalConfig: "required_docs", - }); - expect(prompt).toContain("## Accounts Overview"); - expect(prompt).toContain("## Debit Cards Guide"); - expect(prompt).toContain("## Dispute Resolution Process"); - const accountsIdx = prompt.indexOf("## Accounts Overview"); - const cardsIdx = prompt.indexOf("## Debit Cards Guide"); - const disputesIdx = prompt.indexOf("## Dispute Resolution Process"); - expect(accountsIdx).toBeLessThan(cardsIdx); - expect(cardsIdx).toBeLessThan(disputesIdx); - }); - it("renders documents separated by --- separator", () => { - const prompt = buildBankingAgentSystemPrompt({ - requiredDocIds: ["doc_test_accounts_001", "doc_test_cards_002"], - retrievalConfig: "required_docs", - }); - expect(prompt).toContain("---"); - }); - it("renders zero-doc variant as no documents provided", () => { - const prompt = buildBankingAgentSystemPrompt({ - requiredDocIds: [], - retrievalConfig: "required_docs", - }); - expect(prompt).toContain("(No documents provided)"); - }); - it("includes policy header component", () => { - const prompt = buildBankingAgentSystemPrompt({ - requiredDocIds: ["doc_test_accounts_001"], - retrievalConfig: "required_docs", - }); - expect(prompt).toContain("# Rho-Bank Customer Service Policy"); - expect(prompt).toContain("You are a helpful customer service agent"); - expect(prompt).toContain("## Guidelines"); - }); - it("includes additional instructions component", () => { - const prompt = buildBankingAgentSystemPrompt({ - requiredDocIds: ["doc_test_accounts_001"], - retrievalConfig: "required_docs", - }); - expect(prompt).toContain("## Additional Instructions"); - expect(prompt).toContain("### Discoverable Tools"); - expect(prompt).toContain("### Authenticating Users"); - }); - it("throws when required doc id not in cache", () => { - const [firstDoc] = FIXTURE_DOCS; - if (!firstDoc) { - throw new Error("FIXTURE_DOCS is empty"); - } - seedBankingDocumentsCache([firstDoc]); - expect(() => - buildBankingAgentSystemPrompt({ - requiredDocIds: ["missing_doc_id"], - retrievalConfig: "required_docs", - }) - ).toThrow(/not in cache/); - }); - it("throws with doc id in error message", () => { - seedBankingDocumentsCache([]); - expect(() => - buildBankingAgentSystemPrompt({ - requiredDocIds: ["specific_missing_doc_id"], - retrievalConfig: "required_docs", - }) - ).toThrow(/specific_missing_doc_id/); - }); - it("wraps prompt with instructions and policy tags", () => { - const prompt = buildBankingAgentSystemPrompt({ - requiredDocIds: ["doc_test_accounts_001"], - retrievalConfig: "required_docs", - }); - const instructionsStart = prompt.indexOf(""); - const instructionsEnd = prompt.indexOf(""); - const hasInstructionsTag = prompt.includes(""); - const hasInstructionsCloseTag = prompt.includes(""); - const hasPolicyTag = prompt.includes(""); - const hasPolicyCloseTag = prompt.includes(""); - expect(instructionsStart).toBeGreaterThanOrEqual(0); - expect(instructionsEnd).toBeGreaterThan(instructionsStart); - expect(hasInstructionsTag).toBe(true); - expect(hasInstructionsCloseTag).toBe(true); - expect(hasPolicyTag).toBe(true); - expect(hasPolicyCloseTag).toBe(true); - }); - it("uses the upstream classic BM25 and grep prompt without golden documents", () => { - const prompt = buildBankingAgentSystemPrompt({ - requiredDocIds: ["doc_test_accounts_001"], - retrievalConfig: "bm25_grep", - }); - expect(prompt).toContain( - "**Search the knowledge base** for relevant information when appropriate using the provided `KB_search` (uses BM25 for retrieval) and `grep` tools." - ); - expect(prompt).not.toContain(""); - expect(prompt).not.toContain("Information about bank accounts."); - }); - it("matches the pinned upstream bm25_grep system prompt byte-for-byte", () => { - const prompt = buildBankingAgentSystemPrompt({ - requiredDocIds: [], - retrievalConfig: "bm25_grep", - }); - expect(createHash("sha256").update(prompt).digest("hex")).toBe( - "6106f260b43df9b84e22ad54cc8d6652ce8eee9831fef4a208c7d82145d74462" - ); - }); - it("matches the pinned required_docs system prompt byte-for-byte", () => { - const prompt = buildBankingAgentSystemPrompt({ - requiredDocIds: [], - retrievalConfig: "required_docs", - }); - expect(createHash("sha256").update(prompt).digest("hex")).toBe( - "49d311c73c3034fb255b5d4967e598959bde63908dc8c4fa101e94ab57e78a60" - ); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/agent-prompts.ts b/src/benchmarks/tau3-bench-banking/agent-prompts.ts deleted file mode 100644 index d014fcaa..00000000 --- a/src/benchmarks/tau3-bench-banking/agent-prompts.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { getBankingDocument } from "./documents"; -import type { BankingRetrievalConfig } from "./retrieval-config"; - -const UPSTREAM_POLICY_HEADER = `# Rho-Bank Customer Service Policy - -You are a helpful customer service agent for Rho-Bank. -Your goal is to help customers by searching the knowledge base and providing accurate information. - -## Guidelines - -1. Do not make up policies, information or actions that you can take on behalf of the user. All instructions will be found here or in the knowledge base. If you cannot find relevant information, let the user know.\u0020 -2. Do not ask for any documentation, receipts... from the customer unless it states very clearly in the knowledge base how to process it, and whether you're allowed to do so.\u0020 -3. Be polite and professional -4. If you need the current time, always use the get_current_time() tool. Do not make up or assume the current time.\u0020 -5. Generally, if the issue cannot be resolved or is outside your capabilities, ask the user whether they would like to be transferred to a human agent. If they do, invoke the appropriate transfer_to_human_agents tool. Do this only if you absolutely have to, and you are sure that there are no potential actions you can take as specified in the knowledge base, or in your policy. Do not transfer without asking the user first. This guidance may be overridden by specific scenario-based transfer guidance in the knowledge base.\u0020 -6. If an issue falls within your capabilities and the user still wants to be transferred to a human agent, kindly inform the user that you can help them, and try to help them first. If the user asks for a human agent 4 times, then you may invoke the transfer_to_human_agents tool. This guidance may be overridden by specific scenario-based transfer guidance in the knowledge base.\u0020 -7. Do not give intermediate responses to users while processing that would give away internal rho-bank information/policies.\u0020\n`; - -const UPSTREAM_ADDITIONAL_INSTRUCTIONS = ` -## Additional Instructions - -### Discoverable Tools - -#### Giving Discoverable Tools to Users -The knowledge base may contain instructions that indicate certain actions should be performed by the user themselves rather than by you. These are called "user discoverable tools." A user discoverable tool is a tool that you provide to the user so they can execute it on their own (e.g., through a customer portal or app). - -**When to give user discoverable tools:** -- Only give a tool when the user would like to perform an action, and the knowledge base explicitly has a tool that allows the user to perform this action (e.g., "to do X, have the user call tool_name(args)"). IMPORTANT: Do not unlock tools that you do not plan on giving to the user and actually using: this causes issues in database logging. -- You must search the knowledge base to find tools that you can give. Do not invent or guess user discoverable tools\u0020 -- Only use tool names and arguments discovered in the knowledge base - -**How to give a tool:** -- Use the \`give_discoverable_user_tool(discoverable_tool_name)\` function -- Provide the exact tool name as specified in the knowledge base -- Explain to the user what the tool does and how to use it, and what arguments to provide. Just explaining isn't enough, you must use the \`give_discoverable_user_tool(discoverable_tool_name)\` function. - -#### Unlocking and Using Agent Discoverable Tools -The knowledge base may contain references to specialized internal tools that you can unlock and use. These are called "agent discoverable tools." Unlike regular tools which are always available, these tools must be explicitly unlocked after discovering them in the knowledge base. - -**When to use agent discoverable tools:** -- Only unlock a tool when the knowledge base explicitly mentions it (e.g., "use tool_name to perform X"), and do not unlock tools you do not plan to use. -- You must search the knowledge base to find tools that you can unlock. Do not invent or guess tool names - only use tool names discovered in the knowledge base. - -**How to use agent discoverable tools:** -1. First, unlock the tool using \`unlock_discoverable_agent_tool(agent_tool_name)\` with the exact tool name from the knowledge base: you must unlock the tool before using it to get information on the proper params. IMPORTANT: Do not unlock tools that you do not plan on actually using: this causes issues in database logging. -2. Then, call the tool using \`call_discoverable_agent_tool(agent_tool_name, arguments)\` with the required arguments -3. The unlock step is required before calling - you cannot call a tool that hasn't been unlocked - -### Authenticating Users - -Generally, for any scenario involving accessing customer information in internal databases, you must first verify their identify before proceeding. No need to verify more than once in a single conversation. You should ONLY verify a user's identity if you need to access or modify their customer information in internal databases on their behalf. - -Here are some concrete examples: -* Looking up account balances, transaction history, referral history... -* Changing account settings (e.g., address, phone number, email) -* Closing an account -* Adding or removing authorized users -* Requesting information about specific transactions -* Discussing specific loan or credit details -* Filing a dispute on behalf of the user - -To verify the identity of the user, call the appropriate read tools, and ensure that they are able to give correctly any 2 out of the following values: date of birth, email, phone number, address. Knowing full name or userID is not enough to verify. After verification, you must call the verification logging tool to properly log the information into the verification records. Do not leak any information about the user before they are verified.`; - -const POLICY_HEADER = UPSTREAM_POLICY_HEADER.replaceAll( - /[ \t]+$/gmu, - "" -).trimEnd(); - -const ADDITIONAL_INSTRUCTIONS = UPSTREAM_ADDITIONAL_INSTRUCTIONS.replace( - "- Only give", - "- Only give" -) - .replace("user discoverable tools \n", "user discoverable tools\n") - .replace("tool name as", "tool name as"); - -const AGENT_INSTRUCTION = `You are a customer service agent that helps the user according to the provided below. -In each turn you can either: -- Send a message to the user. -- Make a tool call. -You cannot do both at the same time. - -Try to be helpful and always follow the policy. Always make sure you generate valid JSON only.`; - -export const DEFAULT_FIRST_AGENT_MESSAGE = "Hi! How can I help you today?"; - -export function buildBankingAgentSystemPrompt({ - requiredDocIds, - retrievalConfig, -}: { - readonly requiredDocIds: readonly string[]; - readonly retrievalConfig: BankingRetrievalConfig; -}): string { - const policy = buildBankingPolicy({ requiredDocIds, retrievalConfig }); - return ` -${AGENT_INSTRUCTION} - - -${policy} -`; -} - -function buildBankingPolicy({ - requiredDocIds, - retrievalConfig, -}: { - readonly requiredDocIds: readonly string[]; - readonly retrievalConfig: BankingRetrievalConfig; -}): string { - switch (retrievalConfig) { - case "required_docs": { - return buildRequiredDocumentsPolicy(requiredDocIds); - } - case "bm25_grep": { - return `${UPSTREAM_POLICY_HEADER} - -**Search the knowledge base** for relevant information when appropriate using the provided \`KB_search\` (uses BM25 for retrieval) and \`grep\` tools. - -${UPSTREAM_ADDITIONAL_INSTRUCTIONS} -`; - } - default: { - retrievalConfig satisfies never; - return ""; - } - } -} - -function buildRequiredDocumentsPolicy( - requiredDocIds: readonly string[] -): string { - const requiredDocumentsText = - requiredDocIds.length === 0 - ? "(No documents provided)" - : requiredDocIds - .map((docId) => { - const doc = getBankingDocument(docId); - return `## ${doc.title}\n\n${doc.content}`; - }) - .join("\n\n---\n\n"); - return `${POLICY_HEADER} - -## Documents you need to solve the problem - -The following documents contain the information you need to answer the customer's question and determine what actions to take. - -${ADDITIONAL_INSTRUCTIONS} - - -${requiredDocumentsText} -`; -} diff --git a/src/benchmarks/tau3-bench-banking/benchmark.test.ts b/src/benchmarks/tau3-bench-banking/benchmark.test.ts deleted file mode 100644 index be8022f8..00000000 --- a/src/benchmarks/tau3-bench-banking/benchmark.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { describe, expect, it } from "bun:test"; - -import { FetchHttpClient } from "@effect/platform"; -import { runPromise, scoped } from "effect/Effect"; -import { build, provide } from "effect/Layer"; - -import { assertLeft, assertRight } from "../../internal/testing"; -import { parseSchema } from "../../internal/zod"; -import { Tau3BenchBankingConfigSchema } from "../benchmark-config"; -import { getBenchmarkMeta } from "../benchmark-meta"; -import { getBenchmark } from "../registry"; -import { - TAU3_BENCH_BANKING_BENCHMARK, - TAU3_BENCH_BANKING_ID, -} from "./benchmark"; -describe("tau3_bench_banking registration", () => { - it("resolves from the registry", () => { - const benchmark = getBenchmark("tau3_bench_banking"); - expect(benchmark).toBe(TAU3_BENCH_BANKING_BENCHMARK); - expect(TAU3_BENCH_BANKING_ID).toBe("tau3_bench_banking"); - expect(TAU3_BENCH_BANKING_BENCHMARK.userModel).toBe("openai/gpt-5.4-mini"); - }); - it("has meta with defaultEpochs 5", () => { - const meta = getBenchmarkMeta("tau3_bench_banking"); - expect(meta?.defaultEpochs).toBe(5); - }); - it("resolves AA-parity banking defaults", () => { - const result = parseSchema(Tau3BenchBankingConfigSchema, { - benchmarkId: "tau3_bench_banking", - model: "openai/gpt-4o-mini", - reasoningEffort: "high", - }); - assertRight(result); - expect(result.right.userModel).toBe("openai/gpt-5.4-mini"); - expect(result.right.retrievalConfig).toBe("bm25_grep"); - expect(result.right.userReasoningEffort).toBe("medium"); - }); - it("treats an empty user model as unset", () => { - const result = parseSchema(Tau3BenchBankingConfigSchema, { - benchmarkId: "tau3_bench_banking", - model: "openai/gpt-4o-mini", - reasoningEffort: "high", - userModel: "", - }); - assertRight(result); - expect(result.right.userModel).toBe("openai/gpt-5.4-mini"); - }); - it("treats a whitespace-only user model as unset", () => { - const result = parseSchema(Tau3BenchBankingConfigSchema, { - benchmarkId: "tau3_bench_banking", - model: "openai/gpt-4o-mini", - reasoningEffort: "high", - userModel: " \t", - }); - assertRight(result); - expect(result.right.userModel).toBe("openai/gpt-5.4-mini"); - }); - it("allows overriding the user simulator reasoning effort", () => { - const result = parseSchema(Tau3BenchBankingConfigSchema, { - benchmarkId: "tau3_bench_banking", - model: "openai/gpt-4o-mini", - reasoningEffort: "high", - userReasoningEffort: "high", - }); - assertRight(result); - expect(result.right.userReasoningEffort).toBe("high"); - }); - it("parses a valid config", () => { - const result = parseSchema(Tau3BenchBankingConfigSchema, { - benchmarkId: "tau3_bench_banking", - model: "openai/gpt-4o-mini", - reasoningEffort: "high", - userModel: "google/gemini-2.5-flash", - retrievalConfig: "required_docs", - }); - assertRight(result); - expect(result.right.model).toBe("openai/gpt-4o-mini"); - expect(result.right.retrievalConfig).toBe("required_docs"); - }); - it("parses the bm25_grep retrieval config", () => { - const result = parseSchema(Tau3BenchBankingConfigSchema, { - benchmarkId: "tau3_bench_banking", - model: "openai/gpt-4o-mini", - reasoningEffort: "high", - retrievalConfig: "bm25_grep", - }); - assertRight(result); - expect(result.right.retrievalConfig).toBe("bm25_grep"); - }); - it("rejects a mismatched benchmarkId", () => { - const result = parseSchema(Tau3BenchBankingConfigSchema, { - benchmarkId: "tau_bench_verified_airline", - model: "openai/gpt-4o-mini", - }); - assertLeft(result); - }); - it("rejects an unknown retrievalConfig", () => { - const result = parseSchema(Tau3BenchBankingConfigSchema, { - benchmarkId: "tau3_bench_banking", - model: "openai/gpt-4o-mini", - retrievalConfig: "bm25", - }); - assertLeft(result); - }); - it("makeLayer rejects a mismatched config", async () => { - const layer = TAU3_BENCH_BANKING_BENCHMARK.makeLayer({ - apiKey: "test", - benchmarkConfig: { - benchmarkId: "gpqa_diamond", - model: "openai/gpt-4o-mini", - reasoningEffort: "high", - }, - sessionId: "test-session", - }); - const buildLayer = build(layer.pipe(provide(FetchHttpClient.layer))).pipe( - scoped - ); - await expect(runPromise(buildLayer)).rejects.toThrow( - "invalid benchmarkConfig" - ); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/benchmark.ts b/src/benchmarks/tau3-bench-banking/benchmark.ts deleted file mode 100644 index ca36355c..00000000 --- a/src/benchmarks/tau3-bench-banking/benchmark.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { HttpClient } from "@effect/platform"; -import { gen, makeSemaphore } from "effect/Effect"; -import type { Layer } from "effect/Layer"; -import { - fail as layerFail, - effect as layerEffect, - provide as layerProvide, - mergeAll as layerMergeAll, - succeed as layerSucceed, -} from "effect/Layer"; - -import type { Dataset } from "../../harness/dataset"; -import { Model } from "../../harness/model"; -import { Scorer } from "../../harness/scorer"; -import { Solver } from "../../harness/solver"; -import { Either } from "../../internal/either"; -import { definedValues } from "../../internal/guards"; -import { parseSchema } from "../../internal/zod"; -import { makeOpenRouterModelLayer } from "../../providers/openrouter-model"; -import { - makeResponsesModelLayer, - ResponsesModel, -} from "../../providers/responses-model"; -import { Tau3BenchBankingConfigSchema } from "../benchmark-config"; -import { TAU3_BENCH_BANKING_META } from "../benchmark-meta"; -import type { Benchmark, BenchmarkRunInput } from "../types"; -import { makeBankingDatasetLayer } from "./dataset"; -import { bankingScorer } from "./scorer"; -import { bankingSolver } from "./solver"; -import type { SolverOpts } from "./types"; - -export const TAU3_BENCH_BANKING_TEMPERATURE = 0; - -export const TAU3_BENCH_BANKING_ID = TAU3_BENCH_BANKING_META.id; - -function makeBankingLayer( - input: BenchmarkRunInput -): Layer { - const { benchmarkConfig } = input; - const configParsed = parseSchema( - Tau3BenchBankingConfigSchema, - benchmarkConfig - ); - if (Either.isLeft(configParsed)) { - return layerFail( - new Error( - `tau3_bench_banking received invalid benchmarkConfig: ${configParsed.left.message}` - ) - ); - } - const config = configParsed.right; - const solverOpts: SolverOpts = definedValues({ - endpointId: config.endpointId, - userModelConfig: definedValues({ - apiKey: input.apiKey, - model: config.userModel, - baseUrl: input.baseUrl, - sessionId: input.sessionId, - userReasoningEffort: config.userReasoningEffort, - }), - inference: { - maxTokens: config.maxTokens, - reasoningEffort: config.reasoningEffort, - timeoutMs: config.timeoutMs, - sort: config.sort, - providerOnly: config.providerOnly, - providerIgnore: config.providerIgnore, - allowFallbacks: config.allowFallbacks, - cloudflareVersion: config.cloudflareVersion, - experimentIds: config.experimentIds, - costTier: config.costTier, - costQualityTradeoff: config.costQualityTradeoff, - pinModel: config.pinModel, - switchyardAlgorithm: config.switchyardAlgorithm, - }, - retrievalConfig: config.retrievalConfig, - }); - const datasetLayer = makeBankingDatasetLayer(input.datasetRetry); - const modelLayer = - input.modelLayer ?? - makeOpenRouterModelLayer( - definedValues({ - model: config.model, - models: config.models, - apiKey: input.apiKey, - baseUrl: input.baseUrl, - sessionId: input.sessionId, - retry: input.modelRetry, - traceHeaders: input.traceHeaders, - }) - ); - const userModelLayer = makeResponsesModelLayer( - definedValues({ - model: config.userModel, - apiKey: input.apiKey, - baseUrl: input.baseUrl, - sessionId: input.sessionId, - traceHeaders: input.traceHeaders, - }) - ); - const solverLayer = layerEffect(Solver)( - gen(function* () { - const model = yield* Model; - const userModel = yield* ResponsesModel; - const client = yield* HttpClient.HttpClient; - const dataFetchLock = yield* makeSemaphore(1); - return Solver.of( - bankingSolver({ - model, - userModel, - client, - dataFetchLock, - opts: solverOpts, - }) - ); - }) - ); - const scorerLayer = layerSucceed(Scorer, Scorer.of(bankingScorer)); - return layerMergeAll( - datasetLayer, - solverLayer.pipe(layerProvide(layerMergeAll(modelLayer, userModelLayer))), - scorerLayer - ); -} - -export const TAU3_BENCH_BANKING_BENCHMARK: Benchmark = { - id: TAU3_BENCH_BANKING_ID, - makeDatasetLayer: makeBankingDatasetLayer, - temperature: TAU3_BENCH_BANKING_TEMPERATURE, - defaultEpochs: TAU3_BENCH_BANKING_META.defaultEpochs, - userModel: TAU3_BENCH_BANKING_META.userModel, - makeLayer: makeBankingLayer, -}; diff --git a/src/benchmarks/tau3-bench-banking/dataset.test.ts b/src/benchmarks/tau3-bench-banking/dataset.test.ts deleted file mode 100644 index 379c2cf1..00000000 --- a/src/benchmarks/tau3-bench-banking/dataset.test.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; - -import { flatMap, provide, runPromise } from "effect/Effect"; - -import { Dataset } from "../../harness/dataset"; -import { bankingRecordToSample, makeBankingDatasetLayer } from "./dataset"; -import { - makeEmptyBankingData, - seedBankingCache, - seedBankingTasksRawCache, -} from "./environment"; -import type { BankingData, Tau3Task } from "./types"; -describe("tau3-bench-banking dataset", () => { - let originalFetch: typeof global.fetch; - const fixtureDb: BankingData = { - ...makeEmptyBankingData(), - users: { data: { "1": { name: "Test" } } }, - }; - const fixtureTasks: Tau3Task[] = [ - { - id: "task_001", - description: null, - user_scenario: { instructions: "First task instructions" }, - initial_state: null, - evaluation_criteria: null, - annotations: null, - }, - { - id: "task_002", - description: null, - user_scenario: { instructions: "Second task instructions" }, - initial_state: null, - evaluation_criteria: null, - annotations: null, - }, - { - id: "task_003", - description: null, - user_scenario: { instructions: "Third task instructions" }, - initial_state: null, - evaluation_criteria: null, - annotations: null, - }, - ]; - beforeEach(() => { - originalFetch = global.fetch; - seedBankingCache(fixtureDb, fixtureTasks); - }); - afterEach(() => { - global.fetch = originalFetch; - }); - describe("bankingRecordToSample", () => { - it("converts tasks to samples with correct id, instructions input, and empty target", () => { - const sample1 = bankingRecordToSample(fixtureTasks[0]!); - const sample2 = bankingRecordToSample(fixtureTasks[1]!); - expect(sample1.id).toBe("tau3_bench_banking-task_001"); - expect(sample1.input).toBe("First task instructions"); - expect(sample1.target.text).toBe(""); - expect(sample2.input).toBe("Second task instructions"); - }); - it("includes task in metadata", () => { - const sample = bankingRecordToSample(fixtureTasks[0]!); - expect(sample.metadata?.task).toBe(fixtureTasks[0]); - }); - it("generates unique ids for different tasks", () => { - const sample1 = bankingRecordToSample(fixtureTasks[0]!); - const sample2 = bankingRecordToSample(fixtureTasks[1]!); - expect(sample1.id).not.toBe(sample2.id); - expect(sample1.id).toBe("tau3_bench_banking-task_001"); - expect(sample2.id).toBe("tau3_bench_banking-task_002"); - }); - }); - describe("makeBankingDatasetLayer", () => { - it("loads pinned tasks before resolving dataset size", async () => { - seedBankingTasksRawCache(""); - global.fetch = async () => Response.json(fixtureTasks); - const layer = makeBankingDatasetLayer(); - const size = await runPromise( - Dataset.pipe( - flatMap((dataset) => dataset.size), - provide(layer) - ) - ); - expect(size).toBe(fixtureTasks.length); - }); - it("retries a transient fetch failure per the retry config", async () => { - seedBankingTasksRawCache(""); - let fetchCalls = 0; - global.fetch = async () => { - fetchCalls++; - if (fetchCalls === 1) { - return new Response("upstream hiccup", { status: 503 }); - } - return Response.json(fixtureTasks); - }; - const layer = makeBankingDatasetLayer({ baseDelayMs: 0, maxRetries: 1 }); - const size = await runPromise( - Dataset.pipe( - flatMap((dataset) => dataset.size), - provide(layer) - ) - ); - expect(size).toBe(fixtureTasks.length); - expect(fetchCalls).toBe(2); - }); - it("does not retry when maxRetries is 0", async () => { - seedBankingTasksRawCache(""); - let fetchCalls = 0; - global.fetch = async () => { - fetchCalls++; - return new Response("upstream hiccup", { status: 503 }); - }; - const layer = makeBankingDatasetLayer({ baseDelayMs: 0, maxRetries: 0 }); - await expect( - runPromise( - Dataset.pipe( - flatMap((dataset) => dataset.size), - provide(layer) - ) - ) - ).rejects.toThrow(); - expect(fetchCalls).toBe(1); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/dataset.ts b/src/benchmarks/tau3-bench-banking/dataset.ts deleted file mode 100644 index f1dffd91..00000000 --- a/src/benchmarks/tau3-bench-banking/dataset.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { FetchHttpClient, HttpClient } from "@effect/platform"; -import { - gen, - makeSemaphore, - map, - mapError, - provideService, -} from "effect/Effect"; -import type { Layer as LayerType } from "effect/Layer"; -import { effect as layerEffect, provide as layerProvide } from "effect/Layer"; -import type { Stream as StreamType } from "effect/Stream"; -import { - fail, - flatMap as streamFlatMap, - fromEffect, - fromIterable, -} from "effect/Stream"; - -import type { Sample } from "../../harness/core"; -import { DatasetError } from "../../harness/core"; -import type { DatasetStreamOptions } from "../../harness/dataset"; -import { Dataset } from "../../harness/dataset"; -import type { RetryConfig } from "../../runtime/retry"; -import { ensureBankingTasks, loadBankingTasks } from "./environment"; -import type { Tau3Task } from "./types"; - -export function bankingRecordToSample(task: Tau3Task): Sample { - return { - id: `tau3_bench_banking-${task.id}`, - input: task.user_scenario.instructions, - target: { text: "" }, - metadata: { task }, - }; -} - -export function makeBankingDatasetLayer( - retryConfig?: RetryConfig -): LayerType { - const makeService = gen(function* () { - const client = yield* HttpClient.HttpClient; - const fetchLock = yield* makeSemaphore(1); - const loadTasks = ensureBankingTasks(fetchLock, retryConfig).pipe( - provideService(HttpClient.HttpClient, client), - mapError( - (cause) => - new DatasetError({ - message: `Failed to load tau3 banking tasks: ${String(cause)}`, - }) - ), - map(() => loadBankingTasks()) - ); - const stream = ( - opts?: DatasetStreamOptions - ): StreamType => { - return fromEffect(loadTasks).pipe( - streamFlatMap((tasks) => { - const start = opts?.start ?? 0; - const end = opts?.end ?? tasks.length; - if (start < 0 || end > tasks.length || start > end) { - return fail( - new DatasetError({ - message: `Invalid stream range [${start}, ${end}) for dataset size ${tasks.length}`, - }) - ); - } - return fromIterable( - tasks.slice(start, end).map(bankingRecordToSample) - ); - }) - ); - }; - return Dataset.of({ - stream, - size: loadTasks.pipe(map((tasks) => tasks.length)), - }); - }); - return layerEffect(Dataset, makeService).pipe( - layerProvide(FetchHttpClient.layer) - ); -} diff --git a/src/benchmarks/tau3-bench-banking/documents.test.ts b/src/benchmarks/tau3-bench-banking/documents.test.ts deleted file mode 100644 index 5af3d40b..00000000 --- a/src/benchmarks/tau3-bench-banking/documents.test.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; -import assert from "node:assert/strict"; - -import { FetchHttpClient } from "@effect/platform"; -import { makeSemaphore, provide, runPromise } from "effect/Effect"; - -import { - ensureAllBankingDocuments, - getAllBankingDocuments, - getBankingDocument, - seedBankingDocumentsCache, -} from "./documents"; - -const FIXTURE_DOC_001 = { - id: "doc_bank_accounts_bank_accounts_(general)_001", - title: "Internal: Opening Personal Checking Accounts", - content: - "## Eligibility Requirements\n\nTo open a personal checking account, ensure all of the following are true:\n1. The customer is verified.\n2. The customer is at least 18 years old.\n3. The customer does not exceed 4 personal checking accounts.\n4. The customer has no checking accounts closed for cause in the past 6 months.\n\n## Opening Procedure\n\n1. Verify customer identity.\n2. Check eligibility requirements listed above.\n3. Confirm the customer's desired account_class selection.\n - Personal checking account_class options must use the full official name ending with 'Account' (e.g., 'Blue Account', 'Green Account (checking)').\n4. Use open_bank_account_4821 to open the account.", -}; - -const FIXTURE_DOC_002 = { - id: "doc_test_002", - title: "Test Document 002", - content: "This is test document 002.", -}; -describe("Banking Documents", () => { - let originalFetch: typeof global.fetch; - beforeEach(() => { - originalFetch = global.fetch; - seedBankingDocumentsCache([]); - }); - afterEach(() => { - global.fetch = originalFetch; - }); - describe("getBankingDocument", () => { - it("looks up by id from cache", () => { - seedBankingDocumentsCache([FIXTURE_DOC_001]); - const doc = getBankingDocument(FIXTURE_DOC_001.id); - expect(doc.id).toBe(FIXTURE_DOC_001.id); - expect(doc.title).toBe(FIXTURE_DOC_001.title); - }); - it("looks up by title from cache", () => { - seedBankingDocumentsCache([FIXTURE_DOC_001]); - const doc = getBankingDocument(FIXTURE_DOC_001.title); - expect(doc.id).toBe(FIXTURE_DOC_001.id); - expect(doc.content).toBe(FIXTURE_DOC_001.content); - }); - it("throws when document not found by id or title", () => { - seedBankingDocumentsCache([FIXTURE_DOC_001]); - expect(() => getBankingDocument("missing_doc_id")).toThrow(/not found/u); - }); - }); - describe("seedBankingDocumentsCache", () => { - it("stores multiple fixtures", () => { - seedBankingDocumentsCache([FIXTURE_DOC_001, FIXTURE_DOC_002]); - expect(getBankingDocument(FIXTURE_DOC_001.id).title).toBe( - FIXTURE_DOC_001.title - ); - expect(getBankingDocument(FIXTURE_DOC_002.id).title).toBe( - FIXTURE_DOC_002.title - ); - }); - it("clears previous cache on seed", () => { - seedBankingDocumentsCache([FIXTURE_DOC_001]); - expect(() => getBankingDocument(FIXTURE_DOC_002.id)).toThrow(); - seedBankingDocumentsCache([FIXTURE_DOC_002]); - expect(() => getBankingDocument(FIXTURE_DOC_001.id)).toThrow(); - }); - }); - describe("seeded document cache", () => { - it("stores documents that can be retrieved by id", () => { - seedBankingDocumentsCache([FIXTURE_DOC_001, FIXTURE_DOC_002]); - expect(getBankingDocument(FIXTURE_DOC_001.id).title).toBe( - FIXTURE_DOC_001.title - ); - expect(getBankingDocument(FIXTURE_DOC_002.id).title).toBe( - FIXTURE_DOC_002.title - ); - }); - }); - describe("ensureAllBankingDocuments", () => { - it("loads the corpus listing and document bodies once", async () => { - const requestedUrls: string[] = []; - const listingUrl = - "https://api.github.com/repos/sierra-research/tau2-bench/contents/data/tau2/domains/banking_knowledge/documents?ref=fc0055dc4e0a316c3f83133267fbd6faaa770992"; - const responses = new Map([ - [ - listingUrl, - [ - { - name: "doc_test_001.json", - download_url: "https://example.test/doc_test_001.json", - }, - { - name: "doc_test_002.json", - download_url: "https://example.test/doc_test_002.json", - }, - ], - ], - [ - "https://example.test/doc_test_001.json", - { id: "doc_test_001", title: "One", content: "First document." }, - ], - [ - "https://example.test/doc_test_002.json", - { id: "doc_test_002", title: "Two", content: "Second document." }, - ], - ]); - global.fetch = async (input, init) => { - const request = new Request(input, init); - requestedUrls.push(request.url); - const responseBody = responses.get(request.url); - assert(responseBody !== undefined); - return Response.json(responseBody); - }; - const semaphore = await runPromise(makeSemaphore(1)); - const load = ensureAllBankingDocuments(semaphore).pipe( - provide(FetchHttpClient.layer) - ); - await runPromise(load); - await runPromise(load); - expect(getAllBankingDocuments().map((document) => document.id)).toEqual([ - "doc_test_001", - "doc_test_002", - ]); - expect(requestedUrls).toHaveLength(3); - }); - it("normalizes a partially cached corpus to listing order", async () => { - seedBankingDocumentsCache([ - { id: "doc_test_002", title: "Two", content: "Second document." }, - ]); - const listingUrl = - "https://api.github.com/repos/sierra-research/tau2-bench/contents/data/tau2/domains/banking_knowledge/documents?ref=fc0055dc4e0a316c3f83133267fbd6faaa770992"; - const responses = new Map([ - [ - listingUrl, - [ - { - name: "doc_test_001.json", - download_url: "https://example.test/doc_test_001.json", - }, - { - name: "doc_test_002.json", - download_url: "https://example.test/doc_test_002.json", - }, - ], - ], - [ - "https://example.test/doc_test_001.json", - { id: "doc_test_001", title: "One", content: "First document." }, - ], - ]); - global.fetch = async (input, init) => { - const request = new Request(input, init); - const responseBody = responses.get(request.url); - assert(responseBody !== undefined); - return Response.json(responseBody); - }; - const semaphore = await runPromise(makeSemaphore(1)); - await runPromise( - ensureAllBankingDocuments(semaphore).pipe( - provide(FetchHttpClient.layer) - ) - ); - expect(getAllBankingDocuments().map((document) => document.id)).toEqual([ - "doc_test_001", - "doc_test_002", - ]); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/documents.ts b/src/benchmarks/tau3-bench-banking/documents.ts deleted file mode 100644 index a027e92b..00000000 --- a/src/benchmarks/tau3-bench-banking/documents.ts +++ /dev/null @@ -1,222 +0,0 @@ -import type { HttpClientError } from "@effect/platform"; -import { HttpClient } from "@effect/platform"; -import { TaggedError } from "effect/Data"; -import type { Effect, Semaphore } from "effect/Effect"; -import { fail, forEach, gen, succeed } from "effect/Effect"; - -import { Either } from "../../internal/either"; -import { parseSchema, z } from "../../internal/zod"; -import { - BANKING_SOURCE_BASE_URL, - BANKING_SOURCE_REVISION, -} from "./environment"; - -export interface BankingDocument { - readonly id: string; - readonly title: string; - readonly content: string; -} - -interface DocumentListingEntry { - readonly name: string; - readonly download_url: string; -} - -const BankingDocumentSchema: z.ZodType = z.object({ - id: z.string(), - title: z.string(), - content: z.string(), -}); - -const DocumentListingEntrySchema: z.ZodType = z.object({ - name: z.string(), - download_url: z.string(), -}); - -const DocumentListingSchema = z.array(DocumentListingEntrySchema); - -const BANKING_DOCUMENTS_CACHE = new Map(); - -let isFullCorpusCached = false; - -const DOCUMENT_BASE_URL = `${BANKING_SOURCE_BASE_URL}/documents`; - -const DOCUMENT_LISTING_URL = `https://api.github.com/repos/sierra-research/tau2-bench/contents/data/tau2/domains/banking_knowledge/documents?ref=${BANKING_SOURCE_REVISION}`; - -class DocumentFetchError extends TaggedError("DocumentFetchError")<{ - readonly message: string; -}> {} - -export function ensureBankingDocuments( - docIds: readonly string[], - fetchLock: Semaphore -): Effect< - void, - DocumentFetchError | HttpClientError.HttpClientError, - HttpClient.HttpClient -> { - if (docIds.every((id) => BANKING_DOCUMENTS_CACHE.has(id))) { - return succeed(undefined); - } - return fetchLock.withPermits(1)( - gen(function* () { - const client = yield* HttpClient.HttpClient; - const missingIds = docIds.filter( - (id) => !BANKING_DOCUMENTS_CACHE.has(id) - ); - const fetchedDocuments = yield* forEach( - missingIds, - (id) => { - const encodedId = encodeURIComponent(id); - const url = `${DOCUMENT_BASE_URL}/${encodedId}.json`; - return fetchBankingDocument({ client, id, url }); - }, - { concurrency: 16 } - ); - for (const document of fetchedDocuments) { - BANKING_DOCUMENTS_CACHE.set(document.id, document); - } - }) - ); -} - -export function ensureAllBankingDocuments( - fetchLock: Semaphore -): Effect< - void, - DocumentFetchError | HttpClientError.HttpClientError, - HttpClient.HttpClient -> { - if (isFullCorpusCached) { - return succeed(undefined); - } - return fetchLock.withPermits(1)( - gen(function* () { - if (isFullCorpusCached) { - return; - } - const client = yield* HttpClient.HttpClient; - const response = yield* client.get(DOCUMENT_LISTING_URL, { - headers: { - Accept: "application/vnd.github+json", - "User-Agent": "openrouter-bench-harness", - }, - }); - if (response.status < 200 || response.status >= 300) { - return yield* fail( - new DocumentFetchError({ - message: `HTTP ${response.status} when fetching banking document listing`, - }) - ); - } - const listingJson: unknown = yield* response.json; - const listingResult = parseSchema(DocumentListingSchema, listingJson); - if (Either.isLeft(listingResult)) { - return yield* fail( - new DocumentFetchError({ - message: `Invalid banking document listing: ${listingResult.left.message}`, - }) - ); - } - const missingEntries = listingResult.right.filter((entry) => { - const id = entry.name.replace(/\.json$/u, ""); - return entry.name.endsWith(".json") && !BANKING_DOCUMENTS_CACHE.has(id); - }); - const fetchedDocuments = yield* forEach( - missingEntries, - (entry) => { - const id = entry.name.replace(/\.json$/u, ""); - return fetchBankingDocument({ client, id, url: entry.download_url }); - }, - { concurrency: 16 } - ); - for (const document of fetchedDocuments) { - BANKING_DOCUMENTS_CACHE.set(document.id, document); - } - const orderedDocuments: BankingDocument[] = []; - for (const entry of listingResult.right) { - if (!entry.name.endsWith(".json")) { - continue; - } - const id = entry.name.replace(/\.json$/u, ""); - const document = BANKING_DOCUMENTS_CACHE.get(id); - if (document === undefined) { - return yield* fail( - new DocumentFetchError({ - message: `Banking document listing entry ${entry.name} did not load`, - }) - ); - } - orderedDocuments.push(document); - } - BANKING_DOCUMENTS_CACHE.clear(); - for (const document of orderedDocuments) { - BANKING_DOCUMENTS_CACHE.set(document.id, document); - } - isFullCorpusCached = true; - }) - ); -} - -export function getBankingDocument(idOrTitle: string): BankingDocument { - const byId = BANKING_DOCUMENTS_CACHE.get(idOrTitle); - if (byId) { - return byId; - } - for (const doc of BANKING_DOCUMENTS_CACHE.values()) { - if (doc.title === idOrTitle) { - return doc; - } - } - throw new Error( - `Banking document not found: ${idOrTitle} (not in cache by id or title)` - ); -} - -export function getAllBankingDocuments(): readonly BankingDocument[] { - return [...BANKING_DOCUMENTS_CACHE.values()]; -} - -export function seedBankingDocumentsCache( - docs: readonly BankingDocument[] -): void { - BANKING_DOCUMENTS_CACHE.clear(); - isFullCorpusCached = false; - for (const doc of docs) { - BANKING_DOCUMENTS_CACHE.set(doc.id, doc); - } -} - -function fetchBankingDocument({ - client, - id, - url, -}: { - readonly client: HttpClient.HttpClient; - readonly id: string; - readonly url: string; -}): Effect< - BankingDocument, - DocumentFetchError | HttpClientError.HttpClientError -> { - return gen(function* () { - const response = yield* client.get(url); - if (response.status < 200 || response.status >= 300) { - return yield* fail( - new DocumentFetchError({ - message: `HTTP ${response.status} when fetching document ${id}`, - }) - ); - } - const json: unknown = yield* response.json; - const parseResult = parseSchema(BankingDocumentSchema, json); - if (Either.isLeft(parseResult)) { - return yield* fail( - new DocumentFetchError({ - message: `Invalid document schema for ${id}: ${parseResult.left.message}`, - }) - ); - } - return parseResult.right; - }); -} diff --git a/src/benchmarks/tau3-bench-banking/environment.test.ts b/src/benchmarks/tau3-bench-banking/environment.test.ts deleted file mode 100644 index b3f777a0..00000000 --- a/src/benchmarks/tau3-bench-banking/environment.test.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { beforeEach, describe, expect, it } from "bun:test"; - -import { Either } from "../../internal/either"; -import { parseSchema } from "../../internal/zod"; -import { - applyInitialState, - dbHash, - loadBankingData, - loadBankingTasks, - makeEmptyBankingData, - seedBankingCache, - seedBankingTasksRawCache, -} from "./environment"; -import type { BankingData, Tau3Task } from "./types"; -import { Tau3TaskSchema } from "./types"; -describe("tau3-bench-banking environment", () => { - const fixtureDb: BankingData = makeEmptyBankingData(); - const FIXTURE_TABLES: Partial = { - users: { - data: { - "123": { name: "Sarah Bosch", user_id: "123" }, - }, - }, - accounts: { - data: { - acc_1: { account_id: "acc_1", account_type: "checking" }, - }, - }, - debit_cards: { - data: { - card_1: { card_id: "card_1", cardholder: "Sarah Bosch" }, - }, - }, - }; - Object.assign(fixtureDb, FIXTURE_TABLES); - const fixtureTaskWithoutInitialState: Tau3Task = { - id: "task_001", - description: { - purpose: "Task: task_001", - relevant_policies: null, - notes: null, - }, - user_scenario: { - persona: null, - instructions: "You are playing the role of a customer...", - }, - initial_state: null, - evaluation_criteria: { - actions: [ - { - action_id: "001_0", - requestor: "user", - name: "apply_for_credit_card", - arguments: { card_type: "Gold Rewards Card" }, - }, - ], - communicate_info: [], - reward_basis: ["DB"], - }, - annotations: null, - user_tools: ["apply_for_credit_card"], - required_documents: ["doc_credit_cards_gold_rewards_card_001"], - }; - const fixtureTaskWithInitialState: Tau3Task = { - id: "task_026", - description: { - purpose: "Task: task_026", - relevant_policies: null, - notes: null, - }, - user_scenario: { - persona: null, - instructions: "You have noticed that your recent cash back rewards...", - }, - initial_state: { - initialization_data: { - agent_data: { - task_config: { - data: { dispute_settings: { auto_resolve_disputes: true } }, - }, - }, - }, - initialization_actions: null, - message_history: null, - }, - evaluation_criteria: { - actions: [ - { - action_id: "026_0", - requestor: "user", - name: "log_verification", - arguments: {}, - }, - ], - communicate_info: [], - reward_basis: ["DB"], - }, - annotations: null, - user_tools: [], - required_documents: [], - }; - beforeEach(() => { - seedBankingCache(fixtureDb, [ - fixtureTaskWithoutInitialState, - fixtureTaskWithInitialState, - ]); - }); - describe("loadBankingData", () => { - it("returns a mutable copy of the database", () => { - const data1 = loadBankingData(); - const data2 = loadBankingData(); - expect(data1).not.toBe(data2); - data1.users.data["999"] = { name: "New User" }; - expect(data2.users.data["999"]).toBeUndefined(); - }); - it("normalizes all 23 tables even if only 3 are in fixture", () => { - const data = loadBankingData(); - expect(Object.keys(data).length).toBe(23); - expect(data.users.data["123"]).toEqual({ - name: "Sarah Bosch", - user_id: "123", - }); - expect(data.task_config.data).toEqual({}); - expect(data.task_config.notes).toBe(""); - }); - it('fills missing tables with empty data and notes=""', () => { - const data = loadBankingData(); - expect(data.agent_discoverable_tools).toEqual({ data: {}, notes: "" }); - expect(data.credit_card_orders).toEqual({ data: {}, notes: "" }); - }); - }); - describe("loadBankingTasks", () => { - it("returns parsed and validated tasks array", () => { - const tasks = loadBankingTasks(); - expect(tasks.length).toBe(2); - expect(tasks[0]?.id).toBe("task_001"); - expect(tasks[1]?.id).toBe("task_026"); - }); - it("returns an independent array copy per call", () => { - const tasks1 = loadBankingTasks(); - const tasks2 = loadBankingTasks(); - expect(tasks1).not.toBe(tasks2); - tasks1.pop(); - expect(tasks2.length).toBe(2); - }); - it("validates each task against schema", () => { - const invalidTask = { - id: "invalid", - description: null, - user_scenario: { instructions: "Test" }, - initial_state: null, - evaluation_criteria: { reward_basis: ["INVALID"] }, - }; - const invalidParsed = parseSchema(Tau3TaskSchema, invalidTask); - if (Either.isRight(invalidParsed)) { - throw new Error("fixture unexpectedly valid"); - } - seedBankingTasksRawCache(JSON.stringify([invalidTask])); - expect(() => loadBankingTasks()).toThrow( - /Invalid banking task at index 0/ - ); - }); - it("returns empty array when seeded with no tasks", () => { - seedBankingCache(fixtureDb, []); - const tasks = loadBankingTasks(); - expect(tasks.length).toBe(0); - }); - }); - describe("applyInitialState", () => { - it("merges task_config.data into db.task_config.data", () => { - const db = loadBankingData(); - applyInitialState(db, fixtureTaskWithInitialState); - expect(db.task_config.data.dispute_settings).toEqual({ - auto_resolve_disputes: true, - }); - }); - it("deep-merges partial overlays for every banking table", () => { - const db = loadBankingData(); - const task: Tau3Task = { - ...fixtureTaskWithoutInitialState, - initial_state: { - initialization_data: { - agent_data: { - accounts: { - data: { - acc_1: { status: "CLOSED" }, - }, - }, - users: { - data: { - "123": { phone_number: "555-0100" }, - }, - }, - }, - }, - initialization_actions: null, - message_history: null, - }, - }; - applyInitialState(db, task); - expect(db.accounts.data.acc_1).toEqual({ - account_id: "acc_1", - account_type: "checking", - status: "CLOSED", - }); - expect(db.users.data["123"]).toEqual({ - name: "Sarah Bosch", - user_id: "123", - phone_number: "555-0100", - }); - }); - it("rejects overlays for unknown tables", () => { - const db = loadBankingData(); - const task: Tau3Task = { - ...fixtureTaskWithoutInitialState, - initial_state: { - initialization_data: { - agent_data: { unknown_table: { data: {} } }, - }, - initialization_actions: null, - message_history: null, - }, - }; - expect(() => applyInitialState(db, task)).toThrow( - "unknown table 'unknown_table'" - ); - }); - it("does nothing for null initial_state", () => { - const db = loadBankingData(); - const taskConfigBefore = JSON.stringify(db.task_config.data); - applyInitialState(db, fixtureTaskWithoutInitialState); - expect(JSON.stringify(db.task_config.data)).toBe(taskConfigBefore); - }); - it("does nothing for null initialization_data", () => { - const db = loadBankingData(); - const task = { - ...fixtureTaskWithoutInitialState, - initial_state: { - initialization_data: null, - initialization_actions: null, - message_history: null, - }, - }; - const taskConfigBefore = JSON.stringify(db.task_config.data); - applyInitialState(db, task); - expect(JSON.stringify(db.task_config.data)).toBe(taskConfigBefore); - }); - it("throws if initialization_actions is non-null", () => { - const db = loadBankingData(); - const task = { - ...fixtureTaskWithoutInitialState, - initial_state: { - initialization_data: { agent_data: { task_config: { data: {} } } }, - initialization_actions: [{ action: "test" }] satisfies unknown, - message_history: null, - }, - }; - expect(() => applyInitialState(db, task)).toThrow( - /initialization_actions/ - ); - }); - it("throws if message_history is non-null", () => { - const db = loadBankingData(); - const task = { - ...fixtureTaskWithoutInitialState, - initial_state: { - initialization_data: { agent_data: { task_config: { data: {} } } }, - initialization_actions: null, - message_history: [{ role: "user" }] satisfies unknown, - }, - }; - expect(() => applyInitialState(db, task)).toThrow(/message_history/); - }); - it("hash includes applied task_config", () => { - const db = loadBankingData(); - const hashBefore = dbHash(db); - applyInitialState(db, fixtureTaskWithInitialState); - const hashAfter = dbHash(db); - expect(hashBefore).not.toBe(hashAfter); - }); - }); - describe("dbHash", () => { - it("produces consistent hash for same data", () => { - const data1 = loadBankingData(); - const hash1 = dbHash(data1); - const hash2 = dbHash(data1); - expect(hash1).toBe(hash2); - }); - it("produces different hash for mutated data", () => { - const data = loadBankingData(); - const hashBefore = dbHash(data); - data.users.data["999"] = { name: "New User" }; - const hashAfter = dbHash(data); - expect(hashBefore).not.toBe(hashAfter); - }); - it("hashes are 64 hex characters (sha256)", () => { - const data = loadBankingData(); - const hash = dbHash(data); - expect(hash).toMatch(/^[a-f0-9]{64}$/); - }); - it("hash covers injected task_config", () => { - const data1 = loadBankingData(); - const hash1 = dbHash(data1); - const data2 = loadBankingData(); - if (data2.task_config) { - data2.task_config.data.test_key = { value: "test_value" }; - } - const hash2 = dbHash(data2); - expect(hash1).not.toBe(hash2); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/environment.ts b/src/benchmarks/tau3-bench-banking/environment.ts deleted file mode 100644 index 966c32a4..00000000 --- a/src/benchmarks/tau3-bench-banking/environment.ts +++ /dev/null @@ -1,282 +0,0 @@ -import { createHash } from "node:crypto"; - -import type { HttpClient, HttpClientError } from "@effect/platform"; -import type { Effect, Semaphore } from "effect/Effect"; -import { gen } from "effect/Effect"; - -import type { CachedFileError } from "../../datasets/cached-file"; -import { - fetchCachedTextFile, - jsonTextValidator, -} from "../../datasets/cached-file"; -import { Either } from "../../internal/either"; -import { - definedValues, - isDefinedAndNotNull, - isRecord, -} from "../../internal/guards"; -import { parseSchema } from "../../internal/zod"; -import type { RetryConfig } from "../../runtime/retry"; -import type { BankingData, BankingTable, Tau3Task } from "./types"; -import { BANKING_TABLES, isBankingTableName, Tau3TaskSchema } from "./types"; - -export const BANKING_SOURCE_REVISION = - "fc0055dc4e0a316c3f83133267fbd6faaa770992"; - -export const BANKING_SOURCE_BASE_URL = `https://raw.githubusercontent.com/sierra-research/tau2-bench/${BANKING_SOURCE_REVISION}/data/tau2/domains/banking_knowledge`; - -let bankingDbCache: string | undefined; - -let bankingTasksCache: string | undefined; - -function fetchGithubFile( - filename: string, - expected: "object" | "array", - retryConfig?: RetryConfig -): Effect< - string, - CachedFileError | HttpClientError.HttpClientError, - HttpClient.HttpClient -> { - return fetchCachedTextFile({ - url: `${BANKING_SOURCE_BASE_URL}/${filename}`, - validate: jsonTextValidator(expected), - ...definedValues({ retry: retryConfig }), - }); -} - -export function ensureBankingData( - fetchLock: Semaphore, - retryConfig?: RetryConfig -): Effect< - void, - CachedFileError | HttpClientError.HttpClientError, - HttpClient.HttpClient -> { - return fetchLock.withPermits(1)( - gen(function* () { - if (bankingDbCache) { - return; - } - bankingDbCache = yield* fetchGithubFile("db.json", "object", retryConfig); - }) - ); -} - -export function ensureBankingTasks( - fetchLock: Semaphore, - retryConfig?: RetryConfig -): Effect< - void, - CachedFileError | HttpClientError.HttpClientError, - HttpClient.HttpClient -> { - return fetchLock.withPermits(1)( - gen(function* () { - if (bankingTasksCache) { - return; - } - bankingTasksCache = yield* fetchGithubFile( - "tasks.json", - "array", - retryConfig - ); - }) - ); -} - -export function seedBankingTasksRawCache(rawJson: string): void { - bankingTasksCache = rawJson; -} - -export function seedBankingCache(data: BankingData, tasks: Tau3Task[]): void { - bankingDbCache = JSON.stringify(data); - bankingTasksCache = JSON.stringify(tasks); -} - -function emptyTable(): BankingTable { - return { data: {}, notes: "" }; -} - -export function makeEmptyBankingData(): BankingData { - return { - users: emptyTable(), - accounts: emptyTable(), - debit_cards: emptyTable(), - referrals: emptyTable(), - credit_card_applications: emptyTable(), - user_discoverable_tools: emptyTable(), - user_discoverable_tool_calls: emptyTable(), - verification_history: emptyTable(), - credit_card_transaction_history: emptyTable(), - cash_back_disputes: emptyTable(), - bank_account_transaction_history: emptyTable(), - credit_card_accounts: emptyTable(), - agent_discoverable_tools: emptyTable(), - task_config: emptyTable(), - human_transfer_requests: emptyTable(), - transaction_disputes: emptyTable(), - credit_card_orders: emptyTable(), - debit_card_orders: emptyTable(), - credit_card_closure_reasons: emptyTable(), - credit_card_account_flags: emptyTable(), - credit_limit_increase_requests: emptyTable(), - payment_history: emptyTable(), - debit_card_disputes: emptyTable(), - }; -} - -export function loadBankingData(): BankingData { - const raw = bankingDbCache; - if (!raw) { - throw new Error("Banking data not loaded — call ensureBankingData() first"); - } - const result = Either.try((): Record => JSON.parse(raw)); - if (Either.isLeft(result)) { - throw new Error("Invalid cached banking JSON"); - } - const parsed = result.right; - const normalized = makeEmptyBankingData(); - if (!isRecord(parsed)) { - return normalized; - } - for (const table of BANKING_TABLES) { - const existing = parsed[table]; - if (isRecord(existing) && isRecord(existing.data)) { - const rows: Record> = {}; - for (const [rowId, row] of Object.entries(existing.data)) { - if (isRecord(row)) { - rows[rowId] = row; - } - } - normalized[table] = { data: rows, notes: existing.notes ?? "" }; - } - } - return normalized; -} - -export function loadBankingTasks(): Tau3Task[] { - const raw = bankingTasksCache; - if (!raw) { - throw new Error( - "Banking tasks not loaded — call ensureBankingTasks() first" - ); - } - const result = Either.try((): unknown => JSON.parse(raw)); - if (Either.isLeft(result)) { - throw new Error("Invalid cached banking tasks JSON"); - } - const tasksArray = result.right; - if (!Array.isArray(tasksArray)) { - throw new TypeError("Banking tasks is not an array"); - } - const validated: Tau3Task[] = []; - for (let i = 0; i < tasksArray.length; i++) { - const task = tasksArray[i]; - const parseResult = parseSchema(Tau3TaskSchema, task); - if (Either.isLeft(parseResult)) { - throw new Error( - `Invalid banking task at index ${i}: ${String(parseResult.left)}` - ); - } - validated.push(parseResult.right); - } - return validated; -} - -const PROTOTYPE_KEYS = new Set(["__proto__", "constructor", "prototype"]); - -function deepMergeRecord( - base: Record, - overlay: Record -): void { - for (const [key, overlayValue] of Object.entries(overlay)) { - if (PROTOTYPE_KEYS.has(key)) { - throw new Error( - `Banking initial_state overlay contains forbidden key '${key}'` - ); - } - const baseValue = base[key]; - if (isRecord(baseValue) && isRecord(overlayValue)) { - deepMergeRecord(baseValue, overlayValue); - } else { - base[key] = overlayValue; - } - } -} - -export function applyInitialState(db: BankingData, task: Tau3Task): void { - const state = task.initial_state; - if (!state) { - return; - } - if (isDefinedAndNotNull(state.initialization_actions)) { - throw new Error( - `Banking task ${task.id} has non-null initialization_actions (not supported)` - ); - } - if (isDefinedAndNotNull(state.message_history)) { - throw new Error( - `Banking task ${task.id} has non-null message_history (not supported)` - ); - } - const agentData = state.initialization_data?.agent_data; - if (!isRecord(agentData)) { - return; - } - for (const [tableName, overlay] of Object.entries(agentData)) { - if (!isBankingTableName(tableName)) { - throw new Error( - `Banking task ${task.id} initial_state references unknown table '${tableName}'` - ); - } - if (!isRecord(overlay)) { - throw new Error( - `Banking task ${task.id} initial_state has a non-record overlay for table '${tableName}'` - ); - } - const overlayData = overlay["data"]; - if (isRecord(overlayData)) { - deepMergeRecord(db[tableName].data, overlayData); - } - const overlayNotes = overlay["notes"]; - if (overlayNotes !== undefined) { - db[tableName].notes = overlayNotes; - } - } -} - -type Hashable = - | string - | number - | boolean - | null - | readonly Hashable[] - | readonly [string, Hashable][]; - -function toHashable(item: unknown): Hashable { - if (item === null || item === undefined) { - return null; - } - if ( - typeof item === "string" || - typeof item === "number" || - typeof item === "boolean" - ) { - return item; - } - if (Array.isArray(item)) { - return item.map(toHashable); - } - if (isRecord(item)) { - const entries = Object.entries(item); - entries.sort(([a], [b]) => a.localeCompare(b)); - return entries.map(([k, v]): [string, Hashable] => [k, toHashable(v)]); - } - return String(item); -} - -export function dbHash(data: BankingData): string { - const hashable = toHashable(data); - return createHash("sha256").update(JSON.stringify(hashable)).digest("hex"); -} diff --git a/src/benchmarks/tau3-bench-banking/evaluator.test.ts b/src/benchmarks/tau3-bench-banking/evaluator.test.ts deleted file mode 100644 index 24ff1951..00000000 --- a/src/benchmarks/tau3-bench-banking/evaluator.test.ts +++ /dev/null @@ -1,400 +0,0 @@ -import { beforeEach, describe, expect, it } from "bun:test"; - -import { makeEmptyBankingData, seedBankingCache } from "./environment"; -import { evaluateSimulation } from "./evaluator"; -import { TerminationReason } from "./solver"; -import type { Tau3Task } from "./types"; -beforeEach(() => { - const emptyDb = makeEmptyBankingData(); - seedBankingCache(emptyDb, []); -}); - -function makeTask(overrides?: Partial): Tau3Task { - return { - id: "task_test_001", - user_scenario: { instructions: "test" }, - evaluation_criteria: { - actions: [], - communicate_info: [], - reward_basis: ["DB"], - }, - ...overrides, - }; -} -describe("evaluateSimulation", () => { - describe("termination checks", () => { - it("returns zero reward for MAX_STEPS termination", () => { - const task = makeTask(); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.MaxSteps, - }); - expect(result.reward).toBe(0); - expect(result.note).toContain("prematurely"); - }); - it("returns zero reward for TOO_MANY_ERRORS termination", () => { - const task = makeTask(); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.TooManyErrors, - }); - expect(result.reward).toBe(0); - expect(result.note).toContain("prematurely"); - }); - it("accepts AGENT_STOP termination", () => { - const task = makeTask(); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.AgentStop, - }); - expect(result.reward).toBe(1); - expect(result.dbMatch).toBe(true); - }); - it("accepts USER_STOP termination", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [], - communicate_info: [], - reward_basis: ["DB"], - }, - }); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(1); - expect(result.dbMatch).toBe(true); - }); - }); - describe("null criteria", () => { - it("returns perfect reward when criteria is null", () => { - const task = makeTask({ evaluation_criteria: null }); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(1); - expect(result.note).toContain("No evaluation criteria"); - }); - it("returns perfect reward when task is undefined", () => { - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task: undefined, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(1); - expect(result.note).toContain("No evaluation criteria"); - }); - }); - describe("DB basis", () => { - it("returns 1 when golden actions is null", () => { - const task = makeTask({ - evaluation_criteria: { - actions: null, - communicate_info: [], - reward_basis: ["DB"], - }, - }); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(1); - expect(result.dbMatch).toBe(true); - }); - it("returns 1 when agent DB matches empty golden DB", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [], - communicate_info: [], - reward_basis: ["DB"], - }, - }); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(1); - expect(result.dbMatch).toBe(true); - }); - it("returns 0 when DB hashes do not match", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [], - communicate_info: [], - reward_basis: ["DB"], - }, - }); - const agentData = makeEmptyBankingData(); - agentData.users.data["user_1"] = { name: "Test User" }; - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(0); - expect(result.dbMatch).toBe(false); - }); - }); - describe("ACTION basis", () => { - it("returns 1 when no golden actions required", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [], - communicate_info: [], - reward_basis: ["ACTION"], - }, - }); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(1); - expect(result.actionMet).toBe(true); - }); - it("returns 1 when all golden actions are matched", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [ - { - action_id: "a1", - name: "get_user_information_by_id", - arguments: { user_id: "user_123" }, - requestor: "assistant", - }, - ], - communicate_info: [], - reward_basis: ["ACTION"], - }, - }); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [ - { - name: "get_user_information_by_id", - arguments: { user_id: "user_123" }, - requestor: "assistant", - }, - ], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(1); - expect(result.actionMet).toBe(true); - }); - it("returns 0 when a golden action is missing", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [ - { - action_id: "a1", - name: "get_user_information_by_id", - arguments: { user_id: "user_123" }, - requestor: "assistant", - }, - ], - communicate_info: [], - reward_basis: ["ACTION"], - }, - }); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(0); - expect(result.actionMet).toBe(false); - }); - }); - describe("COMMUNICATE basis", () => { - it("returns 1 when no communicate_info required", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [], - communicate_info: [], - reward_basis: ["COMMUNICATE"], - }, - }); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: ["Hello"], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(1); - expect(result.communicateMet).toBe(true); - }); - it("returns 1 when all communicate_info appears in texts", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [], - communicate_info: ["hello", "world"], - reward_basis: ["COMMUNICATE"], - }, - }); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: ["hello there", "world is great"], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(1); - expect(result.communicateMet).toBe(true); - }); - it("returns 0 when communicate_info is missing", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [], - communicate_info: ["hello", "world"], - reward_basis: ["COMMUNICATE"], - }, - }); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: ["hello there"], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(0); - expect(result.communicateMet).toBe(false); - }); - }); - describe("reward basis composition", () => { - it("multiplies DB and ACTION when both in basis", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [ - { - action_id: "a1", - name: "some_tool", - arguments: {}, - requestor: "assistant", - }, - ], - communicate_info: [], - reward_basis: ["DB", "ACTION"], - }, - }); - const agentData = makeEmptyBankingData(); - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [ - { - name: "some_tool", - arguments: {}, - requestor: "assistant", - }, - ], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(1); - expect(result.breakdown.db).toBe(1); - expect(result.breakdown.action).toBe(1); - }); - it("returns 0 when DB basis fails even if ACTION passes", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [ - { - action_id: "a1", - name: "some_tool", - arguments: {}, - requestor: "assistant", - }, - ], - communicate_info: [], - reward_basis: ["DB", "ACTION"], - }, - }); - const agentData = makeEmptyBankingData(); - agentData.users.data["user_1"] = { name: "Test" }; - const result = evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [ - { - name: "some_tool", - arguments: {}, - requestor: "assistant", - }, - ], - terminationReason: TerminationReason.UserStop, - }); - expect(result.reward).toBe(0); - }); - }); - describe("unsupported reward bases", () => { - it("throws instead of silently satisfying an unimplemented reward basis", () => { - const task = makeTask({ - evaluation_criteria: { - actions: [], - communicate_info: [], - reward_basis: ["ENV_ASSERTION"], - }, - }); - const agentData = makeEmptyBankingData(); - expect(() => - evaluateSimulation({ - task, - agentData, - assistantTexts: [], - toolCalls: [], - terminationReason: TerminationReason.UserStop, - }) - ).toThrow( - "Banking task task_test_001 uses unsupported reward basis: ENV_ASSERTION" - ); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/evaluator.ts b/src/benchmarks/tau3-bench-banking/evaluator.ts deleted file mode 100644 index 73cf4fda..00000000 --- a/src/benchmarks/tau3-bench-banking/evaluator.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { isDefinedAndNotNull } from "../../internal/guards"; -import { compareActionWithToolCall } from "./action-match"; -import { applyInitialState, dbHash, loadBankingData } from "./environment"; -import { TerminationReason } from "./solver"; -import { - invokeBankingAgentTool, - registerInitialDiscoverableTools, -} from "./tools/handlers-meta"; -import { invokeBankingUserTool } from "./tools/handlers-user"; -import { makeBankingEnvState } from "./tools/registry"; -import type { BankingData, Tau3Action, Tau3Task } from "./types"; -import { - DEFAULT_REWARD_BASIS, - deriveReadLogAllowlist, - RewardType, -} from "./types"; - -export interface PredictedToolCall { - readonly name: string; - readonly arguments: Readonly>; - readonly requestor: "assistant" | "user"; -} - -export interface RewardBreakdown { - readonly db: number; - readonly action: number; - readonly communicate: number; -} - -export interface EvaluationResult { - readonly reward: number; - readonly dbMatch: boolean; - readonly actionMet: boolean; - readonly communicateMet: boolean; - readonly terminationReason: string; - readonly note?: string; - readonly breakdown: RewardBreakdown; -} - -export interface EvaluateSimulationOpts { - readonly task: Tau3Task | undefined; - readonly agentData: BankingData; - readonly assistantTexts: readonly string[]; - readonly toolCalls?: readonly PredictedToolCall[]; - readonly terminationReason: string; -} - -export function evaluateSimulation( - opts: EvaluateSimulationOpts -): EvaluationResult { - const { - task, - agentData, - assistantTexts, - toolCalls = [], - terminationReason, - } = opts; - const zero = (note: string): EvaluationResult => ({ - reward: 0, - dbMatch: false, - actionMet: false, - communicateMet: false, - terminationReason, - note, - breakdown: { db: 0, action: 0, communicate: 0 }, - }); - if ( - terminationReason !== TerminationReason.UserStop && - terminationReason !== TerminationReason.AgentStop - ) { - return zero(`Simulation terminated prematurely: ${terminationReason}`); - } - const criteria = task?.evaluation_criteria; - if (!task || criteria === null || criteria === undefined) { - return { - reward: 1, - dbMatch: true, - actionMet: true, - communicateMet: true, - terminationReason, - note: "No evaluation criteria", - breakdown: { db: 1, action: 1, communicate: 1 }, - }; - } - const basis = criteria.reward_basis ?? DEFAULT_REWARD_BASIS; - const unsupportedBases = basis.filter( - (rewardBasis) => !isSupportedRewardBasis(rewardBasis) - ); - if (unsupportedBases.length > 0) { - throw new Error( - `Banking task ${task.id} uses unsupported reward basis: ${unsupportedBases.join(", ")}` - ); - } - const dbReward = computeDbReward(criteria.actions, agentData, task); - const actionReward = computeActionReward(criteria.actions ?? [], toolCalls); - const communicateReward = computeCommunicateReward( - criteria.communicate_info ?? [], - assistantTexts - ); - let reward = 1; - if (basis.includes(RewardType.Db)) { - reward *= dbReward; - } - if (basis.includes(RewardType.Action)) { - reward *= actionReward; - } - if (basis.includes(RewardType.Communicate)) { - reward *= communicateReward; - } - return { - reward, - dbMatch: dbReward >= 1, - actionMet: actionReward >= 1, - communicateMet: communicateReward >= 1, - terminationReason, - breakdown: { - db: dbReward, - action: actionReward, - communicate: communicateReward, - }, - }; -} - -function isSupportedRewardBasis(basis: RewardType): boolean { - switch (basis) { - case RewardType.Db: { - return true; - } - case RewardType.Action: { - return true; - } - case RewardType.Communicate: { - return true; - } - case RewardType.EnvAssertion: { - return false; - } - case RewardType.NlAssertion: { - return false; - } - default: { - return basis satisfies never; - } - } -} - -function computeDbReward( - goldenActions: readonly Tau3Action[] | null | undefined, - agentData: BankingData, - task: Tau3Task -): number { - if (!isDefinedAndNotNull(goldenActions)) { - return 1; - } - const goldData = loadBankingData(); - applyInitialState(goldData, task); - registerInitialDiscoverableTools(); - const envState = makeBankingEnvState(goldData, deriveReadLogAllowlist(task)); - for (const action of goldenActions) { - if (action.requestor === "user") { - invokeBankingUserTool(envState, action.name, action.arguments); - } else { - invokeBankingAgentTool(envState, action.name, action.arguments); - } - } - return dbHash(goldData) === dbHash(agentData) ? 1 : 0; -} - -function computeActionReward( - goldenActions: readonly Tau3Action[], - toolCalls: readonly PredictedToolCall[] -): number { - if (goldenActions.length === 0) { - return 1; - } - const allMet = goldenActions.every((action) => - toolCalls.some( - (tc) => - tc.requestor === action.requestor && - compareActionWithToolCall(action, tc.name, tc.arguments) - ) - ); - return allMet ? 1 : 0; -} - -function computeCommunicateReward( - communicateInfo: readonly string[], - assistantTexts: readonly string[] -): number { - if (communicateInfo.length === 0) { - return 1; - } - const concatenated = assistantTexts.join(" "); - const allMet = communicateInfo.every((info) => concatenated.includes(info)); - return allMet ? 1 : 0; -} diff --git a/src/benchmarks/tau3-bench-banking/retrieval-config.test.ts b/src/benchmarks/tau3-bench-banking/retrieval-config.test.ts deleted file mode 100644 index 52266879..00000000 --- a/src/benchmarks/tau3-bench-banking/retrieval-config.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { describe, expect, it } from "bun:test"; - -import { assertRight } from "../../internal/testing"; -import { parseSchema } from "../../internal/zod"; -import { BankingRetrievalConfigSchema } from "./retrieval-config"; -describe("BankingRetrievalConfigSchema", () => { - it("defaults to bm25_grep for AA parity", () => { - const result = parseSchema(BankingRetrievalConfigSchema, undefined); - assertRight(result); - expect(result.right).toBe("bm25_grep"); - }); - it("accepts bm25_grep", () => { - const result = parseSchema(BankingRetrievalConfigSchema, "bm25_grep"); - assertRight(result); - expect(result.right).toBe("bm25_grep"); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/retrieval-config.ts b/src/benchmarks/tau3-bench-banking/retrieval-config.ts deleted file mode 100644 index 5d901c78..00000000 --- a/src/benchmarks/tau3-bench-banking/retrieval-config.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { z } from "../../internal/zod"; - -export const BANKING_RETRIEVAL_CONFIGS = [ - "required_docs", - "bm25_grep", -] as const; - -export const DEFAULT_BANKING_RETRIEVAL_CONFIG = "bm25_grep" as const; - -export const BankingRetrievalConfigSchema = z - .enum(BANKING_RETRIEVAL_CONFIGS) - .default(DEFAULT_BANKING_RETRIEVAL_CONFIG); - -export type BankingRetrievalConfig = z.infer< - typeof BankingRetrievalConfigSchema ->; diff --git a/src/benchmarks/tau3-bench-banking/retrieval.test.ts b/src/benchmarks/tau3-bench-banking/retrieval.test.ts deleted file mode 100644 index 4019b639..00000000 --- a/src/benchmarks/tau3-bench-banking/retrieval.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { describe, expect, it } from "bun:test"; - -import { - formatGrepResults, - formatKbSearchResults, - makeBankingRetrievalTools, - makeBm25Index, - searchBm25, - searchGrep, -} from "./retrieval"; - -const DOCUMENTS = [ - { - id: "doc_checking", - title: "Checking Accounts", - content: "checking account overdraft fee checking", - }, - { - id: "doc_credit", - title: "Credit Cards", - content: "credit card annual fee and rewards", - }, - { - id: "doc_disputes", - title: "Card Disputes", - content: "credit card dispute chargeback chargeback", - }, -] as const; -describe("tau3 banking retrieval", () => { - it("ranks BM25 documents with upstream whitespace tokenization", () => { - const index = makeBm25Index(DOCUMENTS); - const results = searchBm25({ index, query: "annual rewards", topK: 2 }); - expect(results.map((result) => result.document.id)).toEqual([ - "doc_credit", - "doc_checking", - ]); - expect(results[0]?.score).toBeGreaterThan(0); - expect(results[1]?.score).toBe(0); - }); - it("keeps punctuation attached to tokens like rank-bm25 0.2.2", () => { - const index = makeBm25Index(DOCUMENTS); - const results = searchBm25({ index, query: "fee,", topK: 1 }); - expect(results[0]?.document.id).toBe("doc_checking"); - expect(results[0]?.score).toBe(0); - }); - it("formats KB_search results with upstream labels and timing", () => { - const index = makeBm25Index(DOCUMENTS); - const results = searchBm25({ index, query: "annual rewards", topK: 1 }); - const formatted = formatKbSearchResults({ results, retrievalMs: 12.4 }); - expect(formatted).toMatchInlineSnapshot(` - "1. Credit Cards - ID: doc_credit - Score: 0.9672 - Content: credit card annual fee and rewards - - - [Timing: retrieval=12ms, total=12ms]" - `); - }); - it("ranks grep matches by count and formats full documents", () => { - const results = searchGrep({ - documents: DOCUMENTS, - pattern: "chargeback|annual", - }); - expect(results.map((result) => result.document.id)).toEqual([ - "doc_disputes", - "doc_credit", - ]); - const formatted = formatGrepResults({ - results, - pattern: "chargeback|annual", - }); - expect(formatted).toContain("Score: 2.0000"); - expect(formatted).toContain("Content: credit card dispute"); - }); - it("falls back to a literal grep pattern when the regex is invalid", () => { - const documents = [ - { - id: "doc_literal", - title: "Literal", - content: "Call support (option one).", - }, - ]; - const results = searchGrep({ documents, pattern: "(" }); - expect(results).toHaveLength(1); - expect(results[0]?.score).toBe(1); - }); - it("uses upstream empty-result strings", () => { - expect(formatKbSearchResults({ results: [], retrievalMs: 0 })).toBe( - "No relevant documents found.\n\n[Timing: retrieval=0ms, total=0ms]" - ); - expect(formatGrepResults({ results: [], pattern: "missing" })).toBe( - "No matches found for pattern: missing" - ); - }); - it("exposes retrieval tools only for bm25_grep", () => { - const goldenTools = makeBankingRetrievalTools({ - documents: DOCUMENTS, - retrievalConfig: "required_docs", - }); - const bm25Tools = makeBankingRetrievalTools({ - documents: DOCUMENTS, - retrievalConfig: "bm25_grep", - }); - expect(goldenTools.definitions).toEqual([]); - expect(bm25Tools.definitions.map((tool) => tool.function.name)).toEqual([ - "KB_search", - "grep", - ]); - expect(bm25Tools.definitions[1]?.function.description).toBe( - "Search for a regex pattern in all knowledge base documents.\n\nReturns documents ranked by number of matches, with full content." - ); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/retrieval.ts b/src/benchmarks/tau3-bench-banking/retrieval.ts deleted file mode 100644 index 27e44aa8..00000000 --- a/src/benchmarks/tau3-bench-banking/retrieval.ts +++ /dev/null @@ -1,351 +0,0 @@ -import type { BankingRetrievalConfig } from "./retrieval-config"; -import type { ToolDefinition } from "./tools/definitions"; - -export interface RetrievalDocument { - readonly id: string; - readonly title: string; - readonly content: string; -} - -interface IndexedDocument { - readonly document: RetrievalDocument; - readonly length: number; - readonly termFrequencies: ReadonlyMap; -} - -export interface Bm25Index { - readonly documents: readonly IndexedDocument[]; - readonly averageDocumentLength: number; - readonly inverseDocumentFrequencies: ReadonlyMap; -} - -export interface RetrievalResult { - readonly document: RetrievalDocument; - readonly score: number; -} - -export interface BankingRetrievalTools { - readonly definitions: readonly ToolDefinition[]; - readonly invoke: ( - name: string, - arguments_: Readonly> - ) => string | undefined; -} - -const BM25_K1 = 1.5; - -const BM25_B = 0.75; - -const BM25_EPSILON = 0.25; - -const DEFAULT_TOP_K = 10; - -export const BM25_GREP_TOOL_DEFINITIONS: readonly ToolDefinition[] = [ - { - type: "function", - function: { - name: "KB_search", - description: "Search the knowledge base for relevant documents.", - parameters: { - type: "object", - properties: { - query: { - type: "string", - description: "The search query to find relevant documents", - }, - }, - required: ["query"], - }, - }, - }, - { - type: "function", - function: { - name: "grep", - description: - "Search for a regex pattern in all knowledge base documents.\n\nReturns documents ranked by number of matches, with full content.", - parameters: { - type: "object", - properties: { - pattern: { - type: "string", - description: - "The regex pattern to search for (e.g., 'credit.*card', 'fee|charge')", - }, - }, - required: ["pattern"], - }, - }, - }, -]; - -export function makeBm25Index( - documents: readonly RetrievalDocument[] -): Bm25Index { - const tokenizedDocuments = documents.map((document) => - tokenize(document.content) - ); - const indexedDocuments = documents.map((document, index): IndexedDocument => { - const tokens = tokenizedDocuments[index] ?? []; - return { - document, - length: tokens.length, - termFrequencies: countTerms(tokens), - }; - }); - const totalLength = indexedDocuments.reduce( - (total, document) => total + document.length, - 0 - ); - const averageDocumentLength = - indexedDocuments.length === 0 ? 0 : totalLength / indexedDocuments.length; - const documentFrequencies = countDocumentFrequencies(tokenizedDocuments); - const rawIdfs = new Map( - [...documentFrequencies].map(([term, frequency]) => [ - term, - Math.log(documents.length - frequency + 0.5) - Math.log(frequency + 0.5), - ]) - ); - const idfSum = [...rawIdfs.values()].reduce((sum, idf) => sum + idf, 0); - const averageIdf = rawIdfs.size === 0 ? 0 : idfSum / rawIdfs.size; - const epsilonIdf = BM25_EPSILON * averageIdf; - const inverseDocumentFrequencies = new Map( - [...rawIdfs].map(([term, idf]) => [term, idf < 0 ? epsilonIdf : idf]) - ); - return { - documents: indexedDocuments, - averageDocumentLength, - inverseDocumentFrequencies, - }; -} - -export function searchBm25({ - index, - query, - topK = DEFAULT_TOP_K, -}: { - readonly index: Bm25Index; - readonly query: string; - readonly topK?: number; -}): RetrievalResult[] { - const queryTokens = tokenize(query); - if (queryTokens.length === 0) { - return []; - } - return index.documents - .map((document, position) => ({ - document: document.document, - score: scoreBm25Document({ document, index, queryTokens }), - position, - })) - .toSorted( - (left, right) => - right.score - left.score || left.position - right.position - ) - .slice(0, Math.min(topK, index.documents.length)) - .map(({ document, score }) => ({ document, score })); -} - -function scoreBm25Document({ - document, - index, - queryTokens, -}: { - readonly document: IndexedDocument; - readonly index: Bm25Index; - readonly queryTokens: readonly string[]; -}): number { - if (index.averageDocumentLength === 0) { - return 0; - } - let score = 0; - for (const token of queryTokens) { - const frequency = document.termFrequencies.get(token) ?? 0; - const idf = index.inverseDocumentFrequencies.get(token) ?? 0; - const numerator = frequency * (BM25_K1 + 1); - const lengthRatio = document.length / index.averageDocumentLength; - const denominator = - frequency + BM25_K1 * (1 - BM25_B + BM25_B * lengthRatio); - score += idf * (numerator / denominator); - } - return score; -} - -export function searchGrep({ - documents, - pattern, - topK = DEFAULT_TOP_K, -}: { - readonly documents: readonly RetrievalDocument[]; - readonly pattern: string; - readonly topK?: number; -}): RetrievalResult[] { - if (pattern.trim().length === 0) { - return []; - } - const regex = compileGrepPattern(pattern); - return documents - .map((document, position) => ({ - document, - score: [...document.content.matchAll(regex)].length, - position, - })) - .filter((result) => result.score > 0) - .toSorted( - (left, right) => - right.score - left.score || left.position - right.position - ) - .slice(0, topK) - .map(({ document, score }) => ({ document, score })); -} - -function compileGrepPattern(pattern: string): RegExp { - try { - return new RegExp(pattern, "giu"); - } catch { - return new RegExp(escapeRegex(pattern), "giu"); - } -} - -function escapeRegex(value: string): string { - return value.replaceAll(/[.*+?^${}()|[\]\\]/gu, "\\$&"); -} - -export function formatKbSearchResults({ - results, - retrievalMs, -}: { - readonly results: readonly RetrievalResult[]; - readonly retrievalMs: number; -}): string { - const timing = `[Timing: retrieval=${retrievalMs.toFixed(0)}ms, total=${retrievalMs.toFixed(0)}ms]`; - if (results.length === 0) { - return `No relevant documents found.\n\n${timing}`; - } - return `${formatResults(results)}\n\n${timing}`; -} - -export function formatGrepResults({ - results, - pattern, -}: { - readonly results: readonly RetrievalResult[]; - readonly pattern: string; -}): string { - if (results.length === 0) { - return `No matches found for pattern: ${pattern}`; - } - return formatResults(results); -} - -export function makeBm25GrepRetrievalTools( - documents: readonly RetrievalDocument[] -): BankingRetrievalTools { - const index = makeBm25Index(documents); - return { - definitions: BM25_GREP_TOOL_DEFINITIONS, - invoke: (name, arguments_) => { - if (name === "KB_search") { - return invokeKbSearch({ arguments_, index }); - } - if (name === "grep") { - return invokeGrep({ arguments_, documents }); - } - return undefined; - }, - }; -} - -export function makeBankingRetrievalTools({ - documents, - retrievalConfig, -}: { - readonly documents: readonly RetrievalDocument[]; - readonly retrievalConfig: BankingRetrievalConfig; -}): BankingRetrievalTools { - switch (retrievalConfig) { - case "required_docs": { - return { definitions: [], invoke: () => undefined }; - } - case "bm25_grep": { - return makeBm25GrepRetrievalTools(documents); - } - default: { - retrievalConfig satisfies never; - return { definitions: [], invoke: () => undefined }; - } - } -} - -function invokeKbSearch({ - arguments_, - index, -}: { - readonly arguments_: Readonly>; - readonly index: Bm25Index; -}): string { - const query = arguments_["query"]; - if (typeof query !== "string") { - return "Error: Missing required parameter: query"; - } - const start = performance.now(); - const results = searchBm25({ index, query }); - return formatKbSearchResults({ - results, - retrievalMs: performance.now() - start, - }); -} - -function invokeGrep({ - arguments_, - documents, -}: { - readonly arguments_: Readonly>; - readonly documents: readonly RetrievalDocument[]; -}): string { - const pattern = arguments_["pattern"]; - if (typeof pattern !== "string") { - return "Error: Missing required parameter: pattern"; - } - return formatGrepResults({ - results: searchGrep({ documents, pattern }), - pattern, - }); -} - -function formatResults(results: readonly RetrievalResult[]): string { - return results - .map( - ({ document, score }, index) => - `${index + 1}. ${document.title}\n` + - ` ID: ${document.id}\n` + - ` Score: ${score.toFixed(4)}\n` + - ` Content: ${document.content}\n` - ) - .join("\n"); -} - -function tokenize(text: string): string[] { - const normalized = text.toLowerCase().trim(); - return normalized.length === 0 ? [] : normalized.split(/\s+/u); -} - -function countTerms(tokens: readonly string[]): ReadonlyMap { - const frequencies = new Map(); - for (const token of tokens) { - frequencies.set(token, (frequencies.get(token) ?? 0) + 1); - } - return frequencies; -} - -function countDocumentFrequencies( - documents: readonly (readonly string[])[] -): ReadonlyMap { - const frequencies = new Map(); - for (const tokens of documents) { - for (const token of new Set(tokens)) { - frequencies.set(token, (frequencies.get(token) ?? 0) + 1); - } - } - return frequencies; -} diff --git a/src/benchmarks/tau3-bench-banking/scorer.test.ts b/src/benchmarks/tau3-bench-banking/scorer.test.ts deleted file mode 100644 index 2df2d10b..00000000 --- a/src/benchmarks/tau3-bench-banking/scorer.test.ts +++ /dev/null @@ -1,276 +0,0 @@ -import { beforeEach, describe, expect, it } from "bun:test"; - -import { runSync } from "effect/Effect"; - -import { MessageRole, ScoreValue } from "../../harness/core"; -import { makeEmptyBankingData, seedBankingCache } from "./environment"; -import { bankingScorer } from "./scorer"; -import { TerminationReason } from "./solver"; -import type { Tau3Task } from "./types"; -beforeEach(() => { - const emptyDb = makeEmptyBankingData(); - seedBankingCache(emptyDb, []); -}); - -function makeTask(overrides?: Partial): Tau3Task { - return { - id: "task_test_001", - user_scenario: { instructions: "test" }, - evaluation_criteria: { - actions: [], - communicate_info: [], - reward_basis: ["DB"], - }, - ...overrides, - }; -} -describe("bankingScorer", () => { - it("returns Correct when reward >= 1", () => { - const agentData = makeEmptyBankingData(); - const task = makeTask(); - const score = runSync( - bankingScorer( - { - sample: { - id: "samp_001", - input: "test", - target: { text: "" }, - metadata: { - task, - agentData, - terminationReason: TerminationReason.UserStop, - }, - }, - messages: [ - { - role: MessageRole.System, - content: "system prompt", - }, - { - role: MessageRole.Assistant, - content: "Hello", - }, - ], - completed: true, - }, - { text: "" } - ) - ); - expect(score.value).toBe(ScoreValue.Correct); - }); - it("returns Incorrect when reward < 1", () => { - const agentData = makeEmptyBankingData(); - agentData.users.data["user_1"] = { name: "Test" }; - const task = makeTask({ - evaluation_criteria: { - actions: [], - communicate_info: [], - reward_basis: ["DB"], - }, - }); - const score = runSync( - bankingScorer( - { - sample: { - id: "samp_001", - input: "test", - target: { text: "" }, - metadata: { - task, - agentData, - terminationReason: TerminationReason.UserStop, - }, - }, - messages: [ - { - role: MessageRole.System, - content: "system prompt", - }, - ], - completed: true, - }, - { text: "" } - ) - ); - expect(score.value).toBe(ScoreValue.Incorrect); - }); - it("returns Incorrect when agentData is missing", () => { - const task = makeTask(); - const score = runSync( - bankingScorer( - { - sample: { - id: "samp_001", - input: "test", - target: { text: "" }, - metadata: { - task, - terminationReason: TerminationReason.UserStop, - }, - }, - messages: [], - completed: true, - }, - { text: "" } - ) - ); - expect(score.value).toBe(ScoreValue.Incorrect); - expect(score.explanation).toContain("Missing agentData"); - }); - it("includes breakdown in explanation", () => { - const agentData = makeEmptyBankingData(); - const task = makeTask({ - evaluation_criteria: { - actions: [], - communicate_info: [], - reward_basis: ["DB", "COMMUNICATE"], - }, - }); - const score = runSync( - bankingScorer( - { - sample: { - id: "samp_001", - input: "test", - target: { text: "" }, - metadata: { - task, - agentData, - terminationReason: TerminationReason.UserStop, - }, - }, - messages: [ - { - role: MessageRole.Assistant, - content: "response", - }, - ], - completed: true, - }, - { text: "" } - ) - ); - expect(score.explanation).toContain("breakdown"); - }); - it("returns Incorrect on premature termination", () => { - const agentData = makeEmptyBankingData(); - const task = makeTask(); - const score = runSync( - bankingScorer( - { - sample: { - id: "samp_001", - input: "test", - target: { text: "" }, - metadata: { - task, - agentData, - terminationReason: TerminationReason.MaxSteps, - }, - }, - messages: [], - completed: true, - }, - { text: "" } - ) - ); - expect(score.value).toBe(ScoreValue.Incorrect); - expect(score.explanation).toContain("prematurely"); - }); - it("collects tool calls from messages", () => { - const agentData = makeEmptyBankingData(); - const task = makeTask({ - evaluation_criteria: { - actions: [ - { - action_id: "a1", - name: "test_tool", - arguments: {}, - requestor: "assistant", - }, - ], - communicate_info: [], - reward_basis: ["ACTION"], - }, - }); - const score = runSync( - bankingScorer( - { - sample: { - id: "samp_001", - input: "test", - target: { text: "" }, - metadata: { - task, - agentData, - terminationReason: TerminationReason.UserStop, - }, - }, - messages: [ - { - role: MessageRole.Assistant, - content: "response", - toolCalls: [ - { - id: "call_001", - type: "function" as const, - function: { - name: "test_tool", - arguments: "{}", - }, - }, - ], - }, - ], - completed: true, - }, - { text: "" } - ) - ); - expect(score.value).toBe(ScoreValue.Correct); - }); - it("includes user-simulator tool calls in ACTION scoring", () => { - const agentData = makeEmptyBankingData(); - const task = makeTask({ - evaluation_criteria: { - actions: [ - { - action_id: "a1", - name: "request_human_agent_transfer", - arguments: {}, - requestor: "user", - }, - ], - communicate_info: [], - reward_basis: ["ACTION"], - }, - }); - const score = runSync( - bankingScorer( - { - sample: { - id: "samp_001", - input: "test", - target: { text: "" }, - metadata: { - task, - agentData, - terminationReason: TerminationReason.UserStop, - predictedUserToolCalls: [ - { - name: "request_human_agent_transfer", - arguments: {}, - requestor: "user", - }, - ], - }, - }, - messages: [], - completed: true, - }, - { text: "" } - ) - ); - expect(score.value).toBe(ScoreValue.Correct); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/scorer.ts b/src/benchmarks/tau3-bench-banking/scorer.ts deleted file mode 100644 index 00feeec5..00000000 --- a/src/benchmarks/tau3-bench-banking/scorer.ts +++ /dev/null @@ -1,109 +0,0 @@ -import type { Effect } from "effect/Effect"; -import { succeed } from "effect/Effect"; - -import type { - ModelMessage, - Score, - Target, - TaskState, -} from "../../harness/core"; -import { MessageRole, ScoreValue } from "../../harness/core"; -import type { ScorerService } from "../../harness/scorer"; -import { isDefinedAndNotNull, isRecord } from "../../internal/guards"; -import type { PredictedToolCall } from "./evaluator"; -import { evaluateSimulation } from "./evaluator"; -import { parseToolArguments } from "./tools/helpers"; -import type { BankingData, Tau3Task } from "./types"; - -function isBankingData(val: unknown): val is BankingData { - if (typeof val !== "object" || val === null) { - return false; - } - return "users" in val && "accounts" in val; -} - -function isTau3Task(val: unknown): val is Tau3Task { - if (typeof val !== "object" || val === null) { - return false; - } - return "id" in val && "user_scenario" in val; -} - -function collectToolCalls( - messages: readonly ModelMessage[] -): PredictedToolCall[] { - const calls: PredictedToolCall[] = []; - for (const m of messages) { - for (const tc of m.toolCalls ?? []) { - const args = parseToolArguments(tc.function.arguments); - calls.push({ - name: tc.function.name, - arguments: args, - requestor: "assistant", - }); - } - } - return calls; -} - -function collectUserToolCalls(value: unknown): PredictedToolCall[] { - if (!Array.isArray(value)) { - return []; - } - return value.filter( - (call): call is PredictedToolCall => - isRecord(call) && - typeof call["name"] === "string" && - isRecord(call["arguments"]) && - call["requestor"] === "user" - ); -} - -export const bankingScorer: ScorerService = ( - state: TaskState, - _target: Target -): Effect => { - const meta: Readonly> = state.sample.metadata ?? {}; - const agentData: unknown = meta["agentData"]; - const task: unknown = meta["task"]; - const terminationReason: string = - typeof meta["terminationReason"] === "string" - ? meta["terminationReason"] - : "MAX_STEPS"; - if (!isBankingData(agentData)) { - return succeed({ - value: ScoreValue.Incorrect, - answer: null, - explanation: "Missing agentData in TaskState metadata", - }); - } - if (isDefinedAndNotNull(task) && !isTau3Task(task)) { - return succeed({ - value: ScoreValue.Incorrect, - answer: null, - explanation: "Malformed task in TaskState metadata", - }); - } - const messages = state.messages; - const assistantTexts = messages - .filter((m) => m.role === MessageRole.Assistant) - .map((m) => m.content); - const toolCalls = [ - ...collectToolCalls(messages), - ...collectUserToolCalls(meta["predictedUserToolCalls"]), - ]; - const result = evaluateSimulation({ - task: isTau3Task(task) ? task : undefined, - agentData, - assistantTexts, - toolCalls, - terminationReason, - }); - return succeed({ - value: result.reward >= 1 ? ScoreValue.Correct : ScoreValue.Incorrect, - answer: String(result.reward), - explanation: - result.note ?? - `db_match=${result.dbMatch}, action_met=${result.actionMet}, communicate_met=${result.communicateMet}, termination=${result.terminationReason}, breakdown=${JSON.stringify(result.breakdown)}`, - }); -}; diff --git a/src/benchmarks/tau3-bench-banking/solver.test.ts b/src/benchmarks/tau3-bench-banking/solver.test.ts deleted file mode 100644 index 0354be27..00000000 --- a/src/benchmarks/tau3-bench-banking/solver.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { describe, expect, it } from "bun:test"; - -import { selectUserToolDefinitions, TerminationReason } from "./solver"; -describe("bankingSolver", () => { - it("exports TerminationReason with required values", () => { - expect(TerminationReason.UserStop).toBe("USER_STOP"); - expect(TerminationReason.MaxSteps).toBe("MAX_STEPS"); - }); - it("grants only the user tools declared by the task", () => { - const definitions = selectUserToolDefinitions([ - "apply_for_credit_card", - "call_discoverable_user_tool", - ]); - expect(definitions.map((definition) => definition.function.name)).toEqual([ - "apply_for_credit_card", - "call_discoverable_user_tool", - ]); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/solver.ts b/src/benchmarks/tau3-bench-banking/solver.ts deleted file mode 100644 index 0cbd7aba..00000000 --- a/src/benchmarks/tau3-bench-banking/solver.ts +++ /dev/null @@ -1,370 +0,0 @@ -import { HttpClient } from "@effect/platform"; -import type { Semaphore } from "effect/Effect"; -import { gen, mapError, provideService } from "effect/Effect"; - -import type { - ModelMessage, - ModelUsage, - TaskState, - ToolDefinition, -} from "../../harness/core"; -import { MessageRole, SolverError } from "../../harness/core"; -import type { GenerateConfig, ModelService } from "../../harness/model"; -import type { SolverService } from "../../harness/solver"; -import { Either } from "../../internal/either"; -import { definedValues } from "../../internal/guards"; -import { parseSchema } from "../../internal/zod"; -import type { ResponsesModelService } from "../../providers/responses-model"; -import { - buildBankingAgentSystemPrompt, - DEFAULT_FIRST_AGENT_MESSAGE, -} from "./agent-prompts"; -import { - ensureAllBankingDocuments, - ensureBankingDocuments, - getAllBankingDocuments, -} from "./documents"; -import { - applyInitialState, - ensureBankingData, - ensureBankingTasks, - loadBankingData, -} from "./environment"; -import type { PredictedToolCall } from "./evaluator"; -import { makeBankingRetrievalTools } from "./retrieval"; -import { DEFAULT_BANKING_RETRIEVAL_CONFIG } from "./retrieval-config"; -import { BANKING_TOOL_DEFINITIONS } from "./tools/definitions"; -import { - invokeBankingAgentTool, - registerInitialDiscoverableTools, -} from "./tools/handlers-meta"; -import { - invokeBankingUserTool, - USER_PERMANENT_TOOL_DEFINITIONS, -} from "./tools/handlers-user"; -import { parseToolArguments } from "./tools/helpers"; -import { makeBankingEnvState } from "./tools/registry"; -import type { SolverOpts } from "./types"; -import { deriveReadLogAllowlist, Tau3TaskSchema } from "./types"; -import { UserSimulator } from "./user-simulator"; - -const STOP_TOKENS = ["###STOP###", "###TRANSFER###", "###OUT-OF-SCOPE###"]; - -const MAX_STEPS = 200; - -const MAX_ERRORS = 10; - -const BANKING_TEMPERATURE = 0; - -export const TerminationReason = { - UserStop: "USER_STOP", - AgentStop: "AGENT_STOP", - MaxSteps: "MAX_STEPS", - TooManyErrors: "TOO_MANY_ERRORS", -} as const; - -const Role = { - User: "user", - Agent: "agent", - Env: "env", -} as const; - -type Role = (typeof Role)[keyof typeof Role]; - -function lastAssistantText(messages: readonly ModelMessage[]): string { - for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i]!.role === MessageRole.Assistant) { - return messages[i]!.content; - } - } - return ""; -} - -export function selectUserToolDefinitions( - toolNames: readonly string[] -): readonly ToolDefinition[] { - const grantedToolNames = new Set(toolNames); - return USER_PERMANENT_TOOL_DEFINITIONS.filter((definition) => - grantedToolNames.has(definition.function.name) - ); -} - -type UserTurnState = "initial" | "continuing_tools" | "awaiting_reply"; - -function selectUserTurnEffect(opts: { - readonly userSim: UserSimulator; - readonly turnState: UserTurnState; - readonly messages: readonly ModelMessage[]; -}): ReturnType { - switch (opts.turnState) { - case "initial": { - return opts.userSim.generateInitial(); - } - case "continuing_tools": { - return opts.userSim.continueAfterTools(); - } - case "awaiting_reply": { - return opts.userSim.step(lastAssistantText(opts.messages)); - } - default: { - return opts.turnState satisfies never; - } - } -} - -function isToolErrorResult(result: string): boolean { - return result.startsWith("Error:"); -} - -export function bankingSolver({ - model, - userModel, - client, - dataFetchLock, - opts, -}: { - readonly model: ModelService; - readonly userModel: ResponsesModelService; - readonly client: HttpClient.HttpClient; - readonly dataFetchLock: Semaphore; - readonly opts: SolverOpts; -}): SolverService { - return (state: TaskState) => - gen(function* () { - const userModelConfig = opts?.userModelConfig; - if (!userModelConfig) { - return yield* new SolverError({ - message: - "bankingSolver requires userModelConfig (apiKey + model for user simulator)", - }); - } - yield* ensureBankingData(dataFetchLock).pipe( - provideService(HttpClient.HttpClient, client), - mapError( - (e) => - new SolverError({ - message: `Failed to load banking data from GitHub: ${String(e)}`, - }) - ) - ); - yield* ensureBankingTasks(dataFetchLock).pipe( - provideService(HttpClient.HttpClient, client), - mapError( - (e) => - new SolverError({ - message: `Failed to load banking tasks from GitHub: ${String(e)}`, - }) - ) - ); - const taskMeta = state.sample.metadata?.["task"]; - const parseTaskResult = parseSchema(Tau3TaskSchema, taskMeta); - if (Either.isLeft(parseTaskResult)) { - return yield* new SolverError({ - message: `Invalid task in metadata: ${String(parseTaskResult.left)}`, - }); - } - const task = parseTaskResult.right; - const retrievalConfig = - opts?.retrievalConfig ?? DEFAULT_BANKING_RETRIEVAL_CONFIG; - const requiredDocIds = task.required_documents ?? []; - if (retrievalConfig === "required_docs") { - yield* ensureBankingDocuments(requiredDocIds, dataFetchLock).pipe( - provideService(HttpClient.HttpClient, client), - mapError( - (e) => - new SolverError({ - message: `Failed to load documents: ${String(e)}`, - }) - ) - ); - } else { - yield* ensureAllBankingDocuments(dataFetchLock).pipe( - provideService(HttpClient.HttpClient, client), - mapError( - (e) => - new SolverError({ - message: `Failed to load document corpus: ${String(e)}`, - }) - ) - ); - } - const retrievalTools = makeBankingRetrievalTools({ - documents: getAllBankingDocuments(), - retrievalConfig, - }); - const agentData = loadBankingData(); - applyInitialState(agentData, task); - const envState = makeBankingEnvState( - agentData, - deriveReadLogAllowlist(task) - ); - registerInitialDiscoverableTools(); - const userToolDefs = selectUserToolDefinitions(task.user_tools ?? []); - const userSim = new UserSimulator(userModel, userModelConfig); - userSim.setAvailableTools(userToolDefs); - userSim.reset(state.sample.input, DEFAULT_FIRST_AGENT_MESSAGE); - const systemPrompt = buildBankingAgentSystemPrompt({ - requiredDocIds, - retrievalConfig, - }); - const messages: ModelMessage[] = [ - { role: MessageRole.System, content: systemPrompt }, - { role: MessageRole.Assistant, content: DEFAULT_FIRST_AGENT_MESSAGE }, - ]; - const respondMessages: string[] = []; - const predictedUserToolCalls: PredictedToolCall[] = []; - const genConfig: GenerateConfig = { - temperature: BANKING_TEMPERATURE, - tools: [...BANKING_TOOL_DEFINITIONS, ...retrievalTools.definitions], - ...definedValues(opts.inference), - ...definedValues({ - endpointId: opts.endpointId, - }), - }; - let totalGenerationTimeMs = 0; - const accUsage = { - inputTokens: 0, - outputTokens: 0, - totalTokens: 0, - reasoningTokens: 0, - totalCost: 0, - }; - let role: Role = Role.User; - let stepCount = 0; - let numToolErrors = 0; - let userTurnState: UserTurnState = "initial"; - let terminationReason: string = TerminationReason.MaxSteps; - let done = false; - while (!done) { - if (stepCount >= MAX_STEPS) { - terminationReason = TerminationReason.MaxSteps; - break; - } - if (numToolErrors >= MAX_ERRORS) { - terminationReason = TerminationReason.TooManyErrors; - break; - } - switch (role) { - case Role.User: { - const userTurnEffect = selectUserTurnEffect({ - userSim, - turnState: userTurnState, - messages, - }); - const userTurn = yield* userTurnEffect.pipe( - provideService(HttpClient.HttpClient, client), - mapError( - (e) => - new SolverError({ - message: `User simulator step failed: ${String(e)}`, - }) - ) - ); - if (userTurn.kind === "toolCalls") { - for (const tc of userTurn.calls) { - const args = parseToolArguments(tc.arguments); - predictedUserToolCalls.push({ - name: tc.name, - arguments: args, - requestor: "user", - }); - const toolResultStr = invokeBankingUserTool( - envState, - tc.name, - args - ); - if (isToolErrorResult(toolResultStr)) { - numToolErrors++; - } - userSim.addToolResult(tc.id, toolResultStr); - } - userTurnState = "continuing_tools"; - } else { - userTurnState = "awaiting_reply"; - const userResponse = userTurn.content; - messages.push({ role: MessageRole.User, content: userResponse }); - if (STOP_TOKENS.some((t) => userResponse.includes(t))) { - terminationReason = TerminationReason.UserStop; - done = true; - } else { - role = Role.Agent; - } - } - break; - } - case Role.Agent: { - const output = yield* model.generate(messages, genConfig); - totalGenerationTimeMs += output.generationTimeMs ?? 0; - if (output.usage) { - accUsage.inputTokens += output.usage.inputTokens ?? 0; - accUsage.outputTokens += output.usage.outputTokens ?? 0; - accUsage.totalTokens += output.usage.totalTokens ?? 0; - accUsage.reasoningTokens += output.usage.reasoningTokens ?? 0; - accUsage.totalCost += output.usage.totalCost ?? 0; - } - const assistantMsg = output.message; - messages.push(assistantMsg); - const toolCalls = assistantMsg.toolCalls ?? []; - if (toolCalls.length > 0) { - role = Role.Env; - } else { - respondMessages.push(assistantMsg.content); - role = Role.User; - } - break; - } - case Role.Env: { - const assistantMsg = messages.at(-1); - const toolCalls = assistantMsg?.toolCalls ?? []; - for (const tc of toolCalls) { - const toolName = tc.function.name; - const toolArgs = parseToolArguments(tc.function.arguments); - const toolResultStr = - retrievalTools.invoke(toolName, toolArgs) ?? - invokeBankingAgentTool(envState, toolName, toolArgs); - if (isToolErrorResult(toolResultStr)) { - numToolErrors++; - } - messages.push({ - role: MessageRole.Tool, - content: toolResultStr, - toolCallId: tc.id, - }); - } - role = Role.Agent; - break; - } - } - stepCount++; - } - const finalUsage: ModelUsage = { - inputTokens: accUsage.inputTokens, - outputTokens: accUsage.outputTokens, - totalTokens: accUsage.totalTokens, - reasoningTokens: accUsage.reasoningTokens, - totalCost: accUsage.totalCost, - }; - const completion = respondMessages.join("\n"); - return { - sample: { - ...state.sample, - metadata: { - ...state.sample.metadata, - task, - agentData, - terminationReason, - stepCount, - predictedUserToolCalls, - }, - }, - messages, - output: { - completion, - message: { role: MessageRole.Assistant, content: completion }, - usage: finalUsage, - generationTimeMs: totalGenerationTimeMs, - }, - completed: true, - }; - }); -} diff --git a/src/benchmarks/tau3-bench-banking/tools/db-query.test.ts b/src/benchmarks/tau3-bench-banking/tools/db-query.test.ts deleted file mode 100644 index f03bc4cc..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/db-query.test.ts +++ /dev/null @@ -1,453 +0,0 @@ -import { describe, it, expect } from "bun:test"; -import assert from "node:assert/strict"; - -import { isRecord } from "../../../internal/guards"; -import { makeEmptyBankingData } from "../environment"; -import { - queryDb, - addToDb, - updateRecordInDb, - removeFromDb, - listDatabases, - queryDatabaseTool, -} from "./db-query"; - -function createTestDb() { - const db = makeEmptyBankingData(); - db.users.data = { - usr1: { - user_id: "usr1", - name: "Alice Johnson", - email: "alice@example.com", - status: "active", - balance: 5000, - }, - usr2: { - user_id: "usr2", - name: "Bob Smith", - email: "bob@example.com", - status: "inactive", - balance: 2500, - }, - }; - db.accounts.data = { - acc1: { - account_id: "acc1", - user_id: "usr1", - type: "checking", - balance: 10000, - }, - acc2: { - account_id: "acc2", - user_id: "usr2", - type: "savings", - balance: 50000, - }, - }; - return db; -} -describe("db-query", () => { - describe("queryDb", () => { - it("returns empty array for non-existent database", () => { - const db = createTestDb(); - const results = queryDb({ db, dbName: "nonexistent" }); - expect(results).toEqual([]); - }); - it("queries with exact match (default eq operator)", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { status: "active" }, - }); - expect(results).toHaveLength(1); - expect(results[0]?.user_id).toBe("usr1"); - }); - it("queries with explicit __eq operator", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { status__eq: "active" }, - }); - expect(results).toHaveLength(1); - }); - it("queries with __ne operator", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { status__ne: "active" }, - }); - expect(results).toHaveLength(1); - expect(results[0]?.user_id).toBe("usr2"); - }); - it("queries with __gt operator on numbers", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { balance__gt: 3000 }, - }); - expect(results).toHaveLength(1); - expect(results[0]?.user_id).toBe("usr1"); - }); - it("returns non-match for type-mismatched __gt comparison", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { balance__gt: "abc" satisfies unknown }, - }); - expect(results).toHaveLength(0); - }); - it("queries with __gte operator", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { balance__gte: 5000 }, - }); - expect(results).toHaveLength(1); - }); - it("queries with __lt operator", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { balance__lt: 4000 }, - }); - expect(results).toHaveLength(1); - expect(results[0]?.user_id).toBe("usr2"); - }); - it("queries with __lte operator", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { balance__lte: 2500 }, - }); - expect(results).toHaveLength(1); - }); - it("queries with __contains on string", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { name__contains: "son" }, - }); - expect(results).toHaveLength(1); - expect(results[0]?.name).toBe("Alice Johnson"); - }); - it("queries with __startswith", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { email__startswith: "alice" }, - }); - expect(results).toHaveLength(1); - }); - it("queries with __endswith", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { email__endswith: "@example.com" }, - }); - expect(results).toHaveLength(2); - }); - it("queries with __in operator", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { user_id__in: ["usr1", "usr3"] }, - }); - expect(results).toHaveLength(1); - expect(results[0]?.user_id).toBe("usr1"); - }); - it("queries with __nin operator", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { user_id__nin: ["usr1"] }, - }); - expect(results).toHaveLength(1); - }); - it("queries with returnIds option", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { status: "active" }, - options: { returnIds: true }, - }); - expect(results).toHaveLength(1); - const first = results[0]; - assert(first); - const [id, record] = first; - expect(id).toBe("usr1"); - expect(record.user_id).toBe("usr1"); - }); - it("queries with limit option", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: {}, - options: { limit: 1 }, - }); - expect(results).toHaveLength(1); - }); - it("applies multiple constraints with AND logic", () => { - const db = createTestDb(); - const results = queryDb({ - db, - dbName: "users", - constraints: { status: "active", balance__gte: 4000 }, - }); - expect(results).toHaveLength(1); - expect(results[0]?.user_id).toBe("usr1"); - }); - }); - describe("addToDb", () => { - it("adds a record to database", () => { - const db = createTestDb(); - const success = addToDb({ - db, - dbName: "users", - recordId: "usr3", - record: { - user_id: "usr3", - name: "Charlie", - email: "charlie@example.com", - status: "active", - }, - }); - expect(success).toBe(true); - const results = queryDb({ - db, - dbName: "users", - constraints: { user_id: "usr3" }, - }); - expect(results).toHaveLength(1); - }); - it("returns false if table does not exist", () => { - const db = createTestDb(); - const success = addToDb({ - db, - dbName: "nonexistent", - recordId: "id1", - record: { field: "value" }, - }); - expect(success).toBe(false); - }); - it("returns false if record already exists", () => { - const db = createTestDb(); - const success = addToDb({ - db, - dbName: "users", - recordId: "usr1", - record: { user_id: "usr1" }, - }); - expect(success).toBe(false); - }); - it("stores __proto__ as an own property without polluting the prototype", () => { - const db = createTestDb(); - const protoKey = ["__", "proto__"].join(""); - const success = addToDb({ - db, - dbName: "users", - recordId: protoKey, - record: { - user_id: protoKey, - name: "Proto User", - }, - }); - expect(success).toBe(true); - expect(Object.hasOwn(db.users.data, protoKey)).toBe(true); - expect(Reflect.get(Object.prototype, "user_id")).toBeUndefined(); - const results = queryDb({ - db, - dbName: "users", - constraints: { user_id: protoKey }, - }); - expect(results).toHaveLength(1); - expect(results[0]?.name).toBe("Proto User"); - }); - }); - describe("updateRecordInDb", () => { - it("updates record fields", () => { - const db = createTestDb(); - const [success, updated] = updateRecordInDb({ - db, - dbName: "users", - recordId: "usr1", - updates: { status: "suspended" }, - }); - expect(success).toBe(true); - expect(updated).not.toBeNull(); - expect(updated?.status).toBe("suspended"); - }); - it("returns false if record not found", () => { - const db = createTestDb(); - const [success] = updateRecordInDb({ - db, - dbName: "users", - recordId: "usr999", - updates: { status: "deleted" }, - }); - expect(success).toBe(false); - }); - it("returns false if table does not exist", () => { - const db = createTestDb(); - const [success] = updateRecordInDb({ - db, - dbName: "nonexistent", - recordId: "id1", - updates: { field: "value" }, - }); - expect(success).toBe(false); - }); - it("persists changes to database", () => { - const db = createTestDb(); - updateRecordInDb({ - db, - dbName: "users", - recordId: "usr1", - updates: { status: "suspended" }, - }); - const results = queryDb({ - db, - dbName: "users", - constraints: { user_id: "usr1" }, - }); - expect(results[0]?.status).toBe("suspended"); - }); - it("updates __proto__ as an own property without polluting Object.prototype", () => { - const db = createTestDb(); - const protoKey = ["__", "proto__"].join(""); - const parsed: unknown = JSON.parse('{"__proto__":{"polluted":"yes"}}'); - assert(isRecord(parsed)); - const [success, updated] = updateRecordInDb({ - db, - dbName: "users", - recordId: "usr1", - updates: parsed, - }); - expect(success).toBe(true); - expect(updated).not.toBeNull(); - expect(Object.hasOwn(updated ?? {}, protoKey)).toBe(true); - expect(Reflect.get(Object.prototype, "polluted")).toBeUndefined(); - expect(Reflect.get(updated ?? {}, protoKey)).toEqual({ polluted: "yes" }); - }); - }); - describe("removeFromDb", () => { - it("removes records matching constraints", () => { - const db = createTestDb(); - const removed = removeFromDb({ - db, - dbName: "users", - constraints: { status: "inactive" }, - }); - expect(removed).toHaveLength(1); - const remaining = queryDb({ db, dbName: "users", constraints: {} }); - expect(remaining).toHaveLength(1); - }); - it("returns empty array if no matches", () => { - const db = createTestDb(); - const removed = removeFromDb({ - db, - dbName: "users", - constraints: { status: "deleted" }, - }); - expect(removed).toHaveLength(0); - }); - it("returns empty array if table does not exist", () => { - const db = createTestDb(); - const removed = removeFromDb({ - db, - dbName: "nonexistent", - }); - expect(removed).toHaveLength(0); - }); - }); - describe("listDatabases", () => { - it("lists available databases in correct order", () => { - const db = createTestDb(); - const names = listDatabases(db); - expect(names).toContain("users"); - expect(names).toContain("accounts"); - expect(names).toContain("verification_history"); - expect(names.length).toBeGreaterThan(10); - }); - it("returns databases in Python _load_databases order", () => { - const db = createTestDb(); - const names = listDatabases(db); - const userIdx = names.indexOf("users"); - const accountsIdx = names.indexOf("accounts"); - const verifyIdx = names.indexOf("verification_history"); - expect(userIdx).toBeLessThan(accountsIdx); - expect(accountsIdx).toBeLessThan(verifyIdx); - }); - }); - describe("queryDatabaseTool", () => { - it("returns formatted query result header and records", () => { - const db = createTestDb(); - const result = queryDatabaseTool( - db, - "users", - JSON.stringify({ status: "active" }) - ); - expect(result).toContain("Found 1 record(s) in 'users':"); - expect(result).toContain("Record ID: usr1"); - expect(result).toContain("user_id: usr1"); - }); - it('returns "No records found" message', () => { - const db = createTestDb(); - const result = queryDatabaseTool( - db, - "users", - JSON.stringify({ status: "deleted" }) - ); - expect(result).toBe("No records found in 'users'."); - }); - it("returns error for unknown database", () => { - const db = createTestDb(); - const result = queryDatabaseTool(db, "nonexistent", "{}"); - expect(result).toContain("Error: Database 'nonexistent' not found"); - expect(result).toContain("Available:"); - }); - it("returns error for invalid JSON constraints", () => { - const db = createTestDb(); - const result = queryDatabaseTool(db, "users", "invalid json"); - expect(result).toContain("Error: Invalid JSON"); - }); - it("lists available databases in error message", () => { - const db = createTestDb(); - const result = queryDatabaseTool(db, "unknown", "{}"); - expect(result).toContain("users"); - expect(result).toContain("accounts"); - }); - }); - describe("State isolation", () => { - it("modifications to one db do not affect another", () => { - const db1 = createTestDb(); - const db2 = createTestDb(); - addToDb({ - db: db1, - dbName: "users", - recordId: "usr3", - record: { user_id: "usr3" }, - }); - const results = queryDb({ - db: db2, - dbName: "users", - constraints: { user_id: "usr3" }, - }); - expect(results).toHaveLength(0); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/tools/db-query.ts b/src/benchmarks/tau3-bench-banking/tools/db-query.ts deleted file mode 100644 index b07b5c18..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/db-query.ts +++ /dev/null @@ -1,389 +0,0 @@ -import { Either } from "../../../internal/either"; -import { unknownErrorToString } from "../../../internal/errors"; -import type { BankingData } from "../types"; -import { getBankingTable } from "../types"; - -type BankingDb = BankingData; - -type ComparisonOperator = - | "eq" - | "ne" - | "gt" - | "gte" - | "lt" - | "lte" - | "contains" - | "startswith" - | "endswith" - | "in" - | "nin"; - -interface QueryOptions { - returnIds?: boolean; - limit?: number; -} - -function setOwnProperty( - target: Record, - key: string, - value: unknown -): void { - Object.defineProperty(target, key, { - value, - writable: true, - enumerable: true, - configurable: true, - }); -} - -function isComparisonOperator(op: string): op is ComparisonOperator { - return [ - "eq", - "ne", - "gt", - "gte", - "lt", - "lte", - "contains", - "startswith", - "endswith", - "in", - "nin", - ].includes(op); -} - -function parseConstraint( - key: string, - value: unknown -): [string, ComparisonOperator, unknown] { - if (key.includes("__")) { - const parts = key.split("__", 2); - const fieldName = parts[0]!; - const opName = parts[1]!; - if (!isComparisonOperator(opName)) { - return [key, "eq", value]; - } - return [fieldName, opName, value]; - } - return [key, "eq", value]; -} - -function getComparisonOp( - opName: ComparisonOperator -): (a: unknown, b: unknown) => boolean { - switch (opName) { - case "eq": { - return (a, b) => a === b; - } - case "ne": { - return (a, b) => a !== b; - } - case "gt": { - return (a, b) => { - if ( - (typeof a === "number" && typeof b === "number") || - (typeof a === "string" && typeof b === "string") - ) { - return a > b; - } - return false; - }; - } - case "gte": { - return (a, b) => { - if ( - (typeof a === "number" && typeof b === "number") || - (typeof a === "string" && typeof b === "string") - ) { - return a >= b; - } - return false; - }; - } - case "lt": { - return (a, b) => { - if ( - (typeof a === "number" && typeof b === "number") || - (typeof a === "string" && typeof b === "string") - ) { - return a < b; - } - return false; - }; - } - case "lte": { - return (a, b) => { - if ( - (typeof a === "number" && typeof b === "number") || - (typeof a === "string" && typeof b === "string") - ) { - return a <= b; - } - return false; - }; - } - case "contains": { - return (a, b) => { - if (a === null || a === undefined) { - return false; - } - if (Array.isArray(a)) { - return a.includes(b); - } - if (typeof a === "string" && typeof b === "string") { - return a.includes(b); - } - return false; - }; - } - case "startswith": { - return (a, b) => { - if (a === null || a === undefined) { - return false; - } - return String(a).startsWith(String(b)); - }; - } - case "endswith": { - return (a, b) => { - if (a === null || a === undefined) { - return false; - } - return String(a).endsWith(String(b)); - }; - } - case "in": { - return (a, b) => { - if (Array.isArray(b)) { - return b.includes(a); - } - return false; - }; - } - case "nin": { - return (a, b) => { - if (Array.isArray(b)) { - return !b.includes(a); - } - return false; - }; - } - default: { - return (a, b) => a === b; - } - } -} - -function recordMatches( - record: Record, - constraints: Record -): boolean { - for (const [key, value] of Object.entries(constraints)) { - const [fieldName, opName, expected] = parseConstraint(key, value); - const actual = record[fieldName]; - const compare = getComparisonOp(opName); - try { - if (!compare(actual, expected)) { - return false; - } - } catch { - return false; - } - } - return true; -} - -export function queryDb(opts: { - db: BankingDb; - dbName: string; - constraints?: Record; - options: QueryOptions & { - returnIds: true; - }; -}): [string, Record][]; - -export function queryDb(opts: { - db: BankingDb; - dbName: string; - constraints?: Record; - options?: QueryOptions; -}): Record[]; - -export function queryDb(opts: { - db: BankingDb; - dbName: string; - constraints?: Record; - options?: QueryOptions; -}): Record[] | [string, Record][] { - const { db, dbName, constraints, options } = opts; - const finalConstraints = constraints ?? {}; - const { returnIds = false, limit } = options ?? {}; - const table = getBankingTable(db, dbName); - if (!table) { - return []; - } - const resultsWithIds: [string, Record][] = []; - const resultsNoIds: Record[] = []; - for (const [recordId, record] of Object.entries(table.data)) { - if (!record || typeof record !== "object") { - continue; - } - const recordObj = record satisfies Record; - if (recordMatches(recordObj, finalConstraints)) { - if (returnIds) { - resultsWithIds.push([recordId, recordObj]); - } else { - resultsNoIds.push(recordObj); - } - const currentLength = returnIds - ? resultsWithIds.length - : resultsNoIds.length; - if (limit !== undefined && currentLength >= limit) { - break; - } - } - } - return returnIds ? resultsWithIds : resultsNoIds; -} - -export function addToDb(opts: { - db: BankingDb; - dbName: string; - recordId: string; - record: Record; -}): boolean { - const { db, dbName, recordId, record } = opts; - const table = getBankingTable(db, dbName); - if (!table) { - return false; - } - if (Object.hasOwn(table.data, recordId)) { - return false; - } - setOwnProperty(table.data, recordId, record); - return true; -} - -export function updateRecordInDb(opts: { - db: BankingDb; - dbName: string; - recordId: string; - updates: Record; -}): [boolean, Record | null] { - const { db, dbName, recordId, updates } = opts; - const table = getBankingTable(db, dbName); - if (!table) { - return [false, null]; - } - if (!Object.hasOwn(table.data, recordId)) { - return [false, null]; - } - const record = table.data[recordId]; - if (typeof record !== "object" || record === null) { - return [false, null]; - } - const updated = record satisfies Record; - for (const [field, value] of Object.entries(updates)) { - setOwnProperty(updated, field, value); - } - return [true, updated]; -} - -export function removeFromDb(opts: { - db: BankingDb; - dbName: string; - constraints?: Record; -}): Record[] { - const { db, dbName, constraints = {} } = opts; - const table = getBankingTable(db, dbName); - if (!table) { - return []; - } - const results: Record[] = []; - const toDelete: string[] = []; - for (const [recordId, record] of Object.entries(table.data)) { - if (!record || typeof record !== "object") { - continue; - } - const recordObj = record satisfies Record; - if (recordMatches(recordObj, constraints)) { - results.push(recordObj); - toDelete.push(recordId); - } - } - for (const recordId of toDelete) { - delete table.data[recordId]; - } - return results; -} - -export function listDatabases(db: BankingDb): string[] { - const names = [ - "users", - "accounts", - "referrals", - "credit_card_applications", - "user_discoverable_tools", - "user_discoverable_tool_calls", - "agent_discoverable_tools", - "task_config", - "verification_history", - "credit_card_transaction_history", - "cash_back_disputes", - "bank_account_transaction_history", - "credit_card_accounts", - "human_transfer_requests", - "transaction_disputes", - "credit_card_orders", - "credit_card_closure_reasons", - "credit_card_account_flags", - "debit_cards", - "debit_card_orders", - "debit_card_disputes", - "credit_limit_increase_requests", - "payment_history", - ]; - return names.filter((name) => name in db); -} - -export function queryDatabaseTool( - db: BankingDb, - databaseName: string, - constraints = "{}" -): string { - const available = listDatabases(db); - if (!available.includes(databaseName)) { - return `Error: Database '${databaseName}' not found. Available: ${available.join(", ")}`; - } - let constraintDict: Record = {}; - if (constraints) { - const parsed = Either.try((): Record => - JSON.parse(constraints) - ); - if (Either.isLeft(parsed)) { - return `Error: Invalid JSON: ${unknownErrorToString(parsed.left)}`; - } - constraintDict = parsed.right; - } - const results = queryDb({ - db, - dbName: databaseName, - constraints: constraintDict, - options: { returnIds: true }, - }); - if (results.length === 0) { - return `No records found in '${databaseName}'.`; - } - const lines: string[] = [ - `Found ${results.length} record(s) in '${databaseName}':\n`, - ]; - for (let i = 0; i < results.length; i++) { - const [recordId, record] = results[i] as [string, Record]; - lines.push(`${i + 1}. Record ID: ${recordId}`); - for (const [field, value] of Object.entries(record)) { - lines.push(` ${field}: ${value}`); - } - lines.push(""); - } - return lines.join("\n"); -} diff --git a/src/benchmarks/tau3-bench-banking/tools/definitions.ts b/src/benchmarks/tau3-bench-banking/tools/definitions.ts deleted file mode 100644 index 1ca089d7..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/definitions.ts +++ /dev/null @@ -1,336 +0,0 @@ -interface ToolParameter { - type: "string" | "number" | "integer" | "boolean" | "array" | "object"; - description: string; - enum?: (string | number)[]; - items?: unknown; -} - -export interface ToolDefinition { - type: "function"; - function: { - name: string; - description: string; - parameters: { - type: "object"; - properties: Record; - required?: string[]; - }; - }; -} - -export const BANKING_TOOL_DEFINITIONS: readonly ToolDefinition[] = [ - { - type: "function", - function: { - name: "transfer_to_human_agents", - description: "Transfer the user to a human agent.", - parameters: { - type: "object", - properties: { - summary: { - type: "string", - description: - "A summary of the user's issue and what was attempted before transfer.", - }, - reason: { - type: "string", - enum: [ - "fraud_or_security_concern", - "account_closure_request", - "deceased_account_holder", - "legal_or_regulatory_matter", - "account_ownership_dispute", - "complex_billing_dispute", - "abusive_customer_behavior", - "third_party_inquiry", - "technical_system_error", - "unconfirmed_external_communication", - "customer_demands_after_unavailable_offer_refusal", - "kb_search_unsuccessful_customer_requests_transfer", - "specialized_department_required", - "accessibility_or_special_needs", - "customer_frustrated_demands_human", - "supervisor_request_service_complaint", - "customer_requests_human_no_specific_reason", - "request_completed_customer_wants_human_followup", - "other", - ], - description: "The specific reason code for the transfer.", - }, - }, - required: ["summary"], - }, - }, - }, - { - type: "function", - function: { - name: "get_current_time", - description: - "Get the current time. Use this to get the current timestamp for logging verification records.", - parameters: { - type: "object", - properties: {}, - }, - }, - }, - { - type: "function", - function: { - name: "get_user_information_by_id", - description: - "Get the information (date of birth, email, phone number, address) for a user by their user id.", - parameters: { - type: "object", - properties: { - user_id: { - type: "string", - description: "The ID of the user", - }, - }, - required: ["user_id"], - }, - }, - }, - { - type: "function", - function: { - name: "get_user_information_by_name", - description: - "Get the information (date of birth, email, phone number, address) for a user by their name. Case Sensitive.", - parameters: { - type: "object", - properties: { - customer_name: { - type: "string", - description: "The name of the user", - }, - }, - required: ["customer_name"], - }, - }, - }, - { - type: "function", - function: { - name: "get_user_information_by_email", - description: - "Get the information (date of birth, email, phone number, address) for a user by their email.", - parameters: { - type: "object", - properties: { - email: { - type: "string", - description: "The email of the user", - }, - }, - required: ["email"], - }, - }, - }, - { - type: "function", - function: { - name: "change_user_email", - description: "Change the email address for a user.", - parameters: { - type: "object", - properties: { - user_id: { - type: "string", - description: "The ID of the user whose email should be changed", - }, - new_email: { - type: "string", - description: "The new email address to set for the user", - }, - }, - required: ["user_id", "new_email"], - }, - }, - }, - { - type: "function", - function: { - name: "get_referrals_by_user", - description: "Get all referrals made by a user.", - parameters: { - type: "object", - properties: { - user_id: { - type: "string", - description: - "The ID of the user (referrer) to look up referrals for", - }, - }, - required: ["user_id"], - }, - }, - }, - { - type: "function", - function: { - name: "get_credit_card_transactions_by_user", - description: "Get all credit card transactions for a user.", - parameters: { - type: "object", - properties: { - user_id: { - type: "string", - description: "The ID of the user to look up transactions for", - }, - }, - required: ["user_id"], - }, - }, - }, - { - type: "function", - function: { - name: "get_credit_card_accounts_by_user", - description: - "Get all credit card accounts for a user. Returns information about each credit card account including card type, date opened, current balance, and reward points.", - parameters: { - type: "object", - properties: { - user_id: { - type: "string", - description: - "The ID of the user to look up credit card accounts for", - }, - }, - required: ["user_id"], - }, - }, - }, - { - type: "function", - function: { - name: "log_verification", - description: - "Log a verification record after successfully verifying a user's identity. Call this tool after you have verified a user by confirming 2 out of 4 identity fields (date of birth, email, phone number, address). This creates an audit record of the verification.", - parameters: { - type: "object", - properties: { - name: { - type: "string", - description: "The verified user's full name", - }, - user_id: { - type: "string", - description: "The verified user's ID", - }, - address: { - type: "string", - description: "The verified user's address", - }, - email: { - type: "string", - description: "The verified user's email", - }, - phone_number: { - type: "string", - description: "The verified user's phone number", - }, - date_of_birth: { - type: "string", - description: - "The verified user's date of birth (MM/DD/YYYY format)", - }, - time_verified: { - type: "string", - description: - 'The timestamp of the verification (e.g., "2025-11-14 03:40:00 EST")', - }, - }, - required: [ - "name", - "user_id", - "address", - "email", - "phone_number", - "date_of_birth", - "time_verified", - ], - }, - }, - }, - { - type: "function", - function: { - name: "give_discoverable_user_tool", - description: - 'Pass a tool to the user so they can execute it themselves. Use this when the knowledge base indicates that the user should perform an action themselves (e.g., "to do X, have the user call tool_name(args)"). The user will then be able to call `call_discoverable_user_tool` with the same tool name and arguments to simulate executing the action.', - parameters: { - type: "object", - properties: { - discoverable_tool_name: { - type: "string", - description: - 'The name of the discoverable tool (e.g., "open_webpage", "navigate_to_section")', - }, - arguments: { - type: "string", - description: - 'JSON string of arguments for the tool (e.g., \'{"url": "https://example.com"}\')', - }, - }, - required: ["discoverable_tool_name"], - }, - }, - }, - { - type: "function", - function: { - name: "unlock_discoverable_agent_tool", - description: - "Unlock an agent discoverable tool that was found in the knowledge base. Use this when the knowledge base indicates that you have access to a specialized internal tool. The knowledge base will tell you the tool name to unlock. After unlocking, you can use the tool by calling `call_discoverable_agent_tool` with the tool name and required arguments.", - parameters: { - type: "object", - properties: { - agent_tool_name: { - type: "string", - description: - 'The name of the agent discoverable tool to unlock (e.g., "calculate_apr_adjustment_7842")', - }, - }, - required: ["agent_tool_name"], - }, - }, - }, - { - type: "function", - function: { - name: "call_discoverable_agent_tool", - description: - "Call an agent discoverable tool that you have previously unlocked. Use this after unlocking a tool with `unlock_discoverable_agent_tool`. The knowledge base will tell you which tool to use and what arguments to provide.", - parameters: { - type: "object", - properties: { - agent_tool_name: { - type: "string", - description: "The name of the agent discoverable tool to call", - }, - arguments: { - type: "string", - description: - 'JSON string of arguments for the tool (e.g., \'{"user_id": "abc123"}\')', - }, - }, - required: ["agent_tool_name"], - }, - }, - }, - { - type: "function", - function: { - name: "list_discoverable_agent_tools", - description: - "List all agent discoverable tools that you have called. Use this to see what specialized tools you have used.", - parameters: { - type: "object", - properties: {}, - }, - }, - }, -]; diff --git a/src/benchmarks/tau3-bench-banking/tools/field-access.ts b/src/benchmarks/tau3-bench-banking/tools/field-access.ts deleted file mode 100644 index de090588..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/field-access.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { isDefinedAndNotNull, isRecord } from "../../../internal/guards"; - -export function getFieldAsString( - record: Record, - field: string -): string | undefined { - const value = record[field]; - return typeof value === "string" ? value : undefined; -} - -export function getFieldAsNumber( - record: Record, - field: string -): number | undefined { - const value = record[field]; - return typeof value === "number" ? value : undefined; -} - -export function getFieldAsBoolean( - record: Record, - field: string -): boolean { - const value = record[field]; - return typeof value === "boolean" ? value : false; -} - -export function getFieldAsRecord( - record: Record, - field: string -): Record | undefined { - const value = record[field]; - return isRecord(value) ? value : undefined; -} - -export function getFieldAsArray( - record: Record, - field: string -): unknown[] | undefined { - const value = record[field]; - return Array.isArray(value) ? value : undefined; -} - -export function getFieldCoalesced( - record: Record, - ...fields: string[] -): unknown { - for (const field of fields) { - const value = record[field]; - if (isDefinedAndNotNull(value)) { - return value; - } - } - return undefined; -} diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-accounts.test.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-accounts.test.ts deleted file mode 100644 index 853388ec..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-accounts.test.ts +++ /dev/null @@ -1,721 +0,0 @@ -import { describe, it, expect, beforeEach } from "bun:test"; -import assert from "node:assert"; - -import { makeEmptyBankingData } from "../environment"; -import type { BankingData } from "../types"; -import { registerAccountTools } from "./handlers-accounts"; -import { makeBankingEnvState, DISCOVERABLE_AGENT_TOOLS } from "./registry"; - -function createTestDb(): BankingData { - return { - ...makeEmptyBankingData(), - users: { - data: { - user_123: { - user_id: "user_123", - name: "Alice Johnson", - }, - user_456: { - user_id: "user_456", - name: "Bob Smith", - }, - }, - }, - accounts: { - data: { - acc_001: { - account_id: "acc_001", - user_id: "user_123", - class: "checking", - account_type: "checking", - level: "Blue Account", - status: "OPEN", - date_opened: "10/01/2025", - current_holdings: "1000.00", - }, - acc_002: { - account_id: "acc_002", - user_id: "user_123", - class: "saving", - account_type: "savings", - level: "Bronze Account", - status: "OPEN", - date_opened: "09/01/2025", - current_holdings: "500.00", - }, - acc_003: { - account_id: "acc_003", - user_id: "user_456", - class: "checking", - account_type: "checking", - level: "Green Account", - status: "OPEN", - date_opened: "08/15/2023", - current_holdings: "5000.00", - }, - acc_004: { - account_id: "acc_004", - user_id: "user_456", - class: "saving", - account_type: "savings", - level: "Silver Account", - status: "OPEN", - date_opened: "08/15/2023", - current_holdings: "2000.00", - }, - }, - }, - }; -} -describe("handlers-accounts", () => { - beforeEach(() => { - (DISCOVERABLE_AGENT_TOOLS satisfies unknown).clear(); - registerAccountTools(); - }); - describe("open_bank_account_4821", () => { - it("should create a new checking account successfully", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "open_bank_account_4821" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - user_id: "user_123", - account_type: "checking", - account_class: "Standard Checking", - }); - expect(result).toContain("Bank account opened successfully!"); - expect(result).toContain("Account Type: checking"); - expect(result).toContain("Status: OPEN"); - expect(result).toContain("Initial Balance: $0.00"); - }); - it("should reject invalid account_type", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "open_bank_account_4821" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - user_id: "user_123", - account_type: "invalid_type", - account_class: "Test", - }); - expect(result).toContain("Error: Invalid account_type"); - }); - it("should reject savings without eligible checking account", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - db.accounts.data["acc_new"] = { - account_id: "acc_new", - user_id: "user_999", - class: "checking", - account_type: "checking", - status: "OPEN", - date_opened: "11/10/2025", - current_holdings: "0", - }; - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "open_bank_account_4821" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - user_id: "user_999", - account_type: "savings", - account_class: "Savings", - }); - expect(result).toContain( - "Error: Account eligibility requirements not met" - ); - }); - it("should reject business_checking without personal checking", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "open_bank_account_4821" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - user_id: "user_new", - account_type: "business_checking", - account_class: "Business", - }); - expect(result).toContain( - "Error: Account eligibility requirements not met" - ); - }); - it("should reject business_savings without business_checking", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "open_bank_account_4821" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - user_id: "user_123", - account_type: "business_savings", - account_class: "Business Savings", - }); - expect(result).toContain( - "Error: Account eligibility requirements not met" - ); - }); - it("should reject with missing parameters", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "open_bank_account_4821" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - user_id: "", - account_type: "checking", - }); - expect(result).toContain("Error: Missing required parameters"); - }); - }); - describe("transfer_funds_between_bank_accounts_7291", () => { - it("should transfer funds successfully and update both balances", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "transfer_funds_between_bank_accounts_7291" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - source_account_id: "acc_001", - destination_account_id: "acc_002", - amount: 100, - }); - expect(result).toContain("Transfer completed successfully!"); - expect(result).toContain("Amount: $100.00"); - expect(result).toContain("new balance: $900.00"); - expect(result).toContain("new balance: $600.00"); - const source = db.accounts.data["acc_001"]; - assert(source); - const dest = db.accounts.data["acc_002"]; - assert(dest); - expect(source.current_holdings).toBe("$900.00"); - expect(dest.current_holdings).toBe("$600.00"); - }); - it("should reject transfer with insufficient funds", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "transfer_funds_between_bank_accounts_7291" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - source_account_id: "acc_001", - destination_account_id: "acc_002", - amount: 2000, - }); - expect(result).toContain("Error: Insufficient funds"); - expect(result).toContain("$1000.00"); - expect(result).toContain("$2000.00"); - }); - it("should reject transfer from non-existent account", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "transfer_funds_between_bank_accounts_7291" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - source_account_id: "acc_nonexistent", - destination_account_id: "acc_002", - amount: 100, - }); - expect(result).toContain( - "Error: Source account 'acc_nonexistent' not found" - ); - }); - it("should reject transfer from same account to itself", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "transfer_funds_between_bank_accounts_7291" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - source_account_id: "acc_001", - destination_account_id: "acc_001", - amount: 100, - }); - expect(result).toContain( - "Error: Source and destination accounts cannot be the same" - ); - }); - it("should reject negative transfer amount", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "transfer_funds_between_bank_accounts_7291" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - source_account_id: "acc_001", - destination_account_id: "acc_002", - amount: -50, - }); - expect(result).toContain("Error: Transfer amount must be positive"); - }); - }); - describe("apply_checking_account_credit_5829", () => { - it("should apply credit successfully and create transaction record", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "apply_checking_account_credit_5829" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_001", - amount: 50, - credit_type: "rebate_credit", - }); - expect(result).toContain("Credit applied successfully!"); - expect(result).toContain("Credit Type: rebate_credit"); - expect(result).toContain("Amount: $50.00"); - expect(result).toContain("Previous Balance: $1000.00"); - expect(result).toContain("New Balance: $1050.00"); - const account = db.accounts.data["acc_001"]; - assert(account); - expect(account.current_holdings).toBe("$1050.00"); - const txnKeys = Object.keys(db.bank_account_transaction_history.data); - expect(txnKeys.length).toBeGreaterThan(0); - const txnId = txnKeys[0]; - if (!txnId) { - throw new Error("Expected transaction ID"); - } - const txn = db.bank_account_transaction_history.data[txnId]; - assert(txn); - expect(txn.account_id).toBe("acc_001"); - expect(txn.type).toBe("rebate_credit"); - }); - it("should reject invalid credit_type", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "apply_checking_account_credit_5829" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_001", - amount: 50, - credit_type: "invalid_credit", - }); - expect(result).toContain("Error: Invalid credit_type"); - }); - it("should reject a non-numeric amount without touching the balance", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "apply_checking_account_credit_5829" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_001", - amount: "abc", - credit_type: "rebate_credit", - }); - expect(result).toBe("Error: Invalid credit amount. Must be a number."); - const account = db.accounts.data["acc_001"]; - assert(account); - expect(account.current_holdings).toBe("1000.00"); - }); - it("should reject credit on non-checking account", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "apply_checking_account_credit_5829" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_002", - amount: 50, - credit_type: "rebate_credit", - }); - expect(result).toContain("Error: Account"); - expect(result).toContain("is not a checking account"); - }); - }); - describe("apply_savings_account_credit_6831", () => { - it("should reject a non-numeric amount without touching the balance", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "apply_savings_account_credit_6831" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_002", - amount: "abc", - credit_type: "interest_correction", - }); - expect(result).toBe("Error: Invalid credit amount. Must be a number."); - const account = db.accounts.data["acc_002"]; - assert(account); - expect(account.current_holdings).toBe("500.00"); - }); - it("should apply savings credit successfully", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "apply_savings_account_credit_6831" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_002", - amount: 25.5, - credit_type: "interest_correction", - }); - expect(result).toContain("Credit applied successfully!"); - expect(result).toContain("Credit Type: interest_correction"); - expect(result).toContain("Amount: $25.50"); - expect(result).toContain("Previous Balance: $500.00"); - expect(result).toContain("New Balance: $525.50"); - const account = db.accounts.data["acc_002"]; - assert(account); - expect(account.current_holdings).toBe("525.50"); - }); - it("should reject positive amount requirement", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "apply_savings_account_credit_6831" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_002", - amount: 0, - credit_type: "interest_correction", - }); - expect(result).toContain("Error: Credit amount must be positive"); - }); - it("should reject credit on non-savings account", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "apply_savings_account_credit_6831" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_001", - amount: 25, - credit_type: "interest_correction", - }); - expect(result).toContain("Error: Account"); - expect(result).toContain("is not a savings account"); - }); - }); - describe("submit_interest_discrepancy_report_7294", () => { - it("should submit interest discrepancy report successfully", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "submit_interest_discrepancy_report_7294" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_002", - user_id: "user_123", - expected_apy: 2.775, - actual_apy: 2.5, - amount_difference: 6.88, - }); - expect(result).toContain( - "Interest Discrepancy Report Submitted Successfully!" - ); - expect(result).toContain("Expected APY: 2.775%"); - expect(result).toContain("Actual APY: 2.5%"); - expect(result).toContain("Amount Difference: $6.88"); - expect(result).toContain("Status: PENDING_REVIEW"); - }); - it("should reject non-existent account", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "submit_interest_discrepancy_report_7294" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_nonexistent", - user_id: "user_123", - expected_apy: 2.775, - actual_apy: 2.5, - amount_difference: 6.88, - }); - expect(result).toContain("Error: Account 'acc_nonexistent' not found"); - }); - it("should reject non-existent user", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "submit_interest_discrepancy_report_7294" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_002", - user_id: "user_nonexistent", - expected_apy: 2.775, - actual_apy: 2.5, - amount_difference: 6.88, - }); - expect(result).toContain("Error: User 'user_nonexistent' not found"); - }); - it("should handle APY difference rounding correctly", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "submit_interest_discrepancy_report_7294" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_002", - user_id: "user_123", - expected_apy: 2.7753, - actual_apy: 2.5001, - amount_difference: 6.88, - }); - expect(result).toContain("APY Difference: 0.2752%"); - }); - }); - describe("close_bank_account_7392", () => { - it("should close account with zero balance successfully", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const openAccount = db.accounts.data["acc_001"]; - assert(openAccount); - openAccount.current_holdings = "0.00"; - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "close_bank_account_7392" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_001", - reason: "Customer request", - waive_early_closure_fee: true, - }); - expect(result).toContain("Bank account closed successfully!"); - expect(result).toContain("Status: CLOSED"); - expect(result).toContain("Early Closure Fee Waived: Yes"); - const account = db.accounts.data["acc_001"]; - assert(account); - expect(account.status).toBe("CLOSED"); - }); - it("should reject closing already-closed account", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const closedAccount = db.accounts.data["acc_001"]; - assert(closedAccount); - closedAccount.status = "CLOSED"; - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "close_bank_account_7392" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_001", - reason: "Test", - }); - expect(result).toContain("Error: Account"); - expect(result).toContain("is already closed"); - }); - it("should reject closing account with non-zero balance", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "close_bank_account_7392" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_001", - reason: "Test", - waive_early_closure_fee: true, - }); - expect(result).toContain("Error: Account balance must be $0.00"); - expect(result).toContain("$1000.00"); - }); - it("should reject with missing account_id", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "close_bank_account_7392" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { reason: "Test" }); - expect(result).toContain("Error: Missing required parameter"); - }); - }); - describe("get_all_user_accounts_by_user_id_3847", () => { - it("should return all accounts for user", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "get_all_user_accounts_by_user_id_3847" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - user_id: "user_123", - }); - expect(result).toContain("User accounts retrieved successfully"); - expect(result).toContain("Bank Accounts:"); - expect(result).toContain("Credit Card Accounts:"); - }); - it("should reject with missing user_id", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "get_all_user_accounts_by_user_id_3847" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, {}); - expect(result).toContain("Error: Missing required parameter: user_id"); - }); - }); - describe("get_bank_account_transactions_9173", () => { - it("should return transactions for account", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - db.bank_account_transaction_history.data["txn_001"] = { - transaction_id: "txn_001", - account_id: "acc_001", - date: "11/14/2025", - description: "Test transaction", - amount: 100, - type: "credit", - status: "posted", - }; - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "get_bank_account_transactions_9173" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_001", - }); - expect(result).toContain( - "Bank account transactions retrieved successfully" - ); - expect(result).toContain("Transactions for account acc_001"); - expect(result).toContain( - "Found 1 record(s) in 'bank_account_transaction_history':" - ); - expect(result).toContain("1. Record ID: txn_001"); - }); - it("should return transactions in reverse chronological order", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - db.bank_account_transaction_history.data = { - older: { - transaction_id: "older", - account_id: "acc_001", - date: "11/13/2025", - description: "Older", - }, - newer: { - transaction_id: "newer", - account_id: "acc_001", - date: "11/14/2025 03:40:00", - description: "Newer", - }, - }; - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "get_bank_account_transactions_9173" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { account_id: "acc_001" }); - expect(result.indexOf("Record ID: newer")).toBeLessThan( - result.indexOf("Record ID: older") - ); - }); - it("should reject non-existent account", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "get_bank_account_transactions_9173" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, { - account_id: "acc_nonexistent", - }); - expect(result).toContain("Error: Account 'acc_nonexistent' not found"); - }); - it("should reject with missing account_id", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const handler = DISCOVERABLE_AGENT_TOOLS.get( - "get_bank_account_transactions_9173" - )?.handler; - if (!handler) { - throw new Error("Tool not registered"); - } - const result = handler(state, {}); - expect(result).toContain("Error: Missing required parameter: account_id"); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-accounts.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-accounts.ts deleted file mode 100644 index 8447380f..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-accounts.ts +++ /dev/null @@ -1,925 +0,0 @@ -import { createHash } from "node:crypto"; - -import { addToDb, queryDatabaseTool, queryDb } from "./db-query"; -import { - getFieldAsString, - getFieldAsBoolean, - getFieldCoalesced, -} from "./field-access"; -import { getAccountBalance, getTodayStr } from "./helpers"; -import type { BankingEnvState } from "./registry"; -import { registerDiscoverableAgentTool } from "./registry"; - -function toFiniteNumber(value: unknown): number | undefined { - if (typeof value === "number") { - return Number.isFinite(value) ? value : undefined; - } - if (typeof value === "string") { - const parsed = Number.parseFloat(value); - return Number.isFinite(parsed) ? parsed : undefined; - } - return undefined; -} - -function getNow(): Date { - return new Date(2025, 10, 14, 3, 40, 0); -} - -function getAccountAgeDays(account: Record): number { - const dateOpenedStr = getFieldAsString(account, "date_opened"); - if (!dateOpenedStr) { - return 0; - } - try { - const parts = dateOpenedStr.split("/"); - if (parts.length !== 3) { - return 0; - } - const month = Number.parseInt(parts[0]!, 10); - const day = Number.parseInt(parts[1]!, 10); - const year = Number.parseInt(parts[2]!, 10); - const dateOpened = new Date(year, month - 1, day); - const today = getNow(); - const diffTime = today.getTime() - dateOpened.getTime(); - return Math.floor(diffTime / (1000 * 60 * 60 * 24)); - } catch { - return 0; - } -} - -function openBankAccount( - state: BankingEnvState, - kwargs: Record -): string { - const userId = getFieldAsString(kwargs, "user_id"); - const accountType = getFieldAsString(kwargs, "account_type"); - const accountClass = getFieldAsString(kwargs, "account_class"); - if (!userId || !accountType || !accountClass) { - return "Error: Missing required parameters."; - } - const validTypes = [ - "checking", - "savings", - "business_checking", - "business_savings", - ]; - if (!validTypes.includes(accountType)) { - return `Error: Invalid account_type. Must be one of: ${JSON.stringify(validTypes)}`; - } - const userAccounts = queryDb({ - db: state.db, - dbName: "accounts", - constraints: { user_id: userId }, - }); - if (accountType === "savings") { - let hasEligibleChecking = false; - for (const acc of userAccounts) { - const accType = getFieldCoalesced(acc, "account_type", "class"); - const { status } = acc; - if ( - (accType === "checking" || accType === "personal_checking") && - (status === "OPEN" || status === "ACTIVE") && - getAccountAgeDays(acc) >= 14 - ) { - hasEligibleChecking = true; - break; - } - } - if (!hasEligibleChecking) { - return "Error: Account eligibility requirements not met."; - } - } - if (accountType === "business_checking") { - const hasClosedAccount = userAccounts.some( - (acc) => acc.status === "CLOSED" - ); - if (hasClosedAccount) { - return "Error: Account eligibility requirements not met."; - } - const hasPersonalChecking = userAccounts.some((acc) => { - const accType = getFieldCoalesced(acc, "account_type", "class"); - const { status } = acc; - return ( - (accType === "checking" || accType === "personal_checking") && - (status === "OPEN" || status === "ACTIVE") - ); - }); - if (!hasPersonalChecking) { - return "Error: Account eligibility requirements not met."; - } - } - if (accountType === "business_savings") { - for (const acc of userAccounts) { - const balance = getAccountBalance(acc); - if (balance < 0) { - return "Error: Account eligibility requirements not met."; - } - } - let hasEligibleBusinessChecking = false; - for (const acc of userAccounts) { - const accType = acc.account_type; - const { status } = acc; - if ( - accType === "business_checking" && - (status === "OPEN" || status === "ACTIVE") && - getAccountAgeDays(acc) >= 30 - ) { - hasEligibleBusinessChecking = true; - break; - } - } - if (!hasEligibleBusinessChecking) { - return "Error: Account eligibility requirements not met."; - } - } - const seed = `account:${userId}:${accountType}:${accountClass}`; - const accountId = createHash("sha256") - .update(seed) - .digest() - .slice(0, 8) - .toString("hex"); - const today = getTodayStr(); - const accountRecord: Record = { - account_id: accountId, - user_id: userId, - account_type: accountType, - account_class: accountClass, - current_holdings: "0.00", - status: "OPEN", - date_opened: today, - }; - const success = addToDb({ - db: state.db, - dbName: "accounts", - recordId: accountId, - record: accountRecord, - }); - if (!success) { - return `Failed to open account: Account ID '${accountId}' may already exist.`; - } - return ( - `Bank account opened successfully!\n` + - ` - Account ID: ${accountId}\n` + - ` - User ID: ${userId}\n` + - ` - Account Type: ${accountType}\n` + - ` - Account Class: ${accountClass}\n` + - ` - Status: OPEN\n` + - ` - Initial Balance: $0.00\n` + - ` - Date Opened: ${today}` - ); -} - -function closeBankAccount( - state: BankingEnvState, - kwargs: Record -): string { - const accountId = getFieldAsString(kwargs, "account_id"); - const reason = - getFieldAsString(kwargs, "reason") ?? "Customer requested closure"; - const waiveEarlyClosureFee = getFieldAsBoolean( - kwargs, - "waive_early_closure_fee" - ); - const PERSONAL_CHECKING_EARLY_CLOSURE: Record< - string, - { - fee: number; - window_days: number; - } - > = { - "Light Blue Account": { fee: 15, window_days: 30 }, - "Light Green Account": { fee: 15, window_days: 30 }, - "Green Fee-Free Account": { fee: 15, window_days: 30 }, - "Blue Account": { fee: 25, window_days: 60 }, - "Green Account": { fee: 25, window_days: 60 }, - "Evergreen Account": { fee: 50, window_days: 90 }, - "Bluest Account": { fee: 100, window_days: 180 }, - }; - const PERSONAL_SAVINGS_EARLY_CLOSURE: Record< - string, - { - fee: number; - window_days: number; - } - > = { - "Bronze Account": { fee: 20, window_days: 60 }, - "Silver Account": { fee: 35, window_days: 90 }, - "Silver Plus Account": { fee: 35, window_days: 90 }, - "Gold Account": { fee: 75, window_days: 180 }, - "Gold Plus Account": { fee: 75, window_days: 180 }, - "Gold Years Account": { fee: 75, window_days: 180 }, - "Platinum Account": { fee: 150, window_days: 270 }, - "Platinum Plus Account": { fee: 150, window_days: 270 }, - "Diamond Elite Account": { fee: 150, window_days: 270 }, - }; - if (!accountId) { - return "Error: Missing required parameter (account_id)."; - } - if (!(accountId in state.db.accounts.data)) { - return `Error: Account '${accountId}' not found.`; - } - const account = state.db.accounts.data[accountId]; - if (!account || typeof account !== "object") { - return `Error: Account '${accountId}' not found.`; - } - if (account.status === "CLOSED") { - return `Error: Account '${accountId}' is already closed.`; - } - const balance = getAccountBalance(account); - let earlyClosureFeeApplied = 0; - if (!waiveEarlyClosureFee) { - const accountLevel = getFieldAsString(account, "level"); - const accountClass = getFieldAsString(account, "class"); - const dateOpenedStr = getFieldAsString(account, "date_opened"); - let earlyClosureConfig: - | { - fee: number; - window_days: number; - } - | undefined; - if (accountClass === "checking" && accountLevel) { - earlyClosureConfig = PERSONAL_CHECKING_EARLY_CLOSURE[accountLevel]; - } else if ( - (accountClass === "savings" || accountClass === "saving") && - accountLevel - ) { - earlyClosureConfig = PERSONAL_SAVINGS_EARLY_CLOSURE[accountLevel]; - } - if (earlyClosureConfig && dateOpenedStr) { - try { - const accountAgeDays = getAccountAgeDays(account); - if (accountAgeDays < earlyClosureConfig.window_days) { - const requiredFee = earlyClosureConfig.fee; - if (balance < requiredFee) { - return "Error: Account unable to be closed."; - } - earlyClosureFeeApplied = requiredFee; - } - } catch {} - } - } - const remainingBalance = balance - earlyClosureFeeApplied; - if (remainingBalance !== 0) { - return `Error: Account balance must be $0.00 before closing. Current balance: $${balance.toFixed(2)}`; - } - const today = getTodayStr(); - account.status = "CLOSED"; - account.date_closed = today; - account.closure_reason = reason; - account.early_closure_fee_waived = waiveEarlyClosureFee; - return ( - `Bank account closed successfully!\n` + - ` - Account ID: ${accountId}\n` + - ` - Account Type: ${getFieldAsString(account, "account_type") ?? "N/A"}\n` + - ` - Account Class: ${getFieldAsString(account, "account_class") ?? "N/A"}\n` + - ` - Status: CLOSED\n` + - ` - Date Closed: ${today}\n` + - ` - Reason: ${reason}\n` + - ` - Early Closure Fee Waived: ${waiveEarlyClosureFee ? "Yes" : "No"}` - ); -} - -function getAllUserAccountsByUserId( - state: BankingEnvState, - kwargs: Record -): string { - const userId = getFieldAsString(kwargs, "user_id"); - if (!userId) { - return "Error: Missing required parameter: user_id"; - } - const accountsResult = queryDatabaseTool( - state.db, - "accounts", - JSON.stringify({ user_id: userId }) - ); - const ccResult = queryDatabaseTool( - state.db, - "credit_card_accounts", - JSON.stringify({ user_id: userId }) - ); - const resultParts = [ - "User accounts retrieved successfully.", - "", - `Executed: get_all_user_accounts_by_user_id_3847`, - `Accounts for user ${userId}:`, - "", - "Bank Accounts:", - ]; - if ( - !accountsResult.includes("No records found") && - !accountsResult.includes("No results found") - ) { - resultParts.push(accountsResult); - } else { - resultParts.push(" No bank accounts found."); - } - resultParts.push("\nCredit Card Accounts:"); - if ( - !ccResult.includes("No records found") && - !ccResult.includes("No results found") - ) { - resultParts.push(ccResult); - } else { - resultParts.push(" No credit card accounts found."); - } - return resultParts.join("\n"); -} - -function transferFundsBetweenBankAccounts( - state: BankingEnvState, - kwargs: Record -): string { - const sourceAccountId = getFieldAsString(kwargs, "source_account_id"); - const destinationAccountId = getFieldAsString( - kwargs, - "destination_account_id" - ); - const { amount } = kwargs; - if ( - !sourceAccountId || - !destinationAccountId || - amount === null || - amount === undefined - ) { - return "Error: Missing required parameters (source_account_id, destination_account_id, amount)."; - } - let parsedAmount: number; - try { - parsedAmount = - typeof amount === "string" - ? Number.parseFloat(amount) - : (amount as number); - } catch { - return `Error: Invalid amount '${amount}'. Must be a number.`; - } - if (isNaN(parsedAmount)) { - return `Error: Invalid amount '${amount}'. Must be a number.`; - } - if (parsedAmount <= 0) { - return "Error: Transfer amount must be positive."; - } - if (sourceAccountId === destinationAccountId) { - return "Error: Source and destination accounts cannot be the same."; - } - if (!(sourceAccountId in state.db.accounts.data)) { - return `Error: Source account '${sourceAccountId}' not found.`; - } - if (!(destinationAccountId in state.db.accounts.data)) { - return `Error: Destination account '${destinationAccountId}' not found.`; - } - const source = state.db.accounts.data[sourceAccountId]; - if (!source || typeof source !== "object") { - return `Error: Source account '${sourceAccountId}' not found.`; - } - if (!(source.status === "ACTIVE" || source.status === "OPEN")) { - return `Error: Source account '${sourceAccountId}' is not active.`; - } - const dest = state.db.accounts.data[destinationAccountId]; - if (!dest || typeof dest !== "object") { - return `Error: Destination account '${destinationAccountId}' not found.`; - } - if (!(dest.status === "ACTIVE" || dest.status === "OPEN")) { - return `Error: Destination account '${destinationAccountId}' is not active.`; - } - const sourceBalance = getAccountBalance(source); - if (sourceBalance < parsedAmount) { - return ( - `Error: Insufficient funds. Source account balance is $${sourceBalance.toFixed(2)}, ` + - `but transfer amount is $${parsedAmount.toFixed(2)}.` - ); - } - const destBalance = getAccountBalance(dest); - const newSource = sourceBalance - parsedAmount; - const newDest = destBalance + parsedAmount; - source.current_holdings = `$${newSource.toFixed(2)}`; - dest.current_holdings = `$${newDest.toFixed(2)}`; - return ( - `Transfer completed successfully!\n` + - ` - Amount: $${parsedAmount.toFixed(2)}\n` + - ` - From: ${sourceAccountId} (new balance: $${newSource.toFixed(2)})\n` + - ` - To: ${destinationAccountId} (new balance: $${newDest.toFixed(2)})` - ); -} - -function applyCheckingAccountCredit( - state: BankingEnvState, - kwargs: Record -): string { - const accountId = getFieldAsString(kwargs, "account_id"); - const { amount } = kwargs; - const creditType = getFieldAsString(kwargs, "credit_type"); - if (!accountId || amount === null || amount === undefined || !creditType) { - return "Error: Missing required parameters."; - } - const parsedAmount = - typeof amount === "number" ? amount : Number(String(amount)); - if (!Number.isFinite(parsedAmount)) { - return "Error: Invalid credit amount. Must be a number."; - } - if (parsedAmount <= 0) { - return "Error: Credit amount must be positive."; - } - const validTypes = ["rebate_credit", "fee_refund"]; - if (!validTypes.includes(creditType)) { - return `Error: Invalid credit_type. Must be one of: ${validTypes}`; - } - if (!(accountId in state.db.accounts.data)) { - return `Error: Account '${accountId}' not found.`; - } - const account = state.db.accounts.data[accountId]; - if (!account || typeof account !== "object") { - return `Error: Account '${accountId}' not found.`; - } - const accountClass = getFieldAsString(account, "class"); - if (!accountClass || accountClass.toLowerCase() !== "checking") { - return `Error: Account '${accountId}' is not a checking account. Credits can only be applied to checking accounts.`; - } - if (!(account.status === "ACTIVE" || account.status === "OPEN")) { - return `Error: Account '${accountId}' is not active.`; - } - const currentBalance = getAccountBalance(account); - const newBalance = currentBalance + parsedAmount; - account.current_holdings = `$${newBalance.toFixed(2)}`; - const seed = `checking_credit:${accountId}:${creditType}:${parsedAmount}:${getTodayStr()}`; - const txnHash = createHash("sha256") - .update(seed) - .digest() - .slice(0, 6) - .toString("hex"); - const transactionId = `txn_${txnHash}`; - const description = - creditType === "rebate_credit" - ? "REBATE CREDIT - CUSTOMER SERVICE" - : "FEE REFUND - CUSTOMER SERVICE"; - const transactionRecord: Record = { - transaction_id: transactionId, - account_id: accountId, - date: getTodayStr(), - description, - amount: parsedAmount, - type: creditType, - status: "posted", - }; - state.db.bank_account_transaction_history.data[transactionId] = - transactionRecord; - return ( - `\nCredit applied successfully!\n` + - ` - Transaction ID: ${transactionId}\n` + - ` - Account: ${accountId}\n` + - ` - Credit Type: ${creditType}\n` + - ` - Amount: $${parsedAmount.toFixed(2)}\n` + - ` - Previous Balance: $${currentBalance.toFixed(2)}\n` + - ` - New Balance: $${newBalance.toFixed(2)}` - ); -} - -function applySavingsAccountCredit( - state: BankingEnvState, - kwargs: Record -): string { - const accountId = getFieldAsString(kwargs, "account_id"); - const { amount } = kwargs; - const creditType = getFieldAsString(kwargs, "credit_type"); - if (!accountId || amount === null || amount === undefined || !creditType) { - return "Error: Missing required parameters."; - } - const parsedAmount = - typeof amount === "number" ? amount : Number(String(amount)); - if (!Number.isFinite(parsedAmount)) { - return "Error: Invalid credit amount. Must be a number."; - } - if (parsedAmount <= 0) { - return "Error: Credit amount must be positive."; - } - const validTypes = ["interest_correction", "fee_refund", "goodwill_credit"]; - if (!validTypes.includes(creditType)) { - return `Error: Invalid credit_type. Must be one of: ${validTypes}`; - } - if (!(accountId in state.db.accounts.data)) { - return `Error: Account '${accountId}' not found.`; - } - const account = state.db.accounts.data[accountId]; - if (!account || typeof account !== "object") { - return `Error: Account '${accountId}' not found.`; - } - const accountClass = getFieldAsString(account, "class"); - if ( - !accountClass || - (accountClass.toLowerCase() !== "saving" && - accountClass.toLowerCase() !== "savings") - ) { - return `Error: Account '${accountId}' is not a savings account. This tool only applies to savings accounts.`; - } - if (!(account.status === "ACTIVE" || account.status === "OPEN")) { - return `Error: Account '${accountId}' is not active.`; - } - const currentBalance = getAccountBalance(account); - const newBalance = currentBalance + parsedAmount; - account.current_holdings = `${newBalance.toFixed(2)}`; - const seed = `savings_credit:${accountId}:${creditType}:${parsedAmount}:${getTodayStr()}`; - const txnHash = createHash("sha256") - .update(seed) - .digest() - .slice(0, 6) - .toString("hex"); - const transactionId = `txn_${txnHash}`; - let description: string; - if (creditType === "interest_correction") { - description = "INTEREST CORRECTION - CUSTOMER SERVICE"; - } else if (creditType === "fee_refund") { - description = "FEE REFUND - CUSTOMER SERVICE"; - } else { - description = "GOODWILL CREDIT - CUSTOMER SERVICE"; - } - const transactionRecord: Record = { - transaction_id: transactionId, - account_id: accountId, - date: getTodayStr(), - description, - amount: parsedAmount, - type: creditType, - status: "posted", - }; - state.db.bank_account_transaction_history.data[transactionId] = - transactionRecord; - return ( - `\nCredit applied successfully!\n` + - ` - Transaction ID: ${transactionId}\n` + - ` - Account: ${accountId}\n` + - ` - Credit Type: ${creditType}\n` + - ` - Amount: $${parsedAmount.toFixed(2)}\n` + - ` - Previous Balance: $${currentBalance.toFixed(2)}\n` + - ` - New Balance: $${newBalance.toFixed(2)}` - ); -} - -function submitInterestDiscrepancyReport( - state: BankingEnvState, - kwargs: Record -): string { - const accountId = getFieldAsString(kwargs, "account_id"); - const userId = getFieldAsString(kwargs, "user_id"); - const expectedApy = kwargs.expected_apy; - const actualApy = kwargs.actual_apy; - const amountDifference = kwargs.amount_difference; - if ( - !accountId || - !userId || - expectedApy === null || - expectedApy === undefined || - actualApy === null || - actualApy === undefined || - amountDifference === null || - amountDifference === undefined - ) { - return "Error: Missing required parameters."; - } - const parsedExpectedApy = toFiniteNumber(expectedApy); - const parsedActualApy = toFiniteNumber(actualApy); - const parsedAmountDifference = toFiniteNumber(amountDifference); - if ( - parsedExpectedApy === undefined || - parsedActualApy === undefined || - parsedAmountDifference === undefined - ) { - return "Error: expected_apy, actual_apy, and amount_difference must be numbers."; - } - const account = state.db.accounts.data[accountId]; - if (!account || typeof account !== "object") { - return `Error: Account '${accountId}' not found.`; - } - const user = state.db.users.data[userId]; - if (!user || typeof user !== "object") { - return `Error: User '${userId}' not found.`; - } - const seed = `interest_report:${accountId}:${userId}:${parsedExpectedApy}:${parsedActualApy}:${getTodayStr()}`; - const reportHash = createHash("sha256") - .update(seed) - .digest() - .slice(0, 6) - .toString("hex"); - const reportId = `IDR_${reportHash}`; - const apyDifference = - Math.round((parsedExpectedApy - parsedActualApy) * 10000) / 10000; - const reportRecord: Record = { - report_id: reportId, - account_id: accountId, - user_id: userId, - account_level: getFieldAsString(account, "level") ?? "Unknown", - expected_apy: parsedExpectedApy, - actual_apy: parsedActualApy, - apy_difference: apyDifference, - amount_difference: parsedAmountDifference, - submitted_date: getTodayStr(), - status: "PENDING_REVIEW", - }; - addToDb({ - db: state.db, - dbName: "interest_discrepancy_reports", - recordId: reportId, - record: reportRecord, - }); - return ( - `\nInterest Discrepancy Report Submitted Successfully!\n` + - ` - Report ID: ${reportId}\n` + - ` - Account: ${accountId} (${getFieldAsString(account, "level") ?? "Unknown"})\n` + - ` - Customer: ${getFieldAsString(user, "name") ?? "Unknown"}\n` + - ` - Expected APY: ${parsedExpectedApy}%\n` + - ` - Actual APY: ${parsedActualApy}%\n` + - ` - APY Difference: ${apyDifference}%\n` + - ` - Amount Difference: $${parsedAmountDifference.toFixed(2)}\n` + - ` - Status: PENDING_REVIEW\n` + - `\nThe backend team will investigate this discrepancy and ensure ` + - `correct APY calculations are applied going forward.` - ); -} - -function getBankAccountTransactions( - state: BankingEnvState, - kwargs: Record -): string { - const accountId = getFieldAsString(kwargs, "account_id"); - if (!accountId) { - return "Error: Missing required parameter: account_id"; - } - if (!(accountId in state.db.accounts.data)) { - return `Error: Account '${accountId}' not found.`; - } - const resultParts = [ - "Bank account transactions retrieved successfully.", - "", - `Executed: get_bank_account_transactions_9173`, - `Transactions for account ${accountId}:`, - ]; - const txns = queryDb({ - db: state.db, - dbName: "bank_account_transaction_history", - constraints: { account_id: accountId }, - options: { returnIds: true }, - }); - if (txns.length > 0) { - const parseDate = (value: unknown): number => { - const dateString = String(value ?? ""); - for (const format of ["datetime", "date"] as const) { - const match = - format === "datetime" - ? /^(\d{2})\/(\d{2})\/(\d{4}) (\d{2}):(\d{2}):(\d{2})$/u.exec( - dateString - ) - : /^(\d{2})\/(\d{2})\/(\d{4})$/u.exec(dateString); - if (match) { - const [ - , - month, - day, - year, - hour = "00", - minute = "00", - second = "00", - ] = match; - const timestamp = Date.UTC( - Number(year), - Number(month) - 1, - Number(day), - Number(hour), - Number(minute), - Number(second) - ); - const parsed = new Date(timestamp); - if ( - parsed.getUTCFullYear() === Number(year) && - parsed.getUTCMonth() === Number(month) - 1 && - parsed.getUTCDate() === Number(day) && - parsed.getUTCHours() === Number(hour) && - parsed.getUTCMinutes() === Number(minute) && - parsed.getUTCSeconds() === Number(second) - ) { - return timestamp; - } - } - } - return Number.MIN_SAFE_INTEGER; - }; - const sortedTxns = [...txns].sort( - ([, left], [, right]) => parseDate(right.date) - parseDate(left.date) - ); - const formattedLines = [ - `Found ${sortedTxns.length} record(s) in 'bank_account_transaction_history':\n`, - ]; - for (const [index, [recordId, record]] of sortedTxns.entries()) { - formattedLines.push(`${index + 1}. Record ID: ${recordId}`); - for (const [field, value] of Object.entries(record)) { - formattedLines.push(` ${field}: ${value}`); - } - formattedLines.push(""); - } - resultParts.push(formattedLines.join("\n")); - } else { - resultParts.push("\nNo transactions found for this account."); - } - return resultParts.join("\n"); -} - -export function registerAccountTools(): void { - registerDiscoverableAgentTool({ - name: "open_bank_account_4821", - description: "Open a new bank account for a customer.", - params: [ - { - name: "user_id", - type: "string", - optional: false, - description: "The customer's unique identifier in the system", - }, - { - name: "account_type", - type: "string", - optional: false, - description: - "Type of account to open. Must be one of: 'checking' (personal checking), 'savings' (personal savings), 'business_checking', 'business_savings'", - }, - { - name: "account_class", - type: "string", - optional: false, - description: "The full official account class name", - }, - ], - handler: openBankAccount, - }); - registerDiscoverableAgentTool({ - name: "close_bank_account_7392", - description: "Close a customer's bank account (checking or savings).", - params: [ - { - name: "account_id", - type: "string", - optional: false, - description: "The ID of the bank account to close", - }, - { - name: "reason", - type: "string", - optional: true, - description: "The reason for closing the account", - }, - { - name: "waive_early_closure_fee", - type: "boolean", - optional: true, - description: "Whether to waive early closure fees", - }, - ], - handler: closeBankAccount, - }); - registerDiscoverableAgentTool({ - name: "get_all_user_accounts_by_user_id_3847", - mutatesState: false, - description: - "Retrieve all accounts (checking, savings, credit cards) for a customer.", - params: [ - { - name: "user_id", - type: "string", - optional: false, - description: "The customer's unique identifier in the system", - }, - ], - handler: getAllUserAccountsByUserId, - }); - registerDiscoverableAgentTool({ - name: "transfer_funds_between_bank_accounts_7291", - description: "Transfer funds from one bank account to another.", - params: [ - { - name: "source_account_id", - type: "string", - optional: false, - description: "The account ID to transfer funds from", - }, - { - name: "destination_account_id", - type: "string", - optional: false, - description: "The account ID to transfer funds to", - }, - { - name: "amount", - type: "number", - optional: false, - description: "The amount to transfer in USD", - }, - ], - handler: transferFundsBetweenBankAccounts, - }); - registerDiscoverableAgentTool({ - name: "apply_checking_account_credit_5829", - description: "Apply a credit to a customer's checking account.", - params: [ - { - name: "account_id", - type: "string", - optional: false, - description: "The checking account ID to credit", - }, - { - name: "amount", - type: "number", - optional: false, - description: - "The positive dollar amount to credit (must be greater than 0)", - }, - { - name: "credit_type", - type: "string", - optional: false, - description: - "The type of credit: 'rebate_credit' for missing rebates, 'fee_refund' for incorrect fee charges", - }, - ], - handler: applyCheckingAccountCredit, - }); - registerDiscoverableAgentTool({ - name: "apply_savings_account_credit_6831", - description: - "Apply a credit to a customer's savings account for interest corrections, fee refunds, or goodwill adjustments.", - params: [ - { - name: "account_id", - type: "string", - optional: false, - description: "The savings account ID to credit", - }, - { - name: "amount", - type: "number", - optional: false, - description: - "The positive dollar amount to credit (must be greater than 0)", - }, - { - name: "credit_type", - type: "string", - optional: false, - description: - "The type of credit: 'interest_correction' for APY/interest calculation errors, 'fee_refund' for incorrect fee charges, 'goodwill_credit' for customer service gestures", - }, - ], - handler: applySavingsAccountCredit, - }); - registerDiscoverableAgentTool({ - name: "submit_interest_discrepancy_report_7294", - description: - "Submit a report for interest calculation discrepancies to the backend team for investigation. Use this when the interest credited to a customer's account does not match expected APY calculations.", - params: [ - { - name: "account_id", - type: "string", - optional: false, - description: "The savings account ID with the discrepancy", - }, - { - name: "user_id", - type: "string", - optional: false, - description: "The customer's unique identifier", - }, - { - name: "expected_apy", - type: "number", - optional: false, - description: - "The APY percentage the customer should have received (e.g., 2.775 for 2.775%)", - }, - { - name: "actual_apy", - type: "number", - optional: false, - description: - "The APY percentage that was actually applied (e.g., 2.5 for 2.5%)", - }, - { - name: "amount_difference", - type: "number", - optional: false, - description: - "The dollar amount difference between expected and actual interest credited", - }, - ], - handler: submitInterestDiscrepancyReport, - }); - registerDiscoverableAgentTool({ - name: "get_bank_account_transactions_9173", - mutatesState: false, - description: "Retrieve the transaction history for a bank account.", - params: [ - { - name: "account_id", - type: "string", - optional: false, - description: "The bank account ID to retrieve transactions for", - }, - ], - handler: getBankAccountTransactions, - }); -} diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-credit-2.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-credit-2.ts deleted file mode 100644 index b2fd3044..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-credit-2.ts +++ /dev/null @@ -1,608 +0,0 @@ -import { queryDb, addToDb } from "./db-query"; -import { getAccountBalance, getTodayStr, parseBalance } from "./helpers"; -import { generateCreditLimitIncreaseRequestId } from "./ids"; -import type { - BankingEnvState, - DiscoverableTool, - ToolParameter, -} from "./registry"; -import { registerDiscoverableAgentTool } from "./registry"; - -function formatUsd(amount: number): string { - const formatted = amount.toFixed(2); - const parts = formatted.split("."); - const intPart = parts[0]!; - const decPart = parts[1]!; - const withCommas = intPart.replaceAll(/\B(?=(\d{3})+(?!\d))/g, ","); - return `${withCommas}.${decPart}`; -} - -function payCreditCardFromChecking( - state: BankingEnvState, - kwargs: Record -): string { - const userId = String(kwargs.user_id ?? ""); - const checkingAccountId = String(kwargs.checking_account_id ?? ""); - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - const amountRaw = kwargs.amount; - if ( - !userId || - !checkingAccountId || - !creditCardAccountId || - amountRaw === null || - amountRaw === undefined - ) { - return "Error: Missing required parameters (user_id, checking_account_id, credit_card_account_id, amount)."; - } - const amount = - typeof amountRaw === "number" - ? amountRaw - : Number.parseFloat(String(amountRaw)); - if (Number.isNaN(amount)) { - return "Error: Invalid payment amount. Must be a positive number."; - } - if (amount <= 0) { - return "Error: Payment amount must be a positive number."; - } - const accountsTable = state.db.accounts; - if (!(checkingAccountId in accountsTable.data)) { - return `Error: Checking account '${checkingAccountId}' not found.`; - } - const checkingAccount = accountsTable.data[checkingAccountId]; - if (!checkingAccount) { - return `Error: Checking account '${checkingAccountId}' not found.`; - } - if (checkingAccount.user_id !== userId) { - return `Error: Checking account '${checkingAccountId}' does not belong to user '${userId}'.`; - } - if (checkingAccount.class !== "checking") { - return `Error: Account '${checkingAccountId}' is not a checking account.`; - } - const currentBalance = getAccountBalance(checkingAccount); - if (amount > currentBalance) { - return `Error: Insufficient funds in checking account. Available balance: $${currentBalance.toFixed(2)}, requested payment: $${amount.toFixed(2)}.`; - } - const ccAccountsTable = state.db.credit_card_accounts; - if (!(creditCardAccountId in ccAccountsTable.data)) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - const ccAccount = ccAccountsTable.data[creditCardAccountId]; - if (!ccAccount) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - if (ccAccount.user_id !== userId) { - return `Error: Credit card account '${creditCardAccountId}' does not belong to user '${userId}'.`; - } - const ccBalanceStr = String(ccAccount.current_balance ?? "$0.00"); - const ccBalance = parseBalance(ccBalanceStr); - if (amount > ccBalance) { - return `Error: Payment amount ($${amount.toFixed(2)}) exceeds credit card balance ($${ccBalance.toFixed(2)}). Please specify an amount up to the outstanding balance.`; - } - const newCheckingBalance = currentBalance - amount; - const newCcBalance = ccBalance - amount; - checkingAccount.current_holdings = newCheckingBalance.toFixed(2); - ccAccount.current_balance = `$${newCcBalance.toFixed(2)}`; - return ( - `Payment processed successfully!\n` + - ` - Payment Amount: $${amount.toFixed(2)}\n` + - ` - From Checking Account: ${checkingAccountId}\n` + - ` - To Credit Card Account: ${creditCardAccountId}\n` + - ` - New Checking Balance: $${newCheckingBalance.toFixed(2)}\n` + - ` - New Credit Card Balance: $${newCcBalance.toFixed(2)}\n` + - `The payment has been applied immediately.` - ); -} - -const payCreditCardFromCheckingTool: DiscoverableTool = { - name: "pay_credit_card_from_checking_9182", - description: - "Pay off a credit card balance using funds from the customer's Rho-Bank checking account. This deducts the specified amount from the checking account and reduces the credit card balance by the same amount.", - params: [ - { - name: "user_id", - type: "string", - optional: false, - description: "The customer's unique identifier in the system", - }, - { - name: "checking_account_id", - type: "string", - optional: false, - description: - "The ID of the Rho-Bank checking account to debit funds from", - }, - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The ID of the credit card account to apply the payment to", - }, - { - name: "amount", - type: "number", - optional: false, - description: "The payment amount in dollars. Must be a positive number.", - }, - ] satisfies readonly ToolParameter[], - handler: payCreditCardFromChecking, -}; - -function submitCreditLimitIncreaseRequest( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - const userId = String(kwargs.user_id ?? ""); - const requestedIncreaseAmountRaw = kwargs.requested_increase_amount; - if ( - !creditCardAccountId || - !userId || - requestedIncreaseAmountRaw === null || - requestedIncreaseAmountRaw === undefined - ) { - return "Error: Missing required parameters."; - } - const requestedIncreaseAmountFloat = - typeof requestedIncreaseAmountRaw === "number" - ? requestedIncreaseAmountRaw - : Number(String(requestedIncreaseAmountRaw)); - if (!Number.isFinite(requestedIncreaseAmountFloat)) { - return "Error: Invalid requested_increase_amount. Must be a whole number."; - } - if (!Number.isInteger(requestedIncreaseAmountFloat)) { - return "Error: Invalid requested_increase_amount. Must be a whole number of dollars."; - } - const requestedIncreaseAmount = requestedIncreaseAmountFloat; - if (requestedIncreaseAmount <= 0) { - return "Error: Requested increase amount must be positive."; - } - const ccAccountsTable = state.db.credit_card_accounts; - if (!(creditCardAccountId in ccAccountsTable.data)) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - const ccAccount = ccAccountsTable.data[creditCardAccountId]; - if (!ccAccount) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - if (ccAccount.user_id !== userId) { - return `Error: Credit card account '${creditCardAccountId}' does not belong to user '${userId}'.`; - } - const requestId = generateCreditLimitIncreaseRequestId( - creditCardAccountId, - userId, - requestedIncreaseAmount - ); - const today = getTodayStr(); - const requestRecord = { - request_id: requestId, - credit_card_account_id: creditCardAccountId, - user_id: userId, - requested_increase_amount: requestedIncreaseAmount, - submitted_at: today, - status: "PENDING", - }; - const success = addToDb({ - db: state.db, - dbName: "credit_limit_increase_requests", - recordId: requestId, - record: requestRecord, - }); - if (!success) { - return "Error: A similar request may already exist."; - } - return ( - `Credit limit increase request submitted successfully.\n\n` + - `Executed: submit_credit_limit_increase_request_7392\n` + - ` - Request ID: ${requestId}\n` + - ` - Account: ${creditCardAccountId}\n` + - ` - Requested Increase: $${formatUsd(requestedIncreaseAmount)}\n` + - ` - Status: PENDING` - ); -} - -const submitCreditLimitIncreaseRequestTool: DiscoverableTool = { - name: "submit_credit_limit_increase_request_7392", - description: - "Submit a credit limit increase request for a customer's credit card.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The credit card account ID to request increase for", - }, - { - name: "user_id", - type: "string", - optional: false, - description: "The customer's unique identifier in the system", - }, - { - name: "requested_increase_amount", - type: "integer", - optional: false, - description: - "The dollar amount by which to increase the credit limit (e.g., 2500 for $2,500)", - }, - ] satisfies readonly ToolParameter[], - handler: submitCreditLimitIncreaseRequest, -}; - -function getCreditLimitIncreaseHistory( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - if (!creditCardAccountId) { - return "Error: Missing required parameter: credit_card_account_id"; - } - const cliRequests = queryDb({ - db: state.db, - dbName: "credit_limit_increase_requests", - constraints: { - credit_card_account_id: creditCardAccountId, - }, - }); - const resultParts = [ - "Credit limit increase history retrieved.", - "", - "Executed: get_credit_limit_increase_history_4829", - `Credit limit increase history for account ${creditCardAccountId}:`, - ]; - if (cliRequests.length === 0) { - resultParts.push( - "\nNo credit limit increase requests found for this account." - ); - } else { - resultParts.push(""); - for (const [index, record] of cliRequests.entries()) { - resultParts.push(`${index + 1}. Request ID: ${record.request_id}`); - for (const [field, value] of Object.entries(record)) { - resultParts.push(` ${field}: ${value}`); - } - resultParts.push(""); - } - } - return resultParts.join("\n"); -} - -const getCreditLimitIncreaseHistoryTool: DiscoverableTool = { - name: "get_credit_limit_increase_history_4829", - mutatesState: false, - description: - "Retrieve the credit limit increase request history for a specific credit card account. Returns all previous CLI requests including dates, amounts, and statuses.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The credit card account ID to check for CLI history", - }, - ] satisfies readonly ToolParameter[], - handler: getCreditLimitIncreaseHistory, -}; - -function getPaymentHistory( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - const monthsRaw = kwargs.months; - if (!creditCardAccountId || monthsRaw === null || monthsRaw === undefined) { - return "Error: Missing required parameters (credit_card_account_id, months)."; - } - const months = parsePythonInt(monthsRaw); - if (months === undefined) { - return "Error: Invalid months value. Must be a positive integer."; - } - if (months <= 0) { - return "Error: months must be a positive integer."; - } - const paymentTable = state.db.payment_history; - const payments: Record[] = []; - for (const paymentData of Object.values(paymentTable.data)) { - if (paymentData.credit_card_account_id === creditCardAccountId) { - payments.push(paymentData); - } - } - if (payments.length === 0) { - return `No payment history found for account '${creditCardAccountId}'.`; - } - payments.sort((a, b) => { - const dateA = String(a.payment_date ?? ""); - const dateB = String(b.payment_date ?? ""); - return dateB.localeCompare(dateA); - }); - const limitedPayments = payments.slice(0, months); - let consecutiveOnTime = 0; - for (const payment of limitedPayments) { - if (payment.status === "ON_TIME") { - consecutiveOnTime += 1; - } else { - break; - } - } - const resultParts = [ - `Payment history for account '${creditCardAccountId}' (last ${months} months):`, - `Consecutive on-time payments: ${consecutiveOnTime}`, - ]; - for (const payment of limitedPayments) { - resultParts.push( - `\n - Payment Date: ${payment.payment_date}\n` + - ` Amount: ${payment.amount}\n` + - ` Status: ${payment.status}` - ); - } - return resultParts.join("\n"); -} - -function parsePythonInt(value: unknown): number | undefined { - if (typeof value === "number") { - return Number.isFinite(value) ? Math.trunc(value) : undefined; - } - if (typeof value === "boolean") { - return value ? 1 : 0; - } - if (typeof value !== "string" || !/^[+-]?\d+$/u.test(value.trim())) { - return undefined; - } - const parsed = Number(value); - return Number.isSafeInteger(parsed) ? parsed : undefined; -} - -const getPaymentHistoryTool: DiscoverableTool = { - name: "get_payment_history_6183", - mutatesState: false, - description: "Retrieve payment history for a credit card account.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The credit card account ID to check payment history for", - }, - { - name: "months", - type: "integer", - optional: false, - description: "Number of months of payment history to retrieve", - }, - ] satisfies readonly ToolParameter[], - handler: getPaymentHistory, -}; - -function approveCreditLimitIncrease( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - const userId = String(kwargs.user_id ?? ""); - const newCreditLimitRaw = kwargs.new_credit_limit; - if ( - !creditCardAccountId || - !userId || - newCreditLimitRaw === null || - newCreditLimitRaw === undefined - ) { - return "Error: Missing required parameters."; - } - let newCreditLimit: number; - try { - newCreditLimit = - typeof newCreditLimitRaw === "number" - ? newCreditLimitRaw - : Number.parseInt(String(newCreditLimitRaw), 10); - } catch { - return "Error: Invalid new credit limit."; - } - const ccAccountsTable = state.db.credit_card_accounts; - if (!(creditCardAccountId in ccAccountsTable.data)) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - const ccAccount = ccAccountsTable.data[creditCardAccountId]; - if (!ccAccount) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - if (ccAccount.user_id !== userId) { - return `Error: Credit card account '${creditCardAccountId}' does not belong to user '${userId}'.`; - } - let ineligible = false; - if (state.db.transaction_disputes && state.db.transaction_disputes.data) { - for (const dispute of Object.values(state.db.transaction_disputes.data)) { - if ( - dispute.user_id === userId && - ["SUBMITTED", "UNDER_REVIEW", "PENDING"].includes( - String(dispute.status) - ) - ) { - ineligible = true; - break; - } - } - } - if ( - !ineligible && - state.db.credit_card_orders && - state.db.credit_card_orders.data - ) { - for (const order of Object.values(state.db.credit_card_orders.data)) { - if ( - order.credit_card_account_id === creditCardAccountId && - ["PENDING", "PROCESSING", "SHIPPED"].includes(String(order.status)) - ) { - ineligible = true; - break; - } - } - } - if (!ineligible) { - const accountStatus = String(ccAccount.account_status ?? "").toUpperCase(); - if ( - ["PAST_DUE", "DELINQUENT", "COLLECTIONS", "CLOSED"].includes( - accountStatus - ) - ) { - ineligible = true; - } - } - if (ineligible) { - return "Error: Credit limit increase request cannot be approved at this time."; - } - const currentLimitStr = String(ccAccount.credit_limit ?? "$0.00"); - const currentLimit = parseBalance(currentLimitStr); - const newLimit = Number.parseFloat(String(newCreditLimit)); - ccAccount.credit_limit = `$${newLimit.toFixed(2)}`; - const today = getTodayStr(); - const requestId = generateCreditLimitIncreaseRequestId( - creditCardAccountId, - userId, - newLimit - currentLimit - ); - const approvalRecord = { - request_id: requestId, - credit_card_account_id: creditCardAccountId, - user_id: userId, - previous_limit: `$${currentLimit.toFixed(2)}`, - new_limit: `$${newLimit.toFixed(2)}`, - increase_amount: `$${(newLimit - currentLimit).toFixed(2)}`, - decision_date: today, - status: "APPROVED", - }; - addToDb({ - db: state.db, - dbName: "credit_limit_increase_requests", - recordId: requestId, - record: approvalRecord, - }); - return ( - `Credit limit increase approved!\n` + - ` - Account: ${creditCardAccountId}\n` + - ` - Previous Limit: $${currentLimit.toFixed(2)}\n` + - ` - New Limit: $${newLimit.toFixed(2)}\n` + - ` - Increase: $${(newLimit - currentLimit).toFixed(2)}\n` + - ` - Effective Date: ${today}\n` + - `The customer will receive a confirmation email.` - ); -} - -const approveCreditLimitIncreaseTool: DiscoverableTool = { - name: "approve_credit_limit_increase_5847", - description: - "Approve and apply a credit limit increase for a customer's credit card.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The credit card account ID", - }, - { - name: "user_id", - type: "string", - optional: false, - description: "The customer's unique identifier in the system", - }, - { - name: "new_credit_limit", - type: "integer", - optional: false, - description: - "The new total credit limit in dollars (e.g., 7500 for $7,500)", - }, - ] satisfies readonly ToolParameter[], - handler: approveCreditLimitIncrease, -}; - -function denyCreditLimitIncrease( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - const userId = String(kwargs.user_id ?? ""); - const denialReason = String(kwargs.denial_reason ?? ""); - if (!creditCardAccountId || !userId || !denialReason) { - return "Error: Missing required parameters."; - } - const validReasons = [ - "insufficient_account_age", - "cooldown_period_active", - "pending_disputes", - "pending_replacement_card", - "past_due_balance", - "high_utilization", - "insufficient_payment_history", - "requested_amount_exceeds_limit", - "other", - ]; - if (!validReasons.includes(denialReason)) { - return `Error: Invalid denial_reason. Must be one of: ${validReasons.join(", ")}`; - } - const ccAccountsTable = state.db.credit_card_accounts; - if (!(creditCardAccountId in ccAccountsTable.data)) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - const today = getTodayStr(); - const requestId = generateCreditLimitIncreaseRequestId( - creditCardAccountId, - userId, - 0 - ); - const denialRecord = { - request_id: requestId, - credit_card_account_id: creditCardAccountId, - user_id: userId, - denial_reason: denialReason, - decision_date: today, - status: "DENIED", - }; - addToDb({ - db: state.db, - dbName: "credit_limit_increase_requests", - recordId: requestId, - record: denialRecord, - }); - return ( - `Credit limit increase request denied.\n` + - ` - Account: ${creditCardAccountId}\n` + - ` - Denial Reason: ${denialReason}\n` + - ` - Date: ${today}\n` + - `The customer will receive a notification explaining the denial.` - ); -} - -const denyCreditLimitIncreaseTool: DiscoverableTool = { - name: "deny_credit_limit_increase_5848", - description: - "Deny a credit limit increase request for a customer's credit card.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The credit card account ID", - }, - { - name: "user_id", - type: "string", - optional: false, - description: "The customer's unique identifier in the system", - }, - { - name: "denial_reason", - type: "string", - optional: false, - description: - "The reason for denying the request. Must be one of: 'insufficient_account_age', 'cooldown_period_active', 'pending_disputes', 'pending_replacement_card', 'past_due_balance', 'high_utilization', 'insufficient_payment_history', 'requested_amount_exceeds_limit', 'other'", - }, - ] satisfies readonly ToolParameter[], - handler: denyCreditLimitIncrease, -}; - -export function registerCreditLimitAndPaymentTools(): void { - registerDiscoverableAgentTool(payCreditCardFromCheckingTool); - registerDiscoverableAgentTool(submitCreditLimitIncreaseRequestTool); - registerDiscoverableAgentTool(getCreditLimitIncreaseHistoryTool); - registerDiscoverableAgentTool(getPaymentHistoryTool); - registerDiscoverableAgentTool(approveCreditLimitIncreaseTool); - registerDiscoverableAgentTool(denyCreditLimitIncreaseTool); -} diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-credit.test.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-credit.test.ts deleted file mode 100644 index cefd7249..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-credit.test.ts +++ /dev/null @@ -1,775 +0,0 @@ -import { describe, it, expect, beforeEach } from "bun:test"; -import assert from "node:assert"; - -import { makeEmptyBankingData } from "../environment"; -import type { BankingData } from "../types"; -import { registerCreditTools } from "./handlers-credit"; -import type { BankingEnvState } from "./registry"; -import { makeBankingEnvState, DISCOVERABLE_AGENT_TOOLS } from "./registry"; - -function makeTestDb(): BankingData { - return { - ...makeEmptyBankingData(), - users: { - data: { - user_123: { - user_id: "user_123", - name: "John Doe", - email: "john@example.com", - }, - }, - }, - accounts: { - data: { - checking_001: { - account_id: "checking_001", - user_id: "user_123", - class: "checking", - balance: "5000.00", - current_holdings: "5000.00", - }, - }, - }, - credit_card_accounts: { - data: { - cc_001: { - account_id: "cc_001", - user_id: "user_123", - credit_limit: "$5000.00", - current_balance: "$1500.00", - account_status: "ACTIVE", - status: "ACTIVE", - }, - }, - }, - payment_history: { - data: { - payment_001: { - payment_id: "payment_001", - credit_card_account_id: "cc_001", - payment_date: "11/10/2025", - amount: "$500.00", - status: "ON_TIME", - }, - payment_002: { - payment_id: "payment_002", - credit_card_account_id: "cc_001", - payment_date: "11/03/2025", - amount: "$500.00", - status: "ON_TIME", - }, - }, - }, - }; -} -describe("handlers-credit", () => { - let state: BankingEnvState; - beforeEach(() => { - state = makeBankingEnvState(makeTestDb()); - registerCreditTools(); - }); - describe("order_replacement_credit_card_7291", () => { - it("should order replacement card successfully", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "order_replacement_credit_card_7291" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - shipping_address: "123 Main St, Boston, MA 02101", - reason: "lost", - expedited_shipping: false, - }); - expect(result).toContain("Order ID:"); - expect(result).toContain("Card Account: cc_001"); - expect(result).toContain("Reason: Lost"); - expect(result).toContain("Shipping Method: Standard"); - expect(result).toContain("Expected Delivery: 7-10 business days"); - const ordersCount = Object.keys(state.db.credit_card_orders.data).length; - expect(ordersCount).toBe(1); - const order = Object.values(state.db.credit_card_orders.data)[0]; - assert(order); - expect(order.status).toBe("ORDERED"); - expect(order.reason).toBe("lost"); - }); - it("should close old card on replacement", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "order_replacement_credit_card_7291" - ); - if (!tool) { - throw new Error("Tool not found"); - } - tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - shipping_address: "123 Main St", - reason: "stolen", - }); - const ccAccount = state.db.credit_card_accounts.data.cc_001; - assert(ccAccount); - expect(ccAccount.status).toBe("CLOSED"); - expect(ccAccount.closed_date).toBe("11/14/2025"); - }); - it("should reject missing parameters", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "order_replacement_credit_card_7291" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - }); - expect(result).toContain("Error: Missing required parameters"); - }); - it("should reject invalid reason", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "order_replacement_credit_card_7291" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - shipping_address: "123 Main St", - reason: "invalid_reason", - }); - expect(result).toContain("Error: Invalid reason"); - }); - it("should reject nonexistent card account", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "order_replacement_credit_card_7291" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "nonexistent", - user_id: "user_123", - shipping_address: "123 Main St", - reason: "lost", - }); - expect(result).toContain( - "Error: Credit card account 'nonexistent' not found" - ); - }); - }); - describe("get_user_dispute_history_7291", () => { - it("should return no disputes when none exist", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "get_user_dispute_history_7291" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - user_id: "user_123", - }); - expect(result).toContain( - "User transaction dispute history retrieved successfully" - ); - expect(result).toContain("No transaction disputes found for this user"); - }); - it("should list disputes if they exist", () => { - state.db.transaction_disputes.data.dispute_001 = { - dispute_id: "dispute_001", - user_id: "user_123", - status: "SUBMITTED", - }; - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "get_user_dispute_history_7291" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - user_id: "user_123", - }); - expect(result).toContain("1. Dispute ID: dispute_001"); - }); - }); - describe("get_pending_replacement_orders_5765", () => { - it("should return no orders when none exist", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "get_pending_replacement_orders_5765" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - }); - expect(result).toContain("Pending replacement orders check completed"); - expect(result).toContain("No pending replacement orders found"); - }); - }); - describe("log_credit_card_closure_reason_4521", () => { - it("should log closure reason successfully", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "log_credit_card_closure_reason_4521" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - closure_reason: "annual_fee", - }); - expect(result).toContain("Closure reason logged successfully"); - expect(result).toContain("'annual_fee'"); - const reasonsCount = Object.keys( - state.db.credit_card_closure_reasons.data - ).length; - expect(reasonsCount).toBe(1); - const reason = Object.values( - state.db.credit_card_closure_reasons.data - )[0]; - assert(reason); - expect(reason.closure_reason).toBe("annual_fee"); - expect(reason.status).toBe("LOGGED"); - }); - it("should reject invalid closure reason", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "log_credit_card_closure_reason_4521" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - closure_reason: "invalid_reason", - }); - expect(result).toContain("Error: Invalid closure_reason"); - }); - }); - describe("close_credit_card_account_7834", () => { - it("should close credit card account successfully", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "close_credit_card_account_7834" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - }); - expect(result).toContain("Credit card account closed successfully"); - expect(result).toContain("cc_001"); - const ccAccount = state.db.credit_card_accounts.data.cc_001; - assert(ccAccount); - expect(ccAccount.status).toBe("CLOSED"); - expect(ccAccount.closed_date).toBe("11/14/2025"); - expect(ccAccount.closed_by).toBe("user_123"); - }); - it("should reject nonexistent account", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "close_credit_card_account_7834" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "nonexistent", - user_id: "user_123", - }); - expect(result).toContain( - "Error: Credit card account 'nonexistent' not found" - ); - }); - }); - describe("pay_credit_card_from_checking_9182", () => { - it("should process payment successfully", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "pay_credit_card_from_checking_9182" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - user_id: "user_123", - checking_account_id: "checking_001", - credit_card_account_id: "cc_001", - amount: 500, - }); - expect(result).toContain("Payment processed successfully"); - expect(result).toContain("$500.00"); - expect(result).toContain("$4500.00"); - expect(result).toContain("$1000.00"); - const checkingAccount = state.db.accounts.data.checking_001; - assert(checkingAccount); - expect(checkingAccount.current_holdings).toBe("4500.00"); - const ccAccount = state.db.credit_card_accounts.data.cc_001; - assert(ccAccount); - expect(ccAccount.current_balance).toBe("$1000.00"); - }); - it("should reject a non-numeric amount", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "pay_credit_card_from_checking_9182" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - user_id: "user_123", - checking_account_id: "checking_001", - credit_card_account_id: "cc_001", - amount: "abc", - }); - expect(result).toBe( - "Error: Invalid payment amount. Must be a positive number." - ); - }); - it("should reject insufficient funds", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "pay_credit_card_from_checking_9182" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - user_id: "user_123", - checking_account_id: "checking_001", - credit_card_account_id: "cc_001", - amount: 10000, - }); - expect(result).toContain("Error: Insufficient funds"); - }); - it("should reject payment exceeding card balance", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "pay_credit_card_from_checking_9182" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - user_id: "user_123", - checking_account_id: "checking_001", - credit_card_account_id: "cc_001", - amount: 2000, - }); - expect(result).toContain("Error: Payment amount"); - expect(result).toContain("exceeds credit card balance"); - }); - it("should reject negative amount", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "pay_credit_card_from_checking_9182" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - user_id: "user_123", - checking_account_id: "checking_001", - credit_card_account_id: "cc_001", - amount: -100, - }); - expect(result).toContain( - "Error: Payment amount must be a positive number" - ); - }); - }); - describe("submit_credit_limit_increase_request_7392", () => { - it("should submit CLI request successfully", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "submit_credit_limit_increase_request_7392" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - requested_increase_amount: 2500, - }); - expect(result).toContain( - "Credit limit increase request submitted successfully" - ); - expect(result).toContain("PENDING"); - expect(result).toContain("$2,500"); - const requestsCount = Object.keys( - state.db.credit_limit_increase_requests.data - ).length; - expect(requestsCount).toBe(1); - const request = Object.values( - state.db.credit_limit_increase_requests.data - )[0]; - assert(request); - expect(request.status).toBe("PENDING"); - expect(request.requested_increase_amount).toBe(2500); - }); - it("should reject nonexistent card", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "submit_credit_limit_increase_request_7392" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "nonexistent", - user_id: "user_123", - requested_increase_amount: 2500, - }); - expect(result).toContain( - "Error: Credit card account 'nonexistent' not found" - ); - }); - it("should reject zero increase amount", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "submit_credit_limit_increase_request_7392" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - requested_increase_amount: 0, - }); - expect(result).toContain( - "Error: Requested increase amount must be positive" - ); - }); - it("should reject fractional increase amounts without truncating", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "submit_credit_limit_increase_request_7392" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - requested_increase_amount: 2500.5, - }); - expect(result).toBe( - "Error: Invalid requested_increase_amount. Must be a whole number of dollars." - ); - expect( - Object.keys(state.db.credit_limit_increase_requests.data) - ).toHaveLength(0); - }); - }); - describe("get_credit_limit_increase_history_4829", () => { - it("should return empty history when no requests", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "get_credit_limit_increase_history_4829" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - }); - expect(result).toContain("Credit limit increase history retrieved"); - expect(result).toContain("No credit limit increase requests found"); - }); - }); - describe("get_payment_history_6183", () => { - it("should retrieve payment history", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get("get_payment_history_6183"); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - months: 3, - }); - expect(result).toContain("Payment history for account 'cc_001'"); - expect(result).toContain("Consecutive on-time payments: 2"); - expect(result).toContain("Payment Date:"); - }); - it("should return no history for nonexistent account", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get("get_payment_history_6183"); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "nonexistent", - months: 3, - }); - expect(result).toContain("No payment history found"); - }); - it("should reject non-numeric months", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get("get_payment_history_6183"); - if (!tool) { - throw new Error("Tool not found"); - } - expect( - tool.handler(state, { - credit_card_account_id: "cc_001", - months: "6.5", - }) - ).toBe("Error: Invalid months value. Must be a positive integer."); - }); - it("should truncate numeric float months like Python int()", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get("get_payment_history_6183"); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - months: 1.9, - }); - expect(result).toContain("(last 1 months):"); - expect(result.match(/Payment Date:/g)).toHaveLength(1); - }); - }); - describe("approve_credit_limit_increase_5847", () => { - it("should approve CLI and update limit", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "approve_credit_limit_increase_5847" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - new_credit_limit: 8000, - }); - expect(result).toContain("Credit limit increase approved!"); - expect(result).toContain("$8000.00"); - const ccAccount = state.db.credit_card_accounts.data.cc_001; - assert(ccAccount); - expect(ccAccount.credit_limit).toBe("$8000.00"); - }); - it("should reject if pending disputes exist", () => { - state.db.transaction_disputes.data.dispute_001 = { - dispute_id: "dispute_001", - user_id: "user_123", - status: "PENDING", - }; - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "approve_credit_limit_increase_5847" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - new_credit_limit: 8000, - }); - expect(result).toContain( - "Error: Credit limit increase request cannot be approved" - ); - }); - it("should reject if account is CLOSED", () => { - const ccAccount = state.db.credit_card_accounts.data.cc_001; - assert(ccAccount); - ccAccount.account_status = "CLOSED"; - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "approve_credit_limit_increase_5847" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - new_credit_limit: 8000, - }); - expect(result).toContain( - "Error: Credit limit increase request cannot be approved" - ); - }); - }); - describe("deny_credit_limit_increase_5848", () => { - it("should deny CLI with reason", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "deny_credit_limit_increase_5848" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - denial_reason: "insufficient_account_age", - }); - expect(result).toContain("Credit limit increase request denied"); - expect(result).toContain("insufficient_account_age"); - const requestsCount = Object.keys( - state.db.credit_limit_increase_requests.data - ).length; - expect(requestsCount).toBe(1); - const request = Object.values( - state.db.credit_limit_increase_requests.data - )[0]; - assert(request); - expect(request.status).toBe("DENIED"); - expect(request.denial_reason).toBe("insufficient_account_age"); - }); - it("should reject invalid denial reason", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "deny_credit_limit_increase_5848" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - denial_reason: "invalid_reason", - }); - expect(result).toContain("Error: Invalid denial_reason"); - }); - }); - describe("apply_statement_credit_8472", () => { - it("should apply statement credit successfully", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get("apply_statement_credit_8472"); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - user_id: "user_123", - credit_card_account_id: "cc_001", - amount: 50, - reason: "goodwill_adjustment", - }); - expect(result).toContain("Statement credit applied successfully"); - expect(result).toContain("$50.00"); - expect(result).toContain("Goodwill Adjustment"); - const historyCount = Object.keys( - state.db.credit_card_transaction_history.data - ).length; - expect(historyCount).toBe(1); - const credit = Object.values( - state.db.credit_card_transaction_history.data - )[0]; - assert(credit); - expect(credit.credit_reason).toBe("goodwill_adjustment"); - expect(credit.status).toBe("COMPLETED"); - }); - it("should reject zero amount", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get("apply_statement_credit_8472"); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - user_id: "user_123", - credit_card_account_id: "cc_001", - amount: 0, - reason: "goodwill_adjustment", - }); - expect(result).toContain("Error: Credit amount must be positive"); - }); - it("should reject nonexistent card", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get("apply_statement_credit_8472"); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - user_id: "user_123", - credit_card_account_id: "nonexistent", - amount: 50, - reason: "goodwill_adjustment", - }); - expect(result).toContain( - "Error: Credit card account 'nonexistent' not found" - ); - }); - }); - describe("apply_credit_card_account_flag_6147", () => { - it("should apply account flag successfully", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "apply_credit_card_account_flag_6147" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - flag_type: "annual_fee_waived", - expiration_date: "12/31/2025", - reason: "retention_offer", - }); - expect(result).toContain("Account flag applied successfully"); - expect(result).toContain("Annual Fee Waived"); - expect(result).toContain("Retention Offer"); - const flagsCount = Object.keys( - state.db.credit_card_account_flags.data - ).length; - expect(flagsCount).toBe(1); - const flag = Object.values(state.db.credit_card_account_flags.data)[0]; - assert(flag); - expect(flag.flag_type).toBe("annual_fee_waived"); - expect(flag.reason).toBe("retention_offer"); - expect(flag.status).toBe("ACTIVE"); - }); - it("should reject invalid flag type", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "apply_credit_card_account_flag_6147" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - flag_type: "invalid_flag", - expiration_date: "12/31/2025", - reason: "retention_offer", - }); - expect(result).toContain("Error: Invalid flag_type"); - }); - it("should reject invalid reason", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "apply_credit_card_account_flag_6147" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - user_id: "user_123", - flag_type: "annual_fee_waived", - expiration_date: "12/31/2025", - reason: "invalid_reason", - }); - expect(result).toContain("Error: Invalid reason"); - }); - }); - describe("get_closure_reason_history_8293", () => { - it("should return empty closure reason history", () => { - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "get_closure_reason_history_8293" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - }); - expect(result).toContain("Closure reason history retrieved successfully"); - expect(result).toContain("No closure reason records found"); - }); - it("should list closure reasons if they exist", () => { - state.db.credit_card_closure_reasons.data.reason_001 = { - record_id: "reason_001", - credit_card_account_id: "cc_001", - closure_reason: "annual_fee", - status: "LOGGED", - }; - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "get_closure_reason_history_8293" - ); - if (!tool) { - throw new Error("Tool not found"); - } - const result = tool.handler(state, { - credit_card_account_id: "cc_001", - }); - expect(result).toContain("1. Record ID: reason_001"); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-credit.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-credit.ts deleted file mode 100644 index 3d00a5a2..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-credit.ts +++ /dev/null @@ -1,712 +0,0 @@ -import { queryDb, addToDb, updateRecordInDb } from "./db-query"; -import { getFieldAsRecord } from "./field-access"; -import { registerCreditLimitAndPaymentTools } from "./handlers-credit-2"; -import { getTodayStr } from "./helpers"; -import { - generateCreditCardOrderId, - generateClosureReasonId, - generateAccountFlagId, - generateTransactionId, -} from "./ids"; -import type { - BankingEnvState, - DiscoverableTool, - ToolParameter, -} from "./registry"; -import { registerDiscoverableAgentTool } from "./registry"; - -function titleCase(str: string): string { - return str - .split("_") - .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(" "); -} - -function orderReplacementCreditCard( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - const userId = String(kwargs.user_id ?? ""); - const shippingAddress = String(kwargs.shipping_address ?? ""); - const reason = String(kwargs.reason ?? ""); - const expeditedShipping = kwargs.expedited_shipping === true; - if (!creditCardAccountId || !userId || !shippingAddress || !reason) { - return "Error: Missing required parameters (credit_card_account_id, user_id, shipping_address, reason)."; - } - const validReasons = [ - "fraud_suspected", - "lost", - "stolen", - "damaged", - "expired", - "other", - ]; - if (!validReasons.includes(reason)) { - return `Error: Invalid reason. Must be one of: ${validReasons}`; - } - const ccAccounts = queryDb({ - db: state.db, - dbName: "credit_card_accounts", - constraints: { - account_id: creditCardAccountId, - }, - }); - if (ccAccounts.length === 0) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - const orderId = generateCreditCardOrderId( - creditCardAccountId, - userId, - reason - ); - const today = getTodayStr(); - const orderRecord = { - order_id: orderId, - credit_card_account_id: creditCardAccountId, - user_id: userId, - shipping_address: shippingAddress, - reason, - expedited_shipping: expeditedShipping, - order_date: today, - status: "ORDERED", - old_card_cancelled: true, - }; - const success = addToDb({ - db: state.db, - dbName: "credit_card_orders", - recordId: orderId, - record: orderRecord, - }); - if (!success) { - return "Error: Order may have already been placed for this card replacement."; - } - const ccTable = state.db.credit_card_accounts; - if (creditCardAccountId in ccTable.data) { - const record = getFieldAsRecord(ccTable.data, creditCardAccountId); - if (record) { - record.status = "CLOSED"; - record.closed_date = today; - } - } - const shippingMethod = expeditedShipping ? "Expedited" : "Standard"; - const expectedDelivery = expeditedShipping - ? "2-3 business days" - : "7-10 business days"; - const resultParts = [ - `Order ID: ${orderId}`, - `Card Account: ${creditCardAccountId}`, - `Reason: ${titleCase(reason)}`, - `Shipping Address: ${shippingAddress}`, - `Shipping Method: ${shippingMethod}`, - `Expected Delivery: ${expectedDelivery}`, - "", - "The old card has been cancelled for security. The new card will have the same account number but a new card number and CVV.", - ]; - return resultParts.join("\n"); -} - -const orderReplacementCreditCardTool: DiscoverableTool = { - name: "order_replacement_credit_card_7291", - description: - "Order a replacement credit card for a customer. The old card will be automatically cancelled when the replacement is ordered.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The credit card account ID for the card being replaced", - }, - { - name: "user_id", - type: "string", - optional: false, - description: "The user's unique identifier in the system", - }, - { - name: "shipping_address", - type: "string", - optional: false, - description: "Full shipping address where the new card should be sent", - }, - { - name: "reason", - type: "string", - optional: false, - description: - "Reason for replacement. Must be one of: 'fraud_suspected', 'lost', 'stolen', 'damaged', 'expired', 'other'", - }, - { - name: "expedited_shipping", - type: "boolean", - optional: true, - description: - "Whether to use expedited shipping (2-3 business days instead of 7-10)", - }, - ] satisfies readonly ToolParameter[], - handler: orderReplacementCreditCard, -}; - -function getUserDisputeHistory( - state: BankingEnvState, - kwargs: Record -): string { - const userId = String(kwargs.user_id ?? ""); - if (!userId) { - return "Error: Missing required parameter: user_id"; - } - const disputes = queryDb({ - db: state.db, - dbName: "transaction_disputes", - constraints: { user_id: userId }, - }); - const resultParts = [ - "User transaction dispute history retrieved successfully.", - "", - "Executed: get_user_dispute_history_7291", - `Transaction dispute history for user ${userId}:`, - ]; - if (disputes.length === 0) { - resultParts.push("\nNo transaction disputes found for this user."); - } else { - resultParts.push(""); - for (let i = 0; i < disputes.length; i++) { - const record = disputes[i] as Record; - resultParts.push(`${i + 1}. Dispute ID: ${record.dispute_id}`); - for (const [field, value] of Object.entries(record)) { - resultParts.push(` ${field}: ${value}`); - } - resultParts.push(""); - } - } - return resultParts.join("\n"); -} - -const getUserDisputeHistoryTool: DiscoverableTool = { - name: "get_user_dispute_history_7291", - mutatesState: false, - description: - "Retrieve a user's credit card transaction dispute history from the transaction_disputes table. Returns all credit card transaction disputes filed by the user, including dispute IDs, transaction IDs, dispute reasons, statuses, and submission dates.", - params: [ - { - name: "user_id", - type: "string", - optional: false, - description: "The user's unique identifier in the system", - }, - ] satisfies readonly ToolParameter[], - handler: getUserDisputeHistory, -}; - -function getPendingReplacementOrders( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - if (!creditCardAccountId) { - return "Error: Missing required parameter: credit_card_account_id"; - } - const orders = queryDb({ - db: state.db, - dbName: "credit_card_orders", - constraints: { - credit_card_account_id: creditCardAccountId, - }, - }); - const resultParts = [ - "Pending replacement orders check completed.", - "", - "Executed: get_pending_replacement_orders_5765", - `Replacement orders for credit card account ${creditCardAccountId}:`, - ]; - if (orders.length === 0) { - resultParts.push( - "\nNo pending replacement orders found for this credit card account." - ); - } else { - resultParts.push(""); - for (let i = 0; i < orders.length; i++) { - const record = orders[i] as Record; - resultParts.push(`${i + 1}. Order ID: ${record.order_id}`); - for (const [field, value] of Object.entries(record)) { - resultParts.push(` ${field}: ${value}`); - } - resultParts.push(""); - } - } - return resultParts.join("\n"); -} - -const getPendingReplacementOrdersTool: DiscoverableTool = { - name: "get_pending_replacement_orders_5765", - mutatesState: false, - description: - "Check if a credit card account has any pending replacement card orders.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: - "The credit card account ID to check for pending replacement orders", - }, - ] satisfies readonly ToolParameter[], - handler: getPendingReplacementOrders, -}; - -function logCreditCardClosureReason( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - const userId = String(kwargs.user_id ?? ""); - const closureReason = String(kwargs.closure_reason ?? ""); - if (!creditCardAccountId || !userId || !closureReason) { - return "Error: Missing required parameters."; - } - const validReasons = [ - "annual_fee", - "not_using_card", - "found_better_card", - "unhappy_with_rewards", - "simplifying_finances", - "negative_experience", - "other", - ]; - if (!validReasons.includes(closureReason)) { - return `Error: Invalid closure_reason. Must be one of: ${validReasons}`; - } - const recordId = generateClosureReasonId(creditCardAccountId, userId); - const closureRecord = { - record_id: recordId, - credit_card_account_id: creditCardAccountId, - user_id: userId, - closure_reason: closureReason, - logged_at: getTodayStr(), - status: "LOGGED", - }; - addToDb({ - db: state.db, - dbName: "credit_card_closure_reasons", - recordId, - record: closureRecord, - }); - return ( - `Closure reason logged successfully.\n\n` + - `Executed: log_credit_card_closure_reason_4521\n` + - `Arguments: ${JSON.stringify({ credit_card_account_id: creditCardAccountId, user_id: userId, closure_reason: closureReason }, null, 2)}\n` + - `Closure reason '${closureReason}' logged for account ${creditCardAccountId}.` - ); -} - -const logCreditCardClosureReasonTool: DiscoverableTool = { - name: "log_credit_card_closure_reason_4521", - description: - "Log the reason why a customer wants to close their credit card account.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The credit card account ID the customer wants to close", - }, - { - name: "user_id", - type: "string", - optional: false, - description: "The user's unique identifier in the system", - }, - { - name: "closure_reason", - type: "string", - optional: false, - description: - "Reason for closure. Must be one of: 'annual_fee', 'not_using_card', 'found_better_card', 'unhappy_with_rewards', 'simplifying_finances', 'negative_experience', 'other'", - }, - ] satisfies readonly ToolParameter[], - handler: logCreditCardClosureReason, -}; - -function getClosureReasonHistory( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - if (!creditCardAccountId) { - return "Error: Missing required parameter: credit_card_account_id"; - } - const closureReasons = queryDb({ - db: state.db, - dbName: "credit_card_closure_reasons", - constraints: { - credit_card_account_id: creditCardAccountId, - }, - }); - const resultParts = [ - "Closure reason history retrieved successfully.", - "", - "Executed: get_closure_reason_history_8293", - `Closure reason history for credit card account ${creditCardAccountId}:`, - ]; - if (closureReasons.length === 0) { - resultParts.push( - "\nNo closure reason records found for this credit card account." - ); - } else { - resultParts.push(""); - for (let i = 0; i < closureReasons.length; i++) { - const record = closureReasons[i] as Record; - resultParts.push(`${i + 1}. Record ID: ${record.record_id}`); - for (const [field, value] of Object.entries(record)) { - resultParts.push(` ${field}: ${value}`); - } - resultParts.push(""); - } - } - return resultParts.join("\n"); -} - -const getClosureReasonHistoryTool: DiscoverableTool = { - name: "get_closure_reason_history_8293", - mutatesState: false, - description: - "Retrieve the closure reason history for a specific credit card account.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: - "The credit card account ID to check for previous closure attempts", - }, - ] satisfies readonly ToolParameter[], - handler: getClosureReasonHistory, -}; - -function applyStatementCredit( - state: BankingEnvState, - kwargs: Record -): string { - const userId = String(kwargs.user_id ?? ""); - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - const amountRaw = kwargs.amount; - const reason = String(kwargs.reason ?? ""); - if ( - !userId || - !creditCardAccountId || - amountRaw === null || - amountRaw === undefined || - !reason - ) { - return "Error: Missing required parameters (user_id, credit_card_account_id, amount, reason)."; - } - const amount = - typeof amountRaw === "number" - ? amountRaw - : Number.parseFloat(String(amountRaw)); - if (Number.isNaN(amount)) { - return "Error: Invalid amount."; - } - if (amount <= 0) { - return "Error: Credit amount must be positive."; - } - const validReasons = [ - "goodwill_adjustment", - "promotional_credit", - "annual_fee_reversal", - "late_fee_reversal", - "interest_charge_reversal", - "dispute_resolution", - "price_match", - "retention_offer", - "error_correction", - "other", - ]; - if (!validReasons.includes(reason)) { - return `Error: Invalid reason. Must be one of: ${validReasons}`; - } - const ccAccounts = queryDb({ - db: state.db, - dbName: "credit_card_accounts", - constraints: { - account_id: creditCardAccountId, - }, - }); - if (ccAccounts.length === 0) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - const transactionId = generateTransactionId({ - userId, - creditCardType: "STATEMENT_CREDIT", - merchantName: reason, - amount, - category: "Statement Credit", - }); - const today = getTodayStr(); - const creditRecord = { - transaction_id: transactionId, - user_id: userId, - credit_card_account_id: creditCardAccountId, - credit_card_type: "N/A", - merchant_name: "Rho-Bank Statement Credit", - transaction_amount: `-$${amount.toFixed(2)}`, - transaction_date: today, - category: "Statement Credit", - status: "COMPLETED", - rewards_earned: "0 points", - credit_reason: reason, - }; - const success = addToDb({ - db: state.db, - dbName: "credit_card_transaction_history", - recordId: transactionId, - record: creditRecord, - }); - if (!success) { - return `Error: Failed to apply statement credit. Transaction ID '${transactionId}' may already exist.`; - } - return ( - `Statement credit applied successfully.\n\n` + - `Executed: apply_statement_credit_8472\n` + - ` - Transaction ID: ${transactionId}\n` + - ` - User ID: ${userId}\n` + - ` - Account: ${creditCardAccountId}\n` + - ` - Credit Amount: $${amount.toFixed(2)}\n` + - ` - Reason: ${titleCase(reason)}\n` + - ` - Date: ${today}` - ); -} - -const applyStatementCreditTool: DiscoverableTool = { - name: "apply_statement_credit_8472", - description: "Apply a statement credit to a customer's credit card account.", - params: [ - { - name: "user_id", - type: "string", - optional: false, - description: "The user's unique identifier in the system", - }, - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The credit card account ID to apply the credit to", - }, - { - name: "amount", - type: "number", - optional: false, - description: - "The credit amount in dollars (e.g., 25.00 for a $25 credit)", - }, - { - name: "reason", - type: "string", - optional: false, - description: - "Reason for the statement credit. Must be one of: 'goodwill_adjustment', 'promotional_credit', 'annual_fee_reversal', 'late_fee_reversal', 'interest_charge_reversal', 'dispute_resolution', 'price_match', 'retention_offer', 'error_correction', 'other'", - }, - ] satisfies readonly ToolParameter[], - handler: applyStatementCredit, -}; - -function applyCreditCardAccountFlag( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - const userId = String(kwargs.user_id ?? ""); - const flagType = String(kwargs.flag_type ?? ""); - const expirationDate = String(kwargs.expiration_date ?? ""); - const reason = String(kwargs.reason ?? ""); - if ( - !creditCardAccountId || - !userId || - !flagType || - !expirationDate || - !reason - ) { - return "Error: Missing required parameters (credit_card_account_id, user_id, flag_type, expiration_date, reason)."; - } - const validFlagTypes = [ - "annual_fee_waived", - "promotional_apr", - "rewards_bonus", - "other", - ]; - if (!validFlagTypes.includes(flagType)) { - return `Error: Invalid flag_type. Must be one of: ${validFlagTypes}`; - } - const validReasons = [ - "retention_offer", - "loyalty_benefit", - "promotional", - "error_correction", - "other", - ]; - if (!validReasons.includes(reason)) { - return `Error: Invalid reason. Must be one of: ${validReasons}`; - } - const ccAccounts = queryDb({ - db: state.db, - dbName: "credit_card_accounts", - constraints: { - account_id: creditCardAccountId, - }, - }); - if (ccAccounts.length === 0) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - const flagId = generateAccountFlagId( - creditCardAccountId, - flagType, - expirationDate - ); - const today = getTodayStr(); - const flagRecord = { - flag_id: flagId, - credit_card_account_id: creditCardAccountId, - user_id: userId, - flag_type: flagType, - effective_date: today, - expiration_date: expirationDate, - reason, - applied_at: today, - status: "ACTIVE", - }; - const success = addToDb({ - db: state.db, - dbName: "credit_card_account_flags", - recordId: flagId, - record: flagRecord, - }); - if (!success) { - return `Error: Failed to apply account flag. Flag ID '${flagId}' may already exist.`; - } - const flagTypeDisplay = titleCase(flagType); - const reasonDisplay = titleCase(reason); - return ( - `Account flag applied successfully!\n` + - ` - Flag ID: ${flagId}\n` + - ` - Account: ${creditCardAccountId}\n` + - ` - User ID: ${userId}\n` + - ` - Flag Type: ${flagTypeDisplay}\n` + - ` - Effective Date: ${today}\n` + - ` - Expiration Date: ${expirationDate}\n` + - ` - Reason: ${reasonDisplay}` - ); -} - -const applyCreditCardAccountFlagTool: DiscoverableTool = { - name: "apply_credit_card_account_flag_6147", - description: - "Apply a flag to a customer's credit card account. Flags can include annual fee waivers, promotional APR rates, rewards bonuses, or other account-level modifiers. Each flag has an effective date and expiration date.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The credit card account ID to apply the flag to", - }, - { - name: "user_id", - type: "string", - optional: false, - description: "The user's unique identifier in the system", - }, - { - name: "flag_type", - type: "string", - optional: false, - description: - "Type of flag to apply. Must be one of: 'annual_fee_waived', 'promotional_apr', 'rewards_bonus', 'other'", - }, - { - name: "expiration_date", - type: "string", - optional: false, - description: "Date when the flag expires (MM/DD/YYYY format)", - }, - { - name: "reason", - type: "string", - optional: false, - description: - "Reason for applying this flag. Must be one of: 'retention_offer', 'loyalty_benefit', 'promotional', 'error_correction', 'other'", - }, - ] satisfies readonly ToolParameter[], - handler: applyCreditCardAccountFlag, -}; - -function closeCreditCardAccount( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = String(kwargs.credit_card_account_id ?? ""); - const userId = String(kwargs.user_id ?? ""); - if (!creditCardAccountId || !userId) { - return "Error: Missing required parameters (credit_card_account_id, user_id)."; - } - const ccAccounts = queryDb({ - db: state.db, - dbName: "credit_card_accounts", - constraints: { - account_id: creditCardAccountId, - }, - }); - if (ccAccounts.length === 0) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - const [success] = updateRecordInDb({ - db: state.db, - dbName: "credit_card_accounts", - recordId: creditCardAccountId, - updates: { - status: "CLOSED", - closed_date: getTodayStr(), - closed_by: userId, - }, - }); - if (!success) { - return `Error: Failed to close credit card account '${creditCardAccountId}'.`; - } - return ( - `Credit card account closed successfully.\n\n` + - `Executed: close_credit_card_account_7834\n` + - `Arguments: ${JSON.stringify({ credit_card_account_id: creditCardAccountId, user_id: userId }, null, 2)}\n` + - `Account ${creditCardAccountId} has been closed.` - ); -} - -const closeCreditCardAccountTool: DiscoverableTool = { - name: "close_credit_card_account_7834", - description: "Close a customer's credit card account permanently.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The credit card account ID to close", - }, - { - name: "user_id", - type: "string", - optional: false, - description: "The user's unique identifier in the system", - }, - ] satisfies readonly ToolParameter[], - handler: closeCreditCardAccount, -}; - -export function registerCreditTools(): void { - registerDiscoverableAgentTool(orderReplacementCreditCardTool); - registerDiscoverableAgentTool(getPendingReplacementOrdersTool); - registerDiscoverableAgentTool(logCreditCardClosureReasonTool); - registerDiscoverableAgentTool(closeCreditCardAccountTool); - registerDiscoverableAgentTool(getUserDisputeHistoryTool); - registerDiscoverableAgentTool(getClosureReasonHistoryTool); - registerDiscoverableAgentTool(applyStatementCreditTool); - registerDiscoverableAgentTool(applyCreditCardAccountFlagTool); - registerCreditLimitAndPaymentTools(); -} diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-debit.test.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-debit.test.ts deleted file mode 100644 index dcb76288..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-debit.test.ts +++ /dev/null @@ -1,721 +0,0 @@ -import { describe, it, expect, beforeEach } from "bun:test"; - -import { registerDebitTools } from "./handlers-debit"; -import type { BankingEnvState } from "./registry"; -import { makeBankingEnvState, DISCOVERABLE_AGENT_TOOLS } from "./registry"; - -const fixtureDebitCards = { - dbc_test_active: { - card_id: "dbc_test_active", - account_id: "chk_test_01", - user_id: "user_test_01", - cardholder_name: "JOHN DOE", - last_4_digits: "5678", - cvv: "123", - status: "ACTIVE", - issue_date: "11/14/2025", - expiration_date: "11/30/2029", - card_design: "CLASSIC", - issue_reason: "new_account", - }, - dbc_test_pending: { - card_id: "dbc_test_pending", - account_id: "chk_test_02", - user_id: "user_test_02", - cardholder_name: "JANE SMITH", - last_4_digits: "9876", - cvv: "456", - status: "PENDING", - issue_date: "11/14/2025", - expiration_date: "11/30/2029", - card_design: "PREMIUM", - issue_reason: "first_card", - }, - dbc_test_pending_lost: { - card_id: "dbc_test_pending_lost", - account_id: "chk_test_03", - user_id: "user_test_03", - cardholder_name: "ALICE JOHNSON", - last_4_digits: "1234", - cvv: "789", - status: "PENDING", - issue_date: "11/14/2025", - expiration_date: "11/30/2029", - card_design: "CLASSIC", - issue_reason: "lost", - }, - dbc_test_pending_expired: { - card_id: "dbc_test_pending_expired", - account_id: "chk_test_04", - user_id: "user_test_04", - cardholder_name: "BOB WILLIAMS", - last_4_digits: "4567", - cvv: "321", - status: "PENDING", - issue_date: "11/14/2025", - expiration_date: "11/30/2029", - card_design: "CUSTOM", - issue_reason: "expired", - }, -}; - -const fixtureAccounts = { - chk_test_01: { - account_id: "chk_test_01", - user_id: "user_test_01", - class: "checking", - level: "Blue Account", - date_opened: "11/01/2024", - status: "OPEN", - current_holdings: "5000.00", - }, - chk_test_02: { - account_id: "chk_test_02", - user_id: "user_test_02", - class: "checking", - level: "Green Account", - date_opened: "09/01/2023", - status: "OPEN", - current_holdings: "3200.50", - }, - chk_test_03: { - account_id: "chk_test_03", - user_id: "user_test_03", - class: "checking", - level: "Blue Account", - date_opened: "01/15/2025", - status: "OPEN", - current_holdings: "1500.00", - }, - chk_test_04: { - account_id: "chk_test_04", - user_id: "user_test_04", - class: "checking", - level: "Green Account", - date_opened: "03/10/2024", - status: "OPEN", - current_holdings: "8500.00", - }, - chk_test_new_order: { - account_id: "chk_test_new_order", - user_id: "user_test_order", - class: "checking", - level: "Blue Account", - date_opened: "06/15/2024", - status: "OPEN", - current_holdings: "5500.00", - }, -}; - -const fixtureUsers = { - user_test_order: { - user_id: "user_test_order", - name: "Test Cardholder", - }, -}; - -function deepCopyCards( - cards: Record> -): Record> { - const result: Record> = {}; - for (const key in cards) { - result[key] = { ...cards[key] }; - } - return result; -} - -function createTestState(): BankingEnvState { - const state = makeBankingEnvState({ - users: { data: { ...fixtureUsers } }, - accounts: { data: { ...fixtureAccounts } }, - debit_cards: { data: deepCopyCards(fixtureDebitCards) }, - debit_card_orders: { data: {} }, - bank_account_transaction_history: { data: {} }, - credit_card_transaction_history: { data: {} }, - referrals: { data: {} }, - credit_card_applications: { data: {} }, - user_discoverable_tools: { data: {} }, - user_discoverable_tool_calls: { data: {} }, - verification_history: { data: {} }, - cash_back_disputes: { data: {} }, - credit_card_accounts: { data: {} }, - agent_discoverable_tools: { data: {} }, - task_config: { data: {} }, - human_transfer_requests: { data: {} }, - transaction_disputes: { data: {} }, - credit_card_orders: { data: {} }, - credit_card_closure_reasons: { data: {} }, - credit_card_account_flags: { data: {} }, - credit_limit_increase_requests: { data: {} }, - payment_history: { data: {} }, - debit_card_disputes: { data: {} }, - }); - return state; -} -describe("Debit Card Handlers", () => { - beforeEach(() => { - registerDebitTools(); - }); - describe("activate_debit_card_8291", () => { - it("activates a new card successfully", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get("activate_debit_card_8291"); - expect(tool).toBeDefined(); - const result = tool!.handler(state, { - card_id: "dbc_test_pending", - last_4_digits: "9876", - expiration_date: "11/30/2029", - cvv: "456", - pin: "2580", - }); - expect(result).toContain("New Debit Card Activation Successful"); - expect(result).toContain("Status: ACTIVE"); - const card = state.db.debit_cards.data.dbc_test_pending; - if (!card) { - throw new Error("Expected dbc_test_pending card"); - } - expect(card.status).toBe("ACTIVE"); - }); - it("rejects activation with invalid PIN", () => { - const state = createTestState(); - const badPinCard = { - ...fixtureDebitCards.dbc_test_pending, - card_id: "dbc_badpin", - }; - state.db.debit_cards.data.dbc_badpin = badPinCard; - const tool = DISCOVERABLE_AGENT_TOOLS.get("activate_debit_card_8291"); - const result = tool!.handler(state, { - card_id: "dbc_badpin", - last_4_digits: "9876", - expiration_date: "11/30/2029", - cvv: "456", - pin: "1234", - }); - expect(result).toContain("Error: PIN cannot be sequential"); - expect(state.db.debit_cards.data.dbc_badpin.status).toBe("PENDING"); - }); - it("rejects activation with wrong issue_reason", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get("activate_debit_card_8291"); - const result = tool!.handler(state, { - card_id: "dbc_test_pending_lost", - last_4_digits: "1234", - expiration_date: "11/30/2029", - cvv: "789", - pin: "2580", - }); - expect(result).toContain("Error: Wrong activation tool"); - expect(result).toContain("activate_debit_card_8292"); - }); - it("rejects activation with mismatched last 4 digits", () => { - const state = createTestState(); - const mismatchCard = { - ...fixtureDebitCards.dbc_test_pending, - card_id: "dbc_mismatch", - }; - state.db.debit_cards.data.dbc_mismatch = mismatchCard; - const tool = DISCOVERABLE_AGENT_TOOLS.get("activate_debit_card_8291"); - const result = tool!.handler(state, { - card_id: "dbc_mismatch", - last_4_digits: "0000", - expiration_date: "11/30/2029", - cvv: "456", - pin: "2580", - }); - expect(result).toContain("Error: Card verification failed"); - }); - }); - describe("activate_debit_card_8292", () => { - it("activates a replacement card (lost)", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get("activate_debit_card_8292"); - const result = tool!.handler(state, { - card_id: "dbc_test_pending_lost", - last_4_digits: "1234", - expiration_date: "11/30/2029", - cvv: "789", - pin: "2580", - }); - expect(result).toContain("Replacement Debit Card Activation Successful"); - expect(result).toContain("Replacement Reason: Lost"); - expect(result.toUpperCase()).toContain("STATUS: ACTIVE"); - }); - it("activates a replacement card (stolen)", () => { - const state = createTestState(); - const stolenCard = { - ...fixtureDebitCards.dbc_test_pending_lost, - card_id: "dbc_stolen", - issue_reason: "stolen", - }; - state.db.debit_cards.data.dbc_stolen = stolenCard; - const tool = DISCOVERABLE_AGENT_TOOLS.get("activate_debit_card_8292"); - const result = tool!.handler(state, { - card_id: "dbc_stolen", - last_4_digits: "1234", - expiration_date: "11/30/2029", - cvv: "789", - pin: "2580", - }); - expect(result).toContain("Replacement Debit Card Activation Successful"); - expect(state.db.debit_cards.data.dbc_stolen.status).toBe("ACTIVE"); - }); - it("rejects replacement card activation with wrong issue_reason", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get("activate_debit_card_8292"); - const result = tool!.handler(state, { - card_id: "dbc_test_pending", - last_4_digits: "9876", - expiration_date: "11/30/2029", - cvv: "456", - pin: "2580", - }); - expect(result).toContain("Error: Wrong activation tool"); - expect(result).toContain("activate_debit_card_8291"); - }); - }); - describe("activate_debit_card_8293", () => { - it("activates a reissued card (expired)", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get("activate_debit_card_8293"); - const result = tool!.handler(state, { - card_id: "dbc_test_pending_expired", - last_4_digits: "4567", - expiration_date: "11/30/2029", - cvv: "321", - pin: "2580", - }); - expect(result).toContain("Reissued Debit Card Activation Successful"); - expect(result).toContain("Reissue Reason: Expired"); - const expiredCard = state.db.debit_cards.data.dbc_test_pending_expired; - if (!expiredCard) { - throw new Error("Expected dbc_test_pending_expired"); - } - expect(expiredCard.status).toBe("ACTIVE"); - }); - }); - describe("freeze_debit_card_3892", () => { - it("freezes an active card", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get("freeze_debit_card_3892"); - const result = tool!.handler(state, { - card_id: "dbc_test_active", - }); - expect(result).toContain("Debit Card Frozen Successfully"); - const activeCard1 = state.db.debit_cards.data.dbc_test_active; - if (!activeCard1) { - throw new Error("Expected dbc_test_active"); - } - expect(activeCard1.status).toBe("FROZEN"); - }); - it("rejects freezing a frozen card", () => { - const state = createTestState(); - const activeCard2 = state.db.debit_cards.data.dbc_test_active; - if (!activeCard2) { - throw new Error("Expected dbc_test_active"); - } - activeCard2.status = "FROZEN"; - const tool = DISCOVERABLE_AGENT_TOOLS.get("freeze_debit_card_3892"); - const result = tool!.handler(state, { - card_id: "dbc_test_active", - }); - expect(result).toContain( - "Error: Debit card 'dbc_test_active' is already frozen" - ); - }); - }); - describe("unfreeze_debit_card_3893", () => { - it("unfreezes a frozen card", () => { - const state = createTestState(); - const activeCard3 = state.db.debit_cards.data.dbc_test_active; - if (!activeCard3) { - throw new Error("Expected dbc_test_active"); - } - activeCard3.status = "FROZEN"; - const tool = DISCOVERABLE_AGENT_TOOLS.get("unfreeze_debit_card_3893"); - const result = tool!.handler(state, { - card_id: "dbc_test_active", - }); - expect(result).toContain("Debit Card Unfrozen Successfully"); - expect(activeCard3.status).toBe("ACTIVE"); - }); - it("freeze then unfreeze roundtrip", () => { - const state = createTestState(); - const freezeTool = DISCOVERABLE_AGENT_TOOLS.get("freeze_debit_card_3892"); - const unfreezeTool = DISCOVERABLE_AGENT_TOOLS.get( - "unfreeze_debit_card_3893" - ); - freezeTool!.handler(state, { card_id: "dbc_test_active" }); - const activeCard4 = state.db.debit_cards.data.dbc_test_active; - if (!activeCard4) { - throw new Error("Expected dbc_test_active"); - } - expect(activeCard4.status).toBe("FROZEN"); - const unfreezeResult = unfreezeTool!.handler(state, { - card_id: "dbc_test_active", - }); - expect(unfreezeResult).toContain("Debit Card Unfrozen Successfully"); - expect(activeCard4.status).toBe("ACTIVE"); - }); - }); - describe("close_debit_card_4721", () => { - it("closes an active card", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get("close_debit_card_4721"); - const result = tool!.handler(state, { - card_id: "dbc_test_active", - reason: "no_longer_needed", - }); - expect(result).toContain("Debit Card Closed Successfully"); - expect(result).toContain("Closure Reason: No longer needed"); - const activeCard5 = state.db.debit_cards.data.dbc_test_active; - if (!activeCard5) { - throw new Error("Expected dbc_test_active"); - } - expect(activeCard5.status).toBe("CLOSED"); - }); - it("rejects closing a non-existent card", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get("close_debit_card_4721"); - const result = tool!.handler(state, { - card_id: "dbc_nonexistent", - reason: "lost", - }); - expect(result).toContain("Error: Debit card 'dbc_nonexistent' not found"); - }); - }); - describe("clear_debit_card_fraud_alert_4892", () => { - it("clears velocity block", () => { - const state = createTestState(); - const activeCard6 = state.db.debit_cards.data.dbc_test_active; - if (!activeCard6) { - throw new Error("Expected dbc_test_active"); - } - activeCard6.velocity_blocked = true; - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "clear_debit_card_fraud_alert_4892" - ); - const result = tool!.handler(state, { - card_id: "dbc_test_active", - reason: "velocity_clear", - }); - expect(result).toContain("Velocity Block Cleared Successfully"); - expect(activeCard6.velocity_blocked).toBe(false); - }); - it("clears fraud alert", () => { - const state = createTestState(); - const activeCard7 = state.db.debit_cards.data.dbc_test_active; - if (!activeCard7) { - throw new Error("Expected dbc_test_active"); - } - activeCard7.fraud_alert_active = true; - activeCard7.alert_source = "customer_reported"; - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "clear_debit_card_fraud_alert_4892" - ); - const result = tool!.handler(state, { - card_id: "dbc_test_active", - reason: "customer_verified", - }); - expect(result).toContain("Fraud Alert Cleared Successfully"); - expect(activeCard7.fraud_alert_active).toBe(false); - }); - }); - describe("reset_debit_card_pin_6284", () => { - it("resets PIN on active card", () => { - const state = createTestState(); - const pinResetCard = { - ...fixtureDebitCards.dbc_test_active, - card_id: "dbc_pinreset", - }; - state.db.debit_cards.data.dbc_pinreset = pinResetCard; - const tool = DISCOVERABLE_AGENT_TOOLS.get("reset_debit_card_pin_6284"); - const result = tool!.handler(state, { - card_id: "dbc_pinreset", - last_4_digits: "5678", - new_pin: "3692", - }); - expect(result).toContain("Debit Card PIN Reset Successfully"); - expect(state.db.debit_cards.data.dbc_pinreset.pin_locked).toBe(false); - }); - it("rejects PIN reset with invalid PIN", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get("reset_debit_card_pin_6284"); - const result = tool!.handler(state, { - card_id: "dbc_test_active", - last_4_digits: "5678", - new_pin: "1111", - }); - expect(result).toContain("Error: PIN cannot be all the same digit"); - }); - }); - describe("change_debit_card_pin_6285", () => { - it("changes PIN on active card", () => { - const state = createTestState(); - const pinChangeCard = { - ...fixtureDebitCards.dbc_test_active, - card_id: "dbc_pinchange", - }; - state.db.debit_cards.data.dbc_pinchange = pinChangeCard; - const tool = DISCOVERABLE_AGENT_TOOLS.get("change_debit_card_pin_6285"); - const result = tool!.handler(state, { - card_id: "dbc_pinchange", - current_pin: "3692", - new_pin: "2580", - }); - expect(result).toContain("Debit Card PIN Changed Successfully"); - }); - it("rejects PIN change when new PIN equals current PIN", () => { - const state = createTestState(); - const samePinCard = { - ...fixtureDebitCards.dbc_test_active, - card_id: "dbc_samepin", - }; - state.db.debit_cards.data.dbc_samepin = samePinCard; - const tool = DISCOVERABLE_AGENT_TOOLS.get("change_debit_card_pin_6285"); - const result = tool!.handler(state, { - card_id: "dbc_samepin", - current_pin: "3692", - new_pin: "3692", - }); - expect(result).toContain( - "Error: New PIN must be different from current PIN" - ); - }); - }); - describe("get_debit_cards_by_account_id_7823", () => { - it("retrieves debit cards for an account", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "get_debit_cards_by_account_id_7823" - ); - const result = tool!.handler(state, { - account_id: "chk_test_01", - }); - expect(result).not.toContain("Error"); - const parsed = JSON.parse(result); - expect(Array.isArray(parsed)).toBe(true); - expect(parsed.length).toBeGreaterThan(0); - expect(parsed[0].card_id).toBe("dbc_test_active"); - }); - it("returns error for non-existent account", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "get_debit_cards_by_account_id_7823" - ); - const result = tool!.handler(state, { - account_id: "chk_nonexistent", - }); - expect(result).toContain("Error: Account 'chk_nonexistent' not found"); - }); - }); - describe("order_debit_card_5739", () => { - it("orders a debit card with fees", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get("order_debit_card_5739"); - const result = tool!.handler(state, { - account_id: "chk_test_new_order", - user_id: "user_test_order", - delivery_option: "STANDARD", - delivery_fee: 15, - card_design: "CLASSIC", - design_fee: 5, - shipping_address: "123 Main St, Anytown, USA", - excess_replacement_fee: 0, - }); - expect(result).toContain("Debit Card Order Confirmed"); - expect(result).toContain("Total Fees: $20.00"); - expect(result).toContain("7-10 business days"); - const ordersKeys = Object.keys(state.db.debit_card_orders.data); - expect(ordersKeys.length).toBeGreaterThan(0); - const orderId = ordersKeys[0]; - if (!orderId) { - throw new Error("Expected order ID"); - } - const order = state.db.debit_card_orders.data[orderId]; - if (!order) { - throw new Error("Expected order"); - } - expect(order.total_fee).toBe(20); - expect(order.status).toBe("PENDING"); - }); - it("deducts fees from account balance", () => { - const state = createTestState(); - const account1 = state.db.accounts.data.chk_test_new_order; - if (!account1) { - throw new Error("Expected chk_test_new_order account"); - } - const initialBalance = Number.parseFloat( - String(account1.current_holdings) - ); - const tool = DISCOVERABLE_AGENT_TOOLS.get("order_debit_card_5739"); - tool!.handler(state, { - account_id: "chk_test_new_order", - user_id: "user_test_order", - delivery_option: "EXPEDITED", - delivery_fee: 25, - card_design: "PREMIUM", - design_fee: 10, - shipping_address: "456 Oak Ave, Somewhere, USA", - excess_replacement_fee: 5, - }); - const newBalance = Number.parseFloat(String(account1.current_holdings)); - expect(newBalance).toBe(initialBalance - 40); - }); - it("creates fee transaction record", () => { - const state = createTestState(); - const tool = DISCOVERABLE_AGENT_TOOLS.get("order_debit_card_5739"); - tool!.handler(state, { - account_id: "chk_test_new_order", - user_id: "user_test_order", - delivery_option: "RUSH", - delivery_fee: 50, - card_design: "CUSTOM", - design_fee: 15, - shipping_address: "789 Pine Rd, Elsewhere, USA", - excess_replacement_fee: 10, - }); - const txnKeys = Object.keys( - state.db.bank_account_transaction_history.data - ); - expect(txnKeys.length).toBeGreaterThan(0); - const txnId = txnKeys[0]; - if (!txnId) { - throw new Error("Expected transaction ID"); - } - const txn = state.db.bank_account_transaction_history.data[txnId]; - if (!txn) { - throw new Error("Expected transaction"); - } - expect(txn.type).toBe("debit_card_fee"); - expect(txn.status).toBe("posted"); - expect(txn.amount).toBe(-75); - expect(String(txn.description)).toContain("DEBIT CARD ORDER FEE"); - }); - it("rejects order with insufficient balance for fees", () => { - const state = createTestState(); - const account2 = state.db.accounts.data.chk_test_new_order; - if (!account2) { - throw new Error("Expected chk_test_new_order account"); - } - account2.current_holdings = "30.00"; - const tool = DISCOVERABLE_AGENT_TOOLS.get("order_debit_card_5739"); - const result = tool!.handler(state, { - account_id: "chk_test_new_order", - user_id: "user_test_order", - delivery_option: "STANDARD", - delivery_fee: 15, - card_design: "CLASSIC", - design_fee: 5, - shipping_address: "123 Main St", - excess_replacement_fee: 20, - }); - expect(result).toContain("Error: Insufficient funds for fees"); - }); - it("rejects order with insufficient minimum balance", () => { - const state = createTestState(); - const account3 = state.db.accounts.data.chk_test_new_order; - if (!account3) { - throw new Error("Expected chk_test_new_order account"); - } - account3.current_holdings = "10.00"; - const tool = DISCOVERABLE_AGENT_TOOLS.get("order_debit_card_5739"); - const result = tool!.handler(state, { - account_id: "chk_test_new_order", - user_id: "user_test_order", - delivery_option: "STANDARD", - delivery_fee: 0, - card_design: "CLASSIC", - design_fee: 0, - shipping_address: "123 Main St", - excess_replacement_fee: 0, - }); - expect(result).toContain( - "Error: Account must have a minimum balance of $25" - ); - }); - }); - describe("request_temporary_debit_card_limit_increase_8374", () => { - it("grants ATM limit increase", () => { - const state = createTestState(); - const limitCard = { - ...fixtureDebitCards.dbc_test_active, - card_id: "dbc_lim1", - daily_atm_limit: 500, - }; - state.db.debit_cards.data.dbc_lim1 = limitCard; - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "request_temporary_debit_card_limit_increase_8374" - ); - const result = tool!.handler(state, { - card_id: "dbc_lim1", - limit_type: "atm", - new_limit: 600, - }); - expect(result).toContain( - "Temporary Daily ATM Withdrawal Limit Increase Granted Successfully" - ); - expect(state.db.debit_cards.data.dbc_lim1.daily_atm_limit).toBe(600); - }); - it("rejects limit increase exceeding 150% of current", () => { - const state = createTestState(); - const limitCard = { - ...fixtureDebitCards.dbc_test_active, - card_id: "dbc_lim2", - daily_atm_limit: 500, - }; - state.db.debit_cards.data.dbc_lim2 = limitCard; - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "request_temporary_debit_card_limit_increase_8374" - ); - const result = tool!.handler(state, { - card_id: "dbc_lim2", - limit_type: "atm", - new_limit: 900, - }); - expect(result).toContain( - "Error: Requested limit $900 exceeds the maximum allowed" - ); - }); - it("rejects limit increase not higher than current", () => { - const state = createTestState(); - const limitCard = { - ...fixtureDebitCards.dbc_test_active, - card_id: "dbc_lim3", - daily_atm_limit: 500, - }; - state.db.debit_cards.data.dbc_lim3 = limitCard; - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "request_temporary_debit_card_limit_increase_8374" - ); - const result = tool!.handler(state, { - card_id: "dbc_lim3", - limit_type: "atm", - new_limit: 500, - }); - expect(result).toContain( - "Error: Requested limit $500 is not higher than the current limit" - ); - }); - it("rejects fractional limits without truncating", () => { - const state = createTestState(); - const limitCard = { - ...fixtureDebitCards.dbc_test_active, - card_id: "dbc_fractional", - daily_atm_limit: 500, - }; - state.db.debit_cards.data.dbc_fractional = limitCard; - const tool = DISCOVERABLE_AGENT_TOOLS.get( - "request_temporary_debit_card_limit_increase_8374" - ); - const result = tool!.handler(state, { - card_id: "dbc_fractional", - limit_type: "atm", - new_limit: 600.5, - }); - expect(result).toBe("Error: new_limit must be an integer, got '600.5'."); - expect(state.db.debit_cards.data.dbc_fractional.daily_atm_limit).toBe( - 500 - ); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-debit.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-debit.ts deleted file mode 100644 index d3e27098..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-debit.ts +++ /dev/null @@ -1,1309 +0,0 @@ -import { createHash } from "node:crypto"; - -import { isDefinedAndNotNull } from "../../../internal/guards"; -import { getFieldAsString } from "./field-access"; -import { getTodayStr } from "./helpers"; -import { generateDebitCardOrderId, generateDebitCardId } from "./ids"; -import type { BankingEnvState } from "./registry"; -import { registerDiscoverableAgentTool } from "./registry"; - -function validatePin(pin: string): string | null { - if (!pin || !/^\d{4}$/.test(pin)) { - return "PIN must be exactly 4 digits."; - } - const sequentialPins = [ - "0123", - "1234", - "2345", - "3456", - "4567", - "5678", - "6789", - "9876", - "8765", - "7654", - "6543", - "5432", - "4321", - "3210", - ]; - if (sequentialPins.includes(pin)) { - return "PIN cannot be sequential (e.g., 1234). Please choose a more secure PIN."; - } - const uniqueDigits = new Set(pin).size; - if (uniqueDigits === 1) { - return "PIN cannot be all the same digit (e.g., 1111). Please choose a more secure PIN."; - } - return null; -} - -function validateActivationCommon(opts: { - args: Record; - state: BankingEnvState; - allowedIssueReasons: readonly string[]; - toolName: string; -}): [string | null, Record | null] { - const { args, state, allowedIssueReasons, toolName } = opts; - const cardId = getFieldAsString(args, "card_id"); - const last4Digits = getFieldAsString(args, "last_4_digits"); - const expirationDate = getFieldAsString(args, "expiration_date"); - const cvv = getFieldAsString(args, "cvv"); - const pin = getFieldAsString(args, "pin"); - if (!cardId || !last4Digits || !expirationDate || !cvv || !pin) { - return [ - "Error: Missing required parameters. Required: card_id, last_4_digits, expiration_date, cvv, pin.", - null, - ]; - } - const pinError = validatePin(pin); - if (pinError) { - return [`Error: ${pinError}`, null]; - } - if (!/^\d{3}$/.test(cvv)) { - return ["Error: CVV must be exactly 3 digits.", null]; - } - if (!/^\d{4}$/.test(last4Digits)) { - return ["Error: Last 4 digits must be exactly 4 digits.", null]; - } - const cardsTable = state.db.debit_cards.data; - if (!(cardId in cardsTable)) { - return [`Error: Debit card '${cardId}' not found.`, null]; - } - const card = cardsTable[cardId]!; - const issueReason = (card.issue_reason as string) || "new_account"; - const issueReasonArray = [...allowedIssueReasons]; - if (!issueReasonArray.includes(issueReason)) { - const reasonMap: Record = { - new_account: "activate_debit_card_8291", - first_card: "activate_debit_card_8291", - lost: "activate_debit_card_8292", - stolen: "activate_debit_card_8292", - fraud: "activate_debit_card_8292", - expired: "activate_debit_card_8293", - damaged: "activate_debit_card_8293", - upgrade: "activate_debit_card_8293", - bank_reissue: "activate_debit_card_8293", - }; - const correctTool = reasonMap[issueReason] || "unknown"; - return [ - `Error: Wrong activation tool. This card has issue_reason='${issueReason}'. Please use ${correctTool} instead of ${toolName}.`, - null, - ]; - } - if (card.status === "ACTIVE") { - return [`Error: Debit card '${cardId}' is already active.`, null]; - } - if (card.status !== "PENDING") { - return [ - `Error: Debit card '${cardId}' cannot be activated. Current status: ${card.status}. Only PENDING cards can be activated.`, - null, - ]; - } - if (card.last_4_digits !== last4Digits) { - return [ - "Error: Card verification failed. The last 4 digits do not match our records.", - null, - ]; - } - const accountId = card.account_id as string | undefined; - if (accountId && accountId in state.db.accounts.data) { - const account = state.db.accounts.data[accountId]!; - if (account.status !== "OPEN") { - return [ - `Error: The linked checking account '${accountId}' is no longer open. Card cannot be activated.`, - null, - ]; - } - } - return [null, card]; -} - -function orderDebitCard(opts: { - state: BankingEnvState; - accountId: string; - userId: string; - deliveryOption: string; - deliveryFee: unknown; - cardDesign: string; - designFee: unknown; - shippingAddress: string; - excessReplacementFee: unknown; -}): string { - const { - state, - accountId, - userId, - deliveryOption, - deliveryFee, - cardDesign, - designFee, - shippingAddress, - excessReplacementFee, - } = opts; - let delivery = 0; - let design = 0; - let excess = 0; - try { - delivery = isDefinedAndNotNull(deliveryFee) - ? Number.parseFloat(String(deliveryFee)) - : 0; - } catch { - return "Error: delivery_fee must be a number."; - } - try { - design = isDefinedAndNotNull(designFee) - ? Number.parseFloat(String(designFee)) - : 0; - } catch { - return "Error: design_fee must be a number."; - } - try { - excess = isDefinedAndNotNull(excessReplacementFee) - ? Number.parseFloat(String(excessReplacementFee)) - : 0; - } catch { - excess = 0; - } - if ( - !accountId || - !userId || - !deliveryOption || - !cardDesign || - !shippingAddress - ) { - return "Error: Missing required parameters. Required: account_id, user_id, delivery_option, delivery_fee, card_design, design_fee, shipping_address."; - } - const validDelivery = ["STANDARD", "EXPEDITED", "RUSH"]; - const deliveryUpper = String(deliveryOption).toUpperCase(); - if (!validDelivery.includes(deliveryUpper)) { - return `Error: Invalid delivery_option. Must be one of: ${validDelivery.join(", ")}`; - } - const validDesign = ["CLASSIC", "PREMIUM", "CUSTOM"]; - const designUpper = String(cardDesign).toUpperCase(); - if (!validDesign.includes(designUpper)) { - return `Error: Invalid card_design. Must be one of: ${validDesign.join(", ")}`; - } - const accountsTable = state.db.accounts.data; - if (!(accountId in accountsTable)) { - return `Error: Account '${accountId}' not found.`; - } - const account = accountsTable[accountId]!; - if (account.class !== "checking") { - return `Error: Debit cards can only be ordered for checking accounts. Account '${accountId}' is a ${account.class} account.`; - } - if (account.status !== "OPEN") { - return `Error: Account must be OPEN. Account '${accountId}' has status: ${account.status}`; - } - if (account.user_id !== userId) { - return `Error: Account '${accountId}' does not belong to user '${userId}'.`; - } - let currentHoldings = 0; - try { - const holdings = String(account.current_holdings || 0).replaceAll(",", ""); - currentHoldings = Number.parseFloat(holdings); - } catch { - currentHoldings = 0; - } - if (currentHoldings < 25) { - return `Error: Account must have a minimum balance of $25. Current balance: $${currentHoldings.toFixed(2)}`; - } - const debitCardOrdersTable = state.db.debit_card_orders.data; - for (const orderId in debitCardOrdersTable) { - const order = debitCardOrdersTable[orderId]!; - if (order.account_id === accountId && order.status === "PENDING") { - return `Error: There is already a pending debit card order for account '${accountId}'.`; - } - } - const debitCardsTable = state.db.debit_cards.data; - let activeCardCount = 0; - for (const cardId in debitCardsTable) { - const card = debitCardsTable[cardId]!; - if (card.account_id === accountId && card.status === "ACTIVE") { - activeCardCount += 1; - } - } - if (activeCardCount >= 1) { - return `Error: Account '${accountId}' already has an active debit card. Maximum 1 active card per checking account.`; - } - const totalFee = delivery + design + excess; - if (totalFee > 0 && currentHoldings < totalFee) { - return `Error: Insufficient funds for fees. Total fees: $${totalFee.toFixed(2)}. Current balance: $${currentHoldings.toFixed(2)}`; - } - const deliveryTimes: Record = { - STANDARD: "7-10 business days", - EXPEDITED: "3-5 business days", - RUSH: "1-2 business days", - }; - const expectedDelivery = deliveryTimes[deliveryUpper]; - const orderDate = getTodayStr(); - const orderId = generateDebitCardOrderId(accountId, userId, deliveryUpper); - const orderRecord: Record = { - order_id: orderId, - account_id: accountId, - user_id: userId, - delivery_option: deliveryUpper, - card_design: designUpper, - shipping_address: shippingAddress, - delivery_fee: delivery, - design_fee: design, - excess_replacement_fee: excess, - total_fee: totalFee, - order_date: orderDate, - expected_delivery: expectedDelivery, - status: "PENDING", - }; - state.db.debit_card_orders.data[orderId] = orderRecord; - if (totalFee > 0) { - const newBalance = currentHoldings - totalFee; - account.current_holdings = newBalance.toFixed(2); - const feeDescriptionParts: string[] = []; - if (delivery > 0) { - feeDescriptionParts.push(`Delivery $${delivery.toFixed(2)}`); - } - if (design > 0) { - feeDescriptionParts.push(`Design $${design.toFixed(2)}`); - } - if (excess > 0) { - feeDescriptionParts.push(`Excess Replacement $${excess.toFixed(0)}`); - } - const feeDescription = `DEBIT CARD ORDER FEE - ${feeDescriptionParts.join(", ")}`; - const feeTxnId = `btxn_dcfee_${orderId.slice(-8)}`; - const feeTransaction: Record = { - transaction_id: feeTxnId, - account_id: accountId, - date: orderDate, - description: feeDescription, - amount: -totalFee, - type: "debit_card_fee", - status: "posted", - }; - state.db.bank_account_transaction_history.data[feeTxnId] = feeTransaction; - } - const cardId = generateDebitCardId(accountId, userId, orderDate); - let cardholder = "CARDHOLDER"; - const usersTable = state.db.users.data; - if (userId in usersTable) { - const user = usersTable[userId]!; - cardholder = ((user.name as string) || "CARDHOLDER").toUpperCase(); - } - let last4Digits = "0000"; - let cvvDigits = "000"; - try { - const cardDetailsSeed = `card_details:${cardId}`; - const hash1 = createHash("sha256") - .update(`${cardDetailsSeed}:last4`) - .digest("hex"); - const hash2 = createHash("sha256") - .update(`${cardDetailsSeed}:cvv`) - .digest("hex"); - const digits1 = hash1.replaceAll(/\D/g, ""); - const digits2 = hash2.replaceAll(/\D/g, ""); - last4Digits = (digits1.slice(0, 4) || "0000").padEnd(4, "0"); - cvvDigits = (digits2.slice(0, 3) || "000").padEnd(3, "0"); - } catch { - last4Digits = "0000"; - cvvDigits = "000"; - } - let expirationDate = "12/31/2029"; - try { - const parts = orderDate.split("/"); - if (parts.length === 3) { - const month = parts[0]!; - const year = parts[2]!; - const yearInt = Number.parseInt(year, 10) + 4; - let day = "31"; - if (["01", "03", "05", "07", "08", "10", "12"].includes(month)) { - day = "31"; - } else if (month === "02") { - day = "28"; - } else { - day = "30"; - } - expirationDate = `${month}/${day}/${yearInt}`; - } - } catch { - expirationDate = "12/31/2029"; - } - let issueReason = "new_account"; - const existingCards: Record[] = []; - for (const cid in debitCardsTable) { - const c = debitCardsTable[cid]!; - if (c.account_id === accountId) { - existingCards.push(c); - } - } - const closedCard = existingCards.find((c) => c.status === "CLOSED"); - if (closedCard) { - const closureReason = String(closedCard.closure_reason || "first_card"); - if ( - ["lost", "stolen", "fraud", "fraud_suspected"].includes(closureReason) - ) { - issueReason = - closureReason === "fraud_suspected" ? "fraud" : closureReason; - } else { - issueReason = "first_card"; - } - } else if (existingCards.length > 0) { - issueReason = "first_card"; - } - const cardRecord: Record = { - card_id: cardId, - account_id: accountId, - user_id: userId, - cardholder_name: cardholder, - last_4_digits: last4Digits, - cvv: cvvDigits, - status: "PENDING", - issue_date: orderDate, - expiration_date: expirationDate, - card_design: designUpper, - issue_reason: issueReason, - }; - state.db.debit_cards.data[cardId] = cardRecord; - const resultParts: string[] = [ - "Debit Card Order Confirmed", - `Order ID: ${orderId}`, - `Card ID: ${cardId}`, - `Linked Account: ${accountId}`, - `Delivery Option: ${deliveryUpper}`, - `Card Design: ${designUpper}`, - `Shipping Address: ${shippingAddress}`, - `Expected Delivery: ${expectedDelivery}`, - "", - "Note: Card will arrive with status PENDING. Customer must call to activate after receiving the card.", - ]; - if (totalFee > 0) { - const feeDetails: string[] = []; - if (delivery > 0) { - feeDetails.push(`Delivery: $${delivery.toFixed(2)}`); - } - if (design > 0) { - feeDetails.push(`Design: $${design.toFixed(2)}`); - } - if (excess > 0) { - feeDetails.push(`Excess Replacement: $${excess.toFixed(0)}`); - } - resultParts.push( - `Total Fees: $${totalFee.toFixed(2)} (${feeDetails.join(", ")}) - CHARGED to account ${accountId}` - ); - resultParts.push( - `New Account Balance: $${(currentHoldings - totalFee).toFixed(2)}` - ); - } else { - resultParts.push("Total Fees: $0 (No additional charges)"); - } - return resultParts.join("\n"); -} - -export function registerDebitTools(): void { - registerDiscoverableAgentTool({ - name: "order_debit_card_5739", - description: "Order a new debit card for a customer's checking account.", - params: [ - { - name: "account_id", - type: "string", - optional: false, - description: "The checking account ID to link the debit card to", - }, - { - name: "user_id", - type: "string", - optional: false, - description: "The customer's unique identifier", - }, - { - name: "delivery_option", - type: "string", - optional: false, - description: "Shipping speed: STANDARD, EXPEDITED, or RUSH", - }, - { - name: "delivery_fee", - type: "number", - optional: false, - description: "Fee to charge for delivery in dollars", - }, - { - name: "card_design", - type: "string", - optional: false, - description: "Card design: CLASSIC, PREMIUM, or CUSTOM", - }, - { - name: "design_fee", - type: "number", - optional: false, - description: "Fee to charge for card design in dollars", - }, - { - name: "shipping_address", - type: "string", - optional: false, - description: "Full shipping address for card delivery", - }, - { - name: "excess_replacement_fee", - type: "number", - optional: true, - description: "Fee for exceeding replacement limit, if applicable", - }, - ], - handler: (state, kwargs) => - orderDebitCard({ - state, - accountId: String(kwargs.account_id), - userId: String(kwargs.user_id), - deliveryOption: String(kwargs.delivery_option), - deliveryFee: kwargs.delivery_fee, - cardDesign: String(kwargs.card_design), - designFee: kwargs.design_fee, - shippingAddress: String(kwargs.shipping_address), - excessReplacementFee: kwargs.excess_replacement_fee, - }), - }); - registerDiscoverableAgentTool({ - name: "activate_debit_card_8291", - description: - "Activate a NEW debit card for a customer. Use ONLY for first-time cards on a checking account (issue_reason = 'new_account' or 'first_card').", - params: [ - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID to activate", - }, - { - name: "last_4_digits", - type: "string", - optional: false, - description: "Last 4 digits of the card number (for verification)", - }, - { - name: "expiration_date", - type: "string", - optional: false, - description: "Card expiration date in MM/YY format", - }, - { - name: "cvv", - type: "string", - optional: false, - description: "3-digit CVV from the back of the card", - }, - { - name: "pin", - type: "string", - optional: false, - description: "4-digit PIN chosen by the customer", - }, - ], - handler: (state, kwargs) => { - const [error, card] = validateActivationCommon({ - args: kwargs, - state, - allowedIssueReasons: ["new_account", "first_card"], - toolName: "activate_debit_card_8291", - }); - if (error) { - return error; - } - const cardId = String(kwargs.card_id); - const accountId = String(card!.account_id); - card!.status = "ACTIVE"; - card!.activated_date = getTodayStr(); - const deactivatedCards: string[] = []; - const cardsTable = state.db.debit_cards.data; - for (const otherId in cardsTable) { - const other = cardsTable[otherId]!; - if ( - otherId !== cardId && - other.account_id === accountId && - other.status === "ACTIVE" - ) { - other.status = "DEACTIVATED"; - other.deactivated_date = getTodayStr(); - other.deactivation_reason = "New card activated"; - deactivatedCards.push(otherId); - } - } - const resultParts: string[] = [ - "New Debit Card Activation Successful", - `Card ID: ${cardId}`, - "Status: ACTIVE", - `Activation Date: ${getTodayStr()}`, - "", - "Your card is now ready to use at any ATM or point of sale terminal.", - "For security, please sign the back of your card.", - ]; - if (deactivatedCards.length > 0) { - resultParts.push( - `\nNote: Previous card(s) have been deactivated: ${deactivatedCards.join(", ")}` - ); - } - return resultParts.join("\n"); - }, - }); - registerDiscoverableAgentTool({ - name: "activate_debit_card_8292", - description: - "Activate a REPLACEMENT debit card. Use ONLY for cards replacing lost, stolen, or fraud-suspected cards.", - params: [ - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID to activate", - }, - { - name: "last_4_digits", - type: "string", - optional: false, - description: "Last 4 digits of the card number (for verification)", - }, - { - name: "expiration_date", - type: "string", - optional: false, - description: "Card expiration date in MM/YY format", - }, - { - name: "cvv", - type: "string", - optional: false, - description: "3-digit CVV from the back of the card", - }, - { - name: "pin", - type: "string", - optional: false, - description: "4-digit PIN chosen by the customer", - }, - ], - handler: (state, kwargs) => { - const [error, card] = validateActivationCommon({ - args: kwargs, - state, - allowedIssueReasons: ["lost", "stolen", "fraud"], - toolName: "activate_debit_card_8292", - }); - if (error) { - return error; - } - const cardId = String(kwargs.card_id); - const accountId = String(card!.account_id); - const issueReason = String(card!.issue_reason || "lost"); - card!.status = "ACTIVE"; - card!.activated_date = getTodayStr(); - const deactivatedCards: string[] = []; - const cardsTable = state.db.debit_cards.data; - for (const otherId in cardsTable) { - const other = cardsTable[otherId]!; - if ( - otherId !== cardId && - other.account_id === accountId && - other.status === "ACTIVE" - ) { - other.status = "DEACTIVATED"; - other.deactivated_date = getTodayStr(); - other.deactivation_reason = `Replacement card activated (${issueReason})`; - deactivatedCards.push(otherId); - } - } - const resultParts: string[] = [ - "Replacement Debit Card Activation Successful", - `Card ID: ${cardId}`, - `Replacement Reason: ${issueReason - .replaceAll("_", " ") - .split(" ") - .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(" ")}`, - "Status: ACTIVE", - `Activation Date: ${getTodayStr()}`, - "", - "Your replacement card is now ready to use.", - "", - "IMPORTANT SECURITY REMINDERS:", - "- Please review your recent transactions for any unauthorized charges", - "- Report any suspicious activity immediately", - ]; - if (issueReason === "fraud") { - resultParts.push( - "- Since fraud was suspected, we recommend changing your online banking password" - ); - } - if (deactivatedCards.length > 0) { - resultParts.push( - `\nPrevious card(s) have been deactivated for security: ${deactivatedCards.join(", ")}` - ); - } - return resultParts.join("\n"); - }, - }); - registerDiscoverableAgentTool({ - name: "activate_debit_card_8293", - description: - "Activate a REISSUED debit card. Use ONLY for cards reissued due to expiration, damage, upgrade, or bank-initiated replacement.", - params: [ - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID to activate", - }, - { - name: "last_4_digits", - type: "string", - optional: false, - description: "Last 4 digits of the card number (for verification)", - }, - { - name: "expiration_date", - type: "string", - optional: false, - description: "Card expiration date in MM/YY format", - }, - { - name: "cvv", - type: "string", - optional: false, - description: "3-digit CVV from the back of the card", - }, - { - name: "pin", - type: "string", - optional: false, - description: "4-digit PIN chosen by the customer", - }, - ], - handler: (state, kwargs) => { - const [error, card] = validateActivationCommon({ - args: kwargs, - state, - allowedIssueReasons: ["expired", "damaged", "upgrade", "bank_reissue"], - toolName: "activate_debit_card_8293", - }); - if (error) { - return error; - } - const cardId = String(kwargs.card_id); - const accountId = String(card!.account_id); - const issueReason = String(card!.issue_reason || "expired"); - card!.status = "ACTIVE"; - card!.activated_date = getTodayStr(); - const oldCardsWithGrace: string[] = []; - const cardsTable = state.db.debit_cards.data; - for (const otherId in cardsTable) { - const other = cardsTable[otherId]!; - if ( - otherId !== cardId && - other.account_id === accountId && - other.status === "ACTIVE" - ) { - other.status = "GRACE_PERIOD"; - other.grace_period_ends = getTodayStr(); - other.deactivation_reason = `Reissued card activated (${issueReason})`; - oldCardsWithGrace.push(otherId); - } - } - const resultParts: string[] = [ - "Reissued Debit Card Activation Successful", - `Card ID: ${cardId}`, - `Reissue Reason: ${issueReason.replaceAll("_", " ").replace(/^./, (c) => c.toUpperCase())}`, - "Status: ACTIVE", - `Activation Date: ${getTodayStr()}`, - "", - "Your reissued card is now ready to use.", - ]; - if (oldCardsWithGrace.length > 0) { - resultParts.push( - `\nNote: Your previous card(s) (${oldCardsWithGrace.join(", ")}) will remain active for 24 hours as a grace period.` - ); - resultParts.push( - "After 24 hours, the old card(s) will be automatically deactivated." - ); - } - if (["expired", "bank_reissue"].includes(issueReason)) { - resultParts.push( - "\nReminder: If your card number changed, please update any recurring payments with your new card details." - ); - } - return resultParts.join("\n"); - }, - }); - registerDiscoverableAgentTool({ - name: "close_debit_card_4721", - description: "Close or cancel a debit card permanently.", - params: [ - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID to close", - }, - { - name: "reason", - type: "string", - optional: false, - description: - "Reason for closing: lost, stolen, fraud_suspected, damaged, no_longer_needed, or account_closing", - }, - ], - handler: (state, kwargs) => { - const cardId = String(kwargs.card_id); - const reason = String(kwargs.reason).toLowerCase(); - if (!cardId || !reason) { - return "Error: Missing required parameters. Required: card_id, reason."; - } - const validReasons = [ - "lost", - "stolen", - "fraud_suspected", - "damaged", - "no_longer_needed", - "account_closing", - ]; - if (!validReasons.includes(reason)) { - return `Error: Invalid reason. Must be one of: ${validReasons.join(", ")}`; - } - const cardsTable = state.db.debit_cards.data; - if (!(cardId in cardsTable)) { - return `Error: Debit card '${cardId}' not found.`; - } - const card = cardsTable[cardId]!; - if (!["ACTIVE", "PENDING"].includes(String(card.status))) { - return `Error: Debit card '${cardId}' cannot be closed. Current status: ${card.status}. Only ACTIVE or PENDING cards can be closed.`; - } - const previousStatus = card.status; - card.status = "CLOSED"; - card.closed_date = getTodayStr(); - card.closure_reason = reason; - const resultParts: string[] = [ - "Debit Card Closed Successfully", - `Card ID: ${cardId}`, - `Previous Status: ${previousStatus}`, - "New Status: CLOSED", - `Closure Reason: ${reason.replaceAll("_", " ").replace(/^./, (c) => c.toUpperCase())}`, - `Closure Date: ${getTodayStr()}`, - "", - ]; - if (["lost", "stolen", "fraud_suspected"].includes(reason)) { - resultParts.push( - "IMPORTANT: This card has been immediately deactivated for security." - ); - resultParts.push("Any pending transactions may still be processed."); - if (reason === "fraud_suspected") { - resultParts.push( - "Please advise the customer to review recent transactions and file disputes for any unauthorized charges." - ); - resultParts.push( - "Also recommend changing their online banking password." - ); - } - } - resultParts.push(""); - resultParts.push( - "Note: This card cannot be reactivated. If the customer needs a new card, they can order one through the standard ordering process." - ); - resultParts.push( - "Any recurring payments linked to this card will need to be updated with new payment information." - ); - return resultParts.join("\n"); - }, - }); - registerDiscoverableAgentTool({ - name: "freeze_debit_card_3892", - description: - "Temporarily freeze a debit card. The card can be unfrozen later.", - params: [ - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID to freeze", - }, - ], - handler: (state, kwargs) => { - const cardId = String(kwargs.card_id); - if (!cardId) { - return "Error: Missing required parameter: card_id."; - } - const cardsTable = state.db.debit_cards.data; - if (!(cardId in cardsTable)) { - return `Error: Debit card '${cardId}' not found.`; - } - const card = cardsTable[cardId]!; - if (card.status === "FROZEN") { - return `Error: Debit card '${cardId}' is already frozen.`; - } - if (card.status !== "ACTIVE") { - return `Error: Debit card '${cardId}' cannot be frozen. Current status: ${card.status}. Only ACTIVE cards can be frozen.`; - } - card.status = "FROZEN"; - card.frozen_date = getTodayStr(); - const resultParts: string[] = [ - "Debit Card Frozen Successfully", - `Card ID: ${cardId}`, - "Status: FROZEN", - `Frozen Date: ${getTodayStr()}`, - "", - "While frozen:", - "- All new purchase transactions will be declined", - "- Recurring payments and subscriptions will be declined", - "- Pending transactions already authorized may still process", - "", - "To unfreeze, the customer can call customer service or use the mobile app.", - "If the card is confirmed lost or stolen, recommend closing the card permanently instead.", - ]; - return resultParts.join("\n"); - }, - }); - registerDiscoverableAgentTool({ - name: "unfreeze_debit_card_3893", - description: "Unfreeze a previously frozen debit card.", - params: [ - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID to unfreeze", - }, - ], - handler: (state, kwargs) => { - const cardId = String(kwargs.card_id); - if (!cardId) { - return "Error: Missing required parameter: card_id."; - } - const cardsTable = state.db.debit_cards.data; - if (!(cardId in cardsTable)) { - return `Error: Debit card '${cardId}' not found.`; - } - const card = cardsTable[cardId]!; - if (card.status === "ACTIVE") { - return `Error: Debit card '${cardId}' is already active.`; - } - if (card.status !== "FROZEN") { - return `Error: Debit card '${cardId}' cannot be unfrozen. Current status: ${card.status}. Only FROZEN cards can be unfrozen.`; - } - const accountId = card.account_id as string | undefined; - if (accountId && accountId in state.db.accounts.data) { - const account = state.db.accounts.data[accountId]!; - if (account.status !== "OPEN") { - return `Error: The linked checking account '${accountId}' is no longer open. Card cannot be unfrozen.`; - } - } - card.status = "ACTIVE"; - card.unfrozen_date = getTodayStr(); - const resultParts: string[] = [ - "Debit Card Unfrozen Successfully", - `Card ID: ${cardId}`, - "Status: ACTIVE", - `Unfrozen Date: ${getTodayStr()}`, - "", - "The card is now active and ready to use immediately.", - "All transactions will process normally.", - ]; - return resultParts.join("\n"); - }, - }); - registerDiscoverableAgentTool({ - name: "clear_debit_card_fraud_alert_4892", - description: "Clear a fraud alert or velocity block on a debit card.", - params: [ - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID to clear the alert/block for", - }, - { - name: "reason", - type: "string", - optional: false, - description: "Reason for clearing: customer_verified or velocity_clear", - }, - ], - handler: (state, kwargs) => { - const cardId = String(kwargs.card_id); - const reason = String(kwargs.reason); - if (!cardId) { - return "Error: Missing required parameter: card_id."; - } - if (!reason) { - return "Error: Missing required parameter: reason."; - } - const validReasons = ["customer_verified", "velocity_clear"]; - if (!validReasons.includes(reason)) { - return `Error: Invalid reason '${reason}'. Must be one of: ${validReasons.join(", ")}`; - } - const cardsTable = state.db.debit_cards.data; - if (!(cardId in cardsTable)) { - return `Error: Debit card '${cardId}' not found.`; - } - const card = cardsTable[cardId]!; - if (reason === "velocity_clear") { - if (!card.velocity_blocked) { - return `Error: Debit card '${cardId}' does not have an active velocity block.`; - } - card.velocity_blocked = false; - card.velocity_cleared_date = getTodayStr(); - const resultParts: string[] = [ - "Velocity Block Cleared Successfully", - `Card ID: ${cardId}`, - `Cleared Date: ${getTodayStr()}`, - "", - "The card is now unblocked and ready for normal use.", - "The velocity monitoring will continue - if the same unusual patterns recur,", - "the card may be blocked again automatically.", - ]; - return resultParts.join("\n"); - } - if (reason === "customer_verified") { - if (!card.fraud_alert_active) { - return `Error: Debit card '${cardId}' does not have an active fraud alert.`; - } - const alertSource = card.alert_source as string | undefined; - if (alertSource === "bank_initiated") { - return "Error: BANK_INITIATED_ALERT - This fraud alert was initiated by the bank's fraud detection system and cannot be cleared by customer service agents. Please transfer the customer to the security team using transfer_to_human_agents."; - } - card.fraud_alert_active = false; - card.alert_source = null; - card.fraud_alert_cleared_date = getTodayStr(); - const resultParts: string[] = [ - "Fraud Alert Cleared Successfully", - `Card ID: ${cardId}`, - `Cleared Date: ${getTodayStr()}`, - "", - "The fraud alert has been removed from the card.", - "All transactions will process normally.", - "", - "Remind the customer to review recent transactions and report any unauthorized charges.", - ]; - return resultParts.join("\n"); - } - return "Error: Unexpected error processing the request."; - }, - }); - registerDiscoverableAgentTool({ - name: "reset_debit_card_pin_6284", - description: "Reset a debit card PIN when the customer has forgotten it.", - params: [ - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID to reset PIN for", - }, - { - name: "last_4_digits", - type: "string", - optional: false, - description: "Last 4 digits of the card number (for verification)", - }, - { - name: "new_pin", - type: "string", - optional: false, - description: "The new 4-digit PIN chosen by the customer", - }, - ], - handler: (state, kwargs) => { - const cardId = String(kwargs.card_id); - const last4Digits = String(kwargs.last_4_digits); - const newPin = String(kwargs.new_pin); - if (!cardId || !last4Digits || !newPin) { - return "Error: Missing required parameters. Required: card_id, last_4_digits, new_pin."; - } - if (!/^\d{4}$/.test(last4Digits)) { - return "Error: Last 4 digits must be exactly 4 digits."; - } - const pinError = validatePin(newPin); - if (pinError) { - return `Error: ${pinError}`; - } - const cardsTable = state.db.debit_cards.data; - if (!(cardId in cardsTable)) { - return `Error: Debit card '${cardId}' not found.`; - } - const card = cardsTable[cardId]!; - if (card.last_4_digits !== last4Digits) { - return "Error: Card verification failed. The last 4 digits do not match our records."; - } - if (card.status !== "ACTIVE") { - return `Error: Cannot reset PIN. Card status is ${card.status}. Only ACTIVE cards can have their PIN reset.`; - } - card.pin_last_changed = getTodayStr(); - card.pin_locked = false; - card.pin_attempts_remaining = 3; - const resultParts: string[] = [ - "Debit Card PIN Reset Successfully", - `Card ID: ${cardId}`, - `PIN Changed: ${getTodayStr()}`, - "", - "The new PIN is effective immediately.", - "Your card has been unlocked and is ready to use.", - "Customer can use the new PIN for ATM withdrawals and point-of-sale transactions.", - "", - "Security reminder: Never share your PIN with anyone.", - ]; - return resultParts.join("\n"); - }, - }); - registerDiscoverableAgentTool({ - name: "change_debit_card_pin_6285", - description: - "Change a debit card PIN when the customer knows their current PIN.", - params: [ - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID to change PIN for", - }, - { - name: "current_pin", - type: "string", - optional: false, - description: "The customer's current 4-digit PIN", - }, - { - name: "new_pin", - type: "string", - optional: false, - description: "The new 4-digit PIN chosen by the customer", - }, - ], - handler: (state, kwargs) => { - const cardId = String(kwargs.card_id); - const currentPin = String(kwargs.current_pin); - const newPin = String(kwargs.new_pin); - if (!cardId || !currentPin || !newPin) { - return "Error: Missing required parameters. Required: card_id, current_pin, new_pin."; - } - if (!/^\d{4}$/.test(currentPin)) { - return "Error: Current PIN must be exactly 4 digits."; - } - const pinError = validatePin(newPin); - if (pinError) { - return `Error: ${pinError}`; - } - if (currentPin === newPin) { - return "Error: New PIN must be different from current PIN."; - } - const cardsTable = state.db.debit_cards.data; - if (!(cardId in cardsTable)) { - return `Error: Debit card '${cardId}' not found.`; - } - const card = cardsTable[cardId]!; - if (card.status !== "ACTIVE") { - return `Error: Cannot change PIN. Card status is ${card.status}. Only ACTIVE cards can have their PIN changed.`; - } - card.pin_last_changed = getTodayStr(); - const resultParts: string[] = [ - "Debit Card PIN Changed Successfully", - `Card ID: ${cardId}`, - `PIN Changed: ${getTodayStr()}`, - "", - "The new PIN is effective immediately.", - "Customer can use the new PIN for ATM withdrawals and point-of-sale transactions.", - "", - "Security reminder: Never share your PIN with anyone.", - ]; - return resultParts.join("\n"); - }, - }); - registerDiscoverableAgentTool({ - name: "get_debit_cards_by_account_id_7823", - mutatesState: false, - description: "Retrieve all debit cards associated with a checking account.", - params: [ - { - name: "account_id", - type: "string", - optional: false, - description: "The checking account ID to retrieve debit cards for", - }, - ], - handler: (state, kwargs) => { - const accountId = String(kwargs.account_id); - if (!accountId) { - return "Error: Missing required parameter 'account_id'."; - } - const accountsTable = state.db.accounts.data; - if (!(accountId in accountsTable)) { - return `Error: Account '${accountId}' not found.`; - } - const account = accountsTable[accountId]!; - const accountClass = String(account.class || "").toLowerCase(); - if (!["checking", "business_checking"].includes(accountClass)) { - return `Error: Account '${accountId}' is not a checking account. Debit cards are only available for checking accounts.`; - } - const accountCards: Record[] = []; - const cardsTable = state.db.debit_cards.data; - for (const cardId in cardsTable) { - const card = cardsTable[cardId]!; - if (card.account_id === accountId) { - const cardInfo: Record = { - card_id: cardId, - ...card, - }; - if ( - "last_4_digits" in cardInfo && - !("card_number_last_4" in cardInfo) - ) { - cardInfo.card_number_last_4 = cardInfo.last_4_digits; - delete cardInfo.last_4_digits; - } - if ("issue_date" in cardInfo && !("date_issued" in cardInfo)) { - cardInfo.date_issued = cardInfo.issue_date; - delete cardInfo.issue_date; - } else if ( - "created_date" in cardInfo && - !("date_issued" in cardInfo) - ) { - cardInfo.date_issued = cardInfo.created_date; - delete cardInfo.created_date; - } - accountCards.push(cardInfo); - } - } - if (accountCards.length === 0) { - return `No debit cards found for account '${accountId}'.`; - } - accountCards.sort((a, b) => - (String(b.date_issued) || "").localeCompare(String(a.date_issued) || "") - ); - return JSON.stringify(accountCards, null, 2); - }, - }); - registerDiscoverableAgentTool({ - name: "request_temporary_debit_card_limit_increase_8374", - description: - "Request a temporary 24-hour increase to a debit card's daily ATM or purchase limit.", - params: [ - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID to increase limits for", - }, - { - name: "limit_type", - type: "string", - optional: false, - description: "Type of limit to increase: atm or purchase", - }, - { - name: "new_limit", - type: "number", - optional: false, - description: "The requested new temporary limit amount in dollars", - }, - ], - handler: (state, kwargs) => { - const cardId = String(kwargs.card_id); - const limitType = String(kwargs.limit_type); - const newLimitRaw = kwargs.new_limit; - if (!cardId) { - return "Error: Missing required parameter: card_id."; - } - if (!limitType) { - return "Error: Missing required parameter: limit_type."; - } - if (!["atm", "purchase"].includes(limitType)) { - return `Error: Invalid limit_type '${limitType}'. Must be 'atm' or 'purchase'.`; - } - if (newLimitRaw === null || newLimitRaw === undefined) { - return "Error: Missing required parameter: new_limit."; - } - const newLimit = Number(String(newLimitRaw)); - if (!Number.isFinite(newLimit) || !Number.isInteger(newLimit)) { - return `Error: new_limit must be an integer, got '${newLimitRaw}'.`; - } - if (newLimit <= 0) { - return "Error: new_limit must be a positive amount."; - } - const cardsTable = state.db.debit_cards.data; - if (!(cardId in cardsTable)) { - return `Error: Debit card '${cardId}' not found.`; - } - const card = cardsTable[cardId]!; - if (card.status !== "ACTIVE") { - return `Error: Debit card '${cardId}' is not active. Current status: ${card.status}. Only ACTIVE cards can have limit increases.`; - } - const accountId = card.account_id as string | undefined; - if (!accountId || !(accountId in state.db.accounts.data)) { - return `Error: Could not find linked account for debit card '${cardId}'.`; - } - const account = state.db.accounts.data[accountId]!; - if (account.status !== "OPEN") { - return `Error: The linked account '${accountId}' is not in good standing. Account status: ${account.status}.`; - } - let currentLimit: number | undefined; - let limitField: string; - let limitName: string; - if (limitType === "atm") { - currentLimit = card.daily_atm_limit as number | undefined; - limitField = "daily_atm_limit"; - limitName = "Daily ATM Withdrawal Limit"; - if (currentLimit === null || currentLimit === undefined) { - const accountLevel = String(account.level || "").toLowerCase(); - const defaultAtmLimits: Record = { - "blue account": 500, - "green account": 600, - "light blue account": 400, - "green fee-free account": 500, - "evergreen account": 750, - "bluest account": 1500, - "dark green account": 300, - "gold years account": 600, - "purple account": 1000, - }; - if (accountLevel.includes("light green")) { - return `Error: Light Green Account (teen checking) cards have policy-based limits that cannot be modified. The daily ATM withdrawal limit of $150 is fixed by account policy for safety reasons. The customer may request the parent/guardian to withdraw cash from their own account if needed.`; - } - currentLimit = defaultAtmLimits[accountLevel]; - if (currentLimit === undefined) { - return `Error: Could not determine the default ATM limit for account type '${account.level}'. Please verify the account type.`; - } - } - } else { - currentLimit = card.daily_purchase_limit as number | undefined; - limitField = "daily_purchase_limit"; - limitName = "Daily Purchase Limit"; - if (currentLimit === null || currentLimit === undefined) { - return `Error: The card does not have a ${limitName.toLowerCase()} configured. This may be a restricted account type where limits are set by account policy and cannot be modified.`; - } - } - const maxAllowed = Math.floor(currentLimit * 1.5); - if (newLimit > maxAllowed) { - return `Error: Requested limit $${newLimit} exceeds the maximum allowed temporary increase. Maximum temporary limit is $${maxAllowed} (150% of current $${currentLimit} limit).`; - } - if (newLimit <= currentLimit) { - return `Error: Requested limit $${newLimit} is not higher than the current limit of $${currentLimit}.`; - } - const originalLimit = currentLimit; - card[limitField] = newLimit; - card[`temporary_${limitType}_limit_increase`] = true; - card[`original_${limitType}_limit`] = originalLimit; - card[`temporary_${limitType}_limit_expires`] = getTodayStr(); - const resultParts: string[] = [ - `Temporary ${limitName} Increase Granted Successfully`, - `Card ID: ${cardId}`, - `Previous Limit: $${originalLimit}`, - `New Temporary Limit: $${newLimit}`, - `Increase Amount: $${newLimit - originalLimit}`, - "", - "Important Information:", - "- This temporary increase expires in 24 hours", - `- After expiration, the limit will revert to $${originalLimit}`, - "- Only one temporary increase is allowed per 24-hour period", - "", - "Note: If the customer is at a third-party (non-Rho-Bank) ATM, that ATM may have its own", - "per-transaction or daily limits that Rho-Bank cannot override. The customer may need to", - "use a Rho-Bank ATM or make multiple smaller withdrawals at third-party ATMs.", - ]; - return resultParts.join("\n"); - }, - }); -} diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-disputes.test.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-disputes.test.ts deleted file mode 100644 index 9ca903e9..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-disputes.test.ts +++ /dev/null @@ -1,626 +0,0 @@ -import { expect, test } from "bun:test"; -import assert from "node:assert/strict"; - -import { isRecord } from "../../../internal/guards"; -import { makeEmptyBankingData } from "../environment"; -import type { BankingData } from "../types"; -import { addToDb, queryDb } from "./db-query"; -import { registerDisputeTools } from "./handlers-disputes"; -import { generateDisputeId } from "./ids"; -import type { BankingEnvState } from "./registry"; -import { makeBankingEnvState, DISCOVERABLE_AGENT_TOOLS } from "./registry"; - -function createTestDb(): BankingData { - return { - ...makeEmptyBankingData(), - users: { - data: { - user_123: { - user_id: "user_123", - name: "John Doe", - email: "john@example.com", - phone_number: "555-0123", - address: "123 Main St", - date_of_birth: "01/01/1990", - }, - user_456: { - user_id: "user_456", - name: "Jane Smith", - email: "jane@example.com", - phone_number: "555-0456", - address: "456 Oak Ave", - date_of_birth: "05/15/1985", - }, - }, - }, - credit_card_transaction_history: { - data: { - txn_001: { - transaction_id: "txn_001", - user_id: "user_123", - credit_card_type: "Gold Rewards Card", - merchant_name: "Amazon", - transaction_amount: "$99.99", - transaction_date: "11/10/2025", - category: "Shopping", - status: "COMPLETED", - rewards_earned: "250 points", - }, - txn_002: { - transaction_id: "txn_002", - user_id: "user_456", - credit_card_type: "Platinum Card", - merchant_name: "Costco", - transaction_amount: "$150.00", - transaction_date: "11/08/2025", - category: "Groceries", - status: "COMPLETED", - rewards_earned: "375 points", - }, - }, - }, - debit_cards: { - data: { - dbc_001: { - card_id: "dbc_001", - account_id: "chk_001", - user_id: "user_123", - cardholder_name: "JOHN DOE", - last_4_digits: "5678", - cvv: "123", - status: "ACTIVE", - issue_date: "01/01/2024", - expiration_date: "12/31/2027", - issue_reason: "replacement", - recurring_blocked: false, - }, - }, - }, - bank_account_transaction_history: { - data: { - btxn_deposit_001: { - transaction_id: "btxn_deposit_001", - account_id: "chk_001", - date: "11/10/2025", - description: "ATM DEPOSIT - RHO-BANK ATM #3921 AUSTIN TX", - amount: 500, - type: "atm_deposit", - status: "posted", - }, - btxn_834027370c20: { - transaction_id: "btxn_834027370c20", - account_id: "chk_001", - date: "11/10/2025", - description: "ATM DEPOSIT - RHO-BANK ATM #3921 CEDAR LANE AUSTIN TX", - amount: 1135, - type: "atm_deposit", - status: "posted", - }, - }, - }, - task_config: { - data: { - dispute_settings: { - auto_resolve_disputes: false, - }, - }, - }, - }; -} -test("registerDisputeTools populates the registry", () => { - registerDisputeTools(); - expect(DISCOVERABLE_AGENT_TOOLS.has("update_transaction_rewards_3847")).toBe( - true - ); - expect( - DISCOVERABLE_AGENT_TOOLS.has("file_credit_card_transaction_dispute_4829") - ).toBe(true); - expect( - DISCOVERABLE_AGENT_TOOLS.has("file_debit_card_transaction_dispute_6281") - ).toBe(true); - expect( - DISCOVERABLE_AGENT_TOOLS.has("set_debit_card_recurring_block_7382") - ).toBe(true); - expect(DISCOVERABLE_AGENT_TOOLS.has("get_debit_dispute_status_7483")).toBe( - true - ); - expect(DISCOVERABLE_AGENT_TOOLS.has("get_atm_deposit_images_8473")).toBe( - true - ); -}); -test("update_transaction_rewards_3847: success path", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "update_transaction_rewards_3847" - )!; - const result = handler(state, { - transaction_id: "txn_001", - new_rewards_earned: "5000 points", - }); - expect(result).toContain("Transaction rewards updated successfully"); - expect(result).toContain("txn_001"); - expect(result).toContain("5000 points"); - const updated = db.credit_card_transaction_history.data["txn_001"]; - assert(isRecord(updated)); - expect(updated.rewards_earned).toBe("5000 points"); -}); -test("update_transaction_rewards_3847: validation error - missing transaction_id", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "update_transaction_rewards_3847" - )!; - const result = handler(state, { new_rewards_earned: "5000 points" }); - expect(result).toBe("Error: Missing required parameters."); -}); -test("update_transaction_rewards_3847: error - transaction not found", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "update_transaction_rewards_3847" - )!; - const result = handler(state, { - transaction_id: "txn_nonexistent", - new_rewards_earned: "5000 points", - }); - expect(result).toContain("Error: Transaction 'txn_nonexistent' not found."); -}); -test("file_credit_card_transaction_dispute_4829: success with full_refund", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "file_credit_card_transaction_dispute_4829" - )!; - const result = handler(state, { - transaction_id: "txn_001", - card_action: "keep_active", - card_last_4_digits: "1234", - full_name: "John Doe", - user_id: "user_123", - phone: "555-0123", - email: "john@example.com", - address: "123 Main St", - contacted_merchant: true, - purchase_date: "11/10/2025", - issue_noticed_date: "11/11/2025", - dispute_reason: "duplicate_charge", - resolution_requested: "full_refund", - eligible_for_provisional_credit: true, - }); - expect(result).toContain( - "Credit card transaction dispute filed successfully" - ); - expect(result).toContain("Duplicate Charge"); - expect(result).toContain("Full Refund"); - expect(result).toContain( - "ELIGIBLE - Credit will be applied within 2 business days" - ); - const disputeId = generateDisputeId("user_123", "txn_001"); - const disputes = queryDb({ - db, - dbName: "transaction_disputes", - constraints: { dispute_id: disputeId }, - }); - expect(disputes.length).toBe(1); - const dispute = disputes[0]; - assert(isRecord(dispute)); - expect(dispute.dispute_id).toBe(disputeId); - expect(dispute.transaction_id).toBe("txn_001"); - expect(dispute.user_id).toBe("user_123"); - expect(dispute.status).toBe("SUBMITTED"); - expect(dispute.provisional_credit_given).toBe(true); -}); -test("file_credit_card_transaction_dispute_4829: success with partial_refund", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "file_credit_card_transaction_dispute_4829" - )!; - const result = handler(state, { - transaction_id: "txn_002", - card_action: "cancel_and_reissue", - card_last_4_digits: "5678", - full_name: "Jane Smith", - user_id: "user_456", - phone: "555-0456", - email: "jane@example.com", - address: "456 Oak Ave", - contacted_merchant: false, - purchase_date: "11/08/2025", - issue_noticed_date: "11/09/2025", - dispute_reason: "incorrect_amount", - resolution_requested: "partial_refund", - partial_refund_amount: 50, - eligible_for_provisional_credit: false, - }); - expect(result).toContain("Partial Refund Amount: $50.00"); - expect(result).toContain("Not eligible at this time"); - const disputeId = generateDisputeId("user_456", "txn_002"); - const disputes = queryDb({ - db, - dbName: "transaction_disputes", - constraints: { dispute_id: disputeId }, - }); - expect(disputes.length).toBe(1); - const dispute = disputes[0]; - assert(isRecord(dispute)); - expect(dispute.partial_refund_amount).toBe(50); - expect(dispute.resolution_requested).toBe("partial_refund"); -}); -test("file_credit_card_transaction_dispute_4829: validation error - invalid dispute_reason", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "file_credit_card_transaction_dispute_4829" - )!; - const result = handler(state, { - transaction_id: "txn_001", - card_action: "keep_active", - card_last_4_digits: "1234", - full_name: "John Doe", - user_id: "user_123", - phone: "555-0123", - email: "john@example.com", - address: "123 Main St", - contacted_merchant: true, - purchase_date: "11/10/2025", - issue_noticed_date: "11/11/2025", - dispute_reason: "invalid_reason", - resolution_requested: "full_refund", - eligible_for_provisional_credit: false, - }); - expect(result).toContain("Error: Invalid dispute_reason"); -}); -test("file_credit_card_transaction_dispute_4829: validation error - partial_refund without amount", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "file_credit_card_transaction_dispute_4829" - )!; - const result = handler(state, { - transaction_id: "txn_001", - card_action: "keep_active", - card_last_4_digits: "1234", - full_name: "John Doe", - user_id: "user_123", - phone: "555-0123", - email: "john@example.com", - address: "123 Main St", - contacted_merchant: true, - purchase_date: "11/10/2025", - issue_noticed_date: "11/11/2025", - dispute_reason: "duplicate_charge", - resolution_requested: "partial_refund", - eligible_for_provisional_credit: false, - }); - expect(result).toContain( - "Error: partial_refund_amount is required when resolution_requested is 'partial_refund'." - ); -}); -test("file_debit_card_transaction_dispute_6281: success with fraud category", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "file_debit_card_transaction_dispute_6281" - )!; - const result = handler(state, { - transaction_id: "btxn_001", - account_id: "chk_001", - card_id: "dbc_001", - user_id: "user_123", - dispute_category: "unauthorized_transaction", - transaction_date: "11/10/2025", - discovery_date: "11/11/2025", - disputed_amount: 750, - transaction_type: "pin_purchase", - card_in_possession: true, - pin_compromised: "no", - contacted_merchant: false, - police_report_filed: true, - written_statement_provided: true, - provisional_credit_eligible: true, - customer_max_liability_amount: 0, - card_action: "freeze_pending_investigation", - }); - expect(result).toContain("Dispute ID:"); - expect(result).toContain("btxn_001"); - expect(result).toContain("Disputed Amount: $750.00"); - expect(result).toContain("Freeze Pending Investigation"); - expect(result).toContain("ISSUED"); - const disputeId = generateDisputeId("user_123", "btxn_001"); - const disputes = queryDb({ - db, - dbName: "debit_card_disputes", - constraints: { dispute_id: disputeId }, - }); - expect(disputes.length).toBe(1); - const dispute = disputes[0]; - assert(isRecord(dispute)); - expect(dispute.dispute_id).toBe(disputeId); - expect(dispute.is_fraud_category).toBe(true); - expect(dispute.provisional_credit_issued).toBe(true); - expect(dispute.provisional_credit_amount).toBe(750); - expect(dispute.status).toBe("OPEN"); -}); -test("file_debit_card_transaction_dispute_6281: validation error - invalid dispute category", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "file_debit_card_transaction_dispute_6281" - )!; - const result = handler(state, { - transaction_id: "btxn_001", - account_id: "chk_001", - card_id: "dbc_001", - user_id: "user_123", - dispute_category: "invalid_category", - transaction_date: "11/10/2025", - discovery_date: "11/11/2025", - disputed_amount: 750, - transaction_type: "pin_purchase", - card_in_possession: true, - pin_compromised: "no", - contacted_merchant: false, - police_report_filed: false, - written_statement_provided: true, - provisional_credit_eligible: false, - customer_max_liability_amount: 0, - card_action: "keep_active", - }); - expect(result).toContain("Error: Invalid dispute_category"); -}); -test("file_debit_card_transaction_dispute_6281: validation error - negative amount", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "file_debit_card_transaction_dispute_6281" - )!; - const result = handler(state, { - transaction_id: "btxn_001", - account_id: "chk_001", - card_id: "dbc_001", - user_id: "user_123", - dispute_category: "duplicate_charge", - transaction_date: "11/10/2025", - discovery_date: "11/11/2025", - disputed_amount: -100, - transaction_type: "pin_purchase", - card_in_possession: true, - pin_compromised: "no", - contacted_merchant: false, - police_report_filed: false, - written_statement_provided: true, - provisional_credit_eligible: false, - customer_max_liability_amount: 0, - card_action: "keep_active", - }); - expect(result).toContain("Error: disputed_amount must be a positive number."); -}); -test("file_debit_card_transaction_dispute_6281: validation error - missing boolean field", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "file_debit_card_transaction_dispute_6281" - )!; - const result = handler(state, { - transaction_id: "btxn_001", - account_id: "chk_001", - card_id: "dbc_001", - user_id: "user_123", - dispute_category: "duplicate_charge", - transaction_date: "11/10/2025", - discovery_date: "11/11/2025", - disputed_amount: 100, - transaction_type: "pin_purchase", - pin_compromised: "no", - provisional_credit_eligible: false, - customer_max_liability_amount: 0, - card_action: "keep_active", - }); - expect(result).toContain( - "Error: card_in_possession, contacted_merchant, police_report_filed, and written_statement_provided are required boolean fields." - ); -}); -test("set_debit_card_recurring_block_7382: success - block recurring", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "set_debit_card_recurring_block_7382" - )!; - const result = handler(state, { - card_id: "dbc_001", - block_recurring: true, - }); - expect(result).toContain("Recurring payments BLOCKED"); - expect(result).toContain("dbc_001"); - expect(result).toContain( - "All recurring/subscription charges will be declined" - ); - const card = db.debit_cards.data["dbc_001"]; - assert(isRecord(card)); - expect(card.recurring_blocked).toBe(true); -}); -test("set_debit_card_recurring_block_7382: success - unblock recurring", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "set_debit_card_recurring_block_7382" - )!; - const result = handler(state, { - card_id: "dbc_001", - block_recurring: false, - }); - expect(result).toContain("Recurring payments UNBLOCKED"); - expect(result).toContain("will now be allowed"); - const card = db.debit_cards.data["dbc_001"]; - assert(isRecord(card)); - expect(card.recurring_blocked).toBe(false); -}); -test("set_debit_card_recurring_block_7382: error - card not found", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "set_debit_card_recurring_block_7382" - )!; - const result = handler(state, { - card_id: "dbc_nonexistent", - block_recurring: true, - }); - expect(result).toContain("Error: Debit card 'dbc_nonexistent' not found."); -}); -test("set_debit_card_recurring_block_7382: error - card not active", () => { - registerDisputeTools(); - const db = createTestDb(); - const inactiveCard = db.debit_cards.data["dbc_inactive"]; - if (!inactiveCard) { - db.debit_cards.data["dbc_inactive"] = { - card_id: "dbc_inactive", - account_id: "chk_002", - user_id: "user_456", - cardholder_name: "JANE SMITH", - last_4_digits: "9876", - status: "SUSPENDED", - recurring_blocked: false, - }; - } - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "set_debit_card_recurring_block_7382" - )!; - const result = handler(state, { - card_id: "dbc_inactive", - block_recurring: true, - }); - expect(result).toContain("Error: Cannot update recurring block settings"); - expect(result).toContain("Card must be ACTIVE"); -}); -test("get_debit_dispute_status_7483: success with disputes", () => { - registerDisputeTools(); - const db = createTestDb(); - const disputeId = generateDisputeId("user_123", "btxn_001"); - addToDb({ - db, - dbName: "debit_card_disputes", - recordId: disputeId, - record: { - dispute_id: disputeId, - transaction_id: "btxn_001", - user_id: "user_123", - status: "OPEN", - disputed_amount: 500, - }, - }); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "get_debit_dispute_status_7483" - )!; - const result = handler(state, { user_id: "user_123" }); - expect(result).toContain("Debit card dispute history retrieved successfully"); - expect(result).toContain("user_123"); -}); -test("get_debit_dispute_status_7483: success - no disputes", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "get_debit_dispute_status_7483" - )!; - const result = handler(state, { user_id: "user_456" }); - expect(result).toContain("Debit card dispute history retrieved successfully"); - expect(result).toContain("No debit card disputes found for this user"); -}); -test("get_atm_deposit_images_8473: success with hardcoded image data", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "get_atm_deposit_images_8473" - )!; - const result = handler(state, { transaction_id: "btxn_834027370c20" }); - expect(result).toContain("ATM Deposit Image Retrieval Results"); - expect(result).toContain("btxn_834027370c20"); - expect(result).toContain("ATM DEPOSIT ENVELOPE SCAN"); - expect(result).toContain("$1,135.00"); - expect(result).toContain("Derek Yamamoto"); - expect(result).toContain("$750.00 difference"); -}); -test("get_atm_deposit_images_8473: success with generic image response", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "get_atm_deposit_images_8473" - )!; - const result = handler(state, { transaction_id: "btxn_deposit_001" }); - expect(result).toContain("ATM Deposit Image Retrieval Results"); - expect(result).toContain("btxn_deposit_001"); - expect(result).toContain("IMAGE STATUS"); - expect(result).toContain("IMAGES NOT AVAILABLE"); -}); -test("get_atm_deposit_images_8473: error - transaction not found", () => { - registerDisputeTools(); - const db = createTestDb(); - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "get_atm_deposit_images_8473" - )!; - const result = handler(state, { transaction_id: "btxn_nonexistent" }); - expect(result).toContain("Error: Transaction 'btxn_nonexistent' not found."); -}); -test("get_atm_deposit_images_8473: error - not an ATM deposit", () => { - registerDisputeTools(); - const db = createTestDb(); - db.bank_account_transaction_history.data["btxn_withdrawal"] = { - transaction_id: "btxn_withdrawal", - account_id: "chk_001", - date: "11/10/2025", - description: "ATM WITHDRAWAL - RHO-BANK ATM #3921 AUSTIN TX", - amount: -100, - type: "atm_withdrawal", - status: "posted", - }; - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "get_atm_deposit_images_8473" - )!; - const result = handler(state, { transaction_id: "btxn_withdrawal" }); - expect(result).toContain( - "Error: Transaction 'btxn_withdrawal' is not an ATM deposit." - ); -}); -test("get_atm_deposit_images_8473: error - third-party ATM", () => { - registerDisputeTools(); - const db = createTestDb(); - db.bank_account_transaction_history.data["btxn_third_party"] = { - transaction_id: "btxn_third_party", - account_id: "chk_001", - date: "11/10/2025", - description: "ATM DEPOSIT - CHASE BANK ATM #5000 CHICAGO IL", - amount: 500, - type: "atm_deposit", - status: "posted", - }; - const state: BankingEnvState = makeBankingEnvState(db); - const { handler } = DISCOVERABLE_AGENT_TOOLS.get( - "get_atm_deposit_images_8473" - )!; - const result = handler(state, { transaction_id: "btxn_third_party" }); - expect(result).toContain("Error: Transaction"); - expect(result).toContain("is from a third-party ATM"); - expect(result).toContain("Rho-Bank ATM"); -}); diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-disputes.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-disputes.ts deleted file mode 100644 index 7a6c0591..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-disputes.ts +++ /dev/null @@ -1,885 +0,0 @@ -import { addToDb, queryDatabaseTool, updateRecordInDb } from "./db-query"; -import { FROZEN_TODAY_STR as FROZEN_TODAY } from "./helpers"; -import { generateDisputeId } from "./ids"; -import type { BankingEnvState } from "./registry"; -import { registerDiscoverableAgentTool } from "./registry"; - -function str(value: unknown): string { - return String(value ?? ""); -} - -function num(value: unknown): number { - if (typeof value === "number") { - return value; - } - if (typeof value === "string") { - const parsed = Number.parseFloat(value); - return Number.isNaN(parsed) ? 0 : parsed; - } - return 0; -} - -function bool(value: unknown): boolean { - if (typeof value === "boolean") { - return value; - } - if (typeof value === "string") { - return value.toLowerCase() === "true"; - } - return !!value; -} - -function updateTransactionRewards( - state: BankingEnvState, - kwargs: Record -): string { - const transactionId = str(kwargs.transaction_id); - const newRewardsEarned = str(kwargs.new_rewards_earned); - if (!transactionId || !newRewardsEarned) { - return "Error: Missing required parameters."; - } - const [success] = updateRecordInDb({ - db: state.db, - dbName: "credit_card_transaction_history", - recordId: transactionId, - updates: { - rewards_earned: newRewardsEarned, - }, - }); - if (!success) { - return `Error: Transaction '${transactionId}' not found.`; - } - return ( - `Transaction rewards updated successfully.\n\n` + - `Executed: update_transaction_rewards_3847\n` + - `Arguments: ${JSON.stringify({ transaction_id: transactionId, new_rewards_earned: newRewardsEarned }, null, 2)}\n` + - `Transaction updated:\n` + - ` - Transaction ID: ${transactionId}\n` + - ` - New Rewards: ${newRewardsEarned}` - ); -} - -function fileCreditCardTransactionDispute( - state: BankingEnvState, - kwargs: Record -): string { - const transactionId = str(kwargs.transaction_id); - const cardAction = str(kwargs.card_action); - const cardLast4Digits = str(kwargs.card_last_4_digits); - const fullName = str(kwargs.full_name); - const userId = str(kwargs.user_id); - const phone = str(kwargs.phone); - const email = str(kwargs.email); - const address = str(kwargs.address); - const contactedMerchant = bool(kwargs.contacted_merchant); - const purchaseDate = str(kwargs.purchase_date); - const issueNoticedDate = str(kwargs.issue_noticed_date); - const disputeReason = str(kwargs.dispute_reason); - const resolutionRequested = str(kwargs.resolution_requested); - const eligibleForProvisionalCredit = bool( - kwargs.eligible_for_provisional_credit - ); - const partialRefundAmount = kwargs.partial_refund_amount - ? num(kwargs.partial_refund_amount) - : null; - if (!transactionId || !userId) { - return "Error: Missing required parameters."; - } - const validCardActions = ["keep_active", "cancel_and_reissue"]; - if (!validCardActions.includes(cardAction)) { - return `Error: Invalid card_action. Must be one of: ${validCardActions}`; - } - const validReasons = [ - "unauthorized_fraudulent_charge", - "duplicate_charge", - "incorrect_amount", - "goods_services_not_received", - "goods_services_not_as_described", - "canceled_subscription_still_charging", - "refund_never_processed", - ]; - if (!validReasons.includes(disputeReason)) { - return `Error: Invalid dispute_reason. Must be one of: ${validReasons}`; - } - const validResolutions = ["full_refund", "partial_refund"]; - if (!validResolutions.includes(resolutionRequested)) { - return `Error: Invalid resolution_requested. Must be one of: ${validResolutions}`; - } - if ( - resolutionRequested === "partial_refund" && - partialRefundAmount === null - ) { - return "Error: partial_refund_amount is required when resolution_requested is 'partial_refund'."; - } - const disputeId = generateDisputeId(userId, transactionId); - const disputeRecord = { - dispute_id: disputeId, - transaction_id: transactionId, - user_id: userId, - card_action: cardAction, - card_last_4_digits: cardLast4Digits, - full_name: fullName, - phone, - email, - address, - contacted_merchant: contactedMerchant, - purchase_date: purchaseDate, - issue_noticed_date: issueNoticedDate, - dispute_reason: disputeReason, - resolution_requested: resolutionRequested, - partial_refund_amount: partialRefundAmount, - eligible_for_provisional_credit: eligibleForProvisionalCredit, - provisional_credit_given: eligibleForProvisionalCredit, - submitted_at: FROZEN_TODAY, - status: "SUBMITTED", - }; - const success = addToDb({ - db: state.db, - dbName: "transaction_disputes", - recordId: disputeId, - record: disputeRecord, - }); - if (!success) { - return "Error: Dispute may have already been filed for this transaction."; - } - const resultParts = [ - "Credit card transaction dispute filed successfully. A case has been opened and will be reviewed within 10 business days.", - "", - "Executed: file_credit_card_transaction_dispute_4829", - `Dispute ID: ${disputeId}`, - `Transaction: ${transactionId}`, - `Reason: ${disputeReason - .replaceAll("_", " ") - .split(" ") - .map((w) => w.charAt(0).toUpperCase() + w.slice(1)) - .join(" ")}`, - `Resolution Requested: ${resolutionRequested - .replaceAll("_", " ") - .split(" ") - .map((w) => w.charAt(0).toUpperCase() + w.slice(1)) - .join(" ")}`, - ]; - if (partialRefundAmount !== null) { - resultParts.push( - `Partial Refund Amount: $${partialRefundAmount.toFixed(2)}` - ); - } - if (eligibleForProvisionalCredit) { - resultParts.push( - "Provisional Credit: ELIGIBLE - Credit will be applied within 2 business days." - ); - } else { - resultParts.push("Provisional Credit: Not eligible at this time."); - } - return resultParts.join("\n"); -} - -function fileDebitCardTransactionDispute( - state: BankingEnvState, - kwargs: Record -): string { - const transactionId = str(kwargs.transaction_id); - const accountId = str(kwargs.account_id); - const cardId = str(kwargs.card_id); - const userId = str(kwargs.user_id); - const disputeCategory = str(kwargs.dispute_category); - const transactionDate = str(kwargs.transaction_date); - const discoveryDate = str(kwargs.discovery_date); - const disputedAmount = num(kwargs.disputed_amount); - const transactionType = str(kwargs.transaction_type); - const cardInPossession = bool(kwargs.card_in_possession); - const pinCompromised = str(kwargs.pin_compromised); - const contactedMerchant = bool(kwargs.contacted_merchant); - const policeReportFiled = bool(kwargs.police_report_filed); - const writtenStatementProvided = bool(kwargs.written_statement_provided); - const provisionalCreditEligible = bool(kwargs.provisional_credit_eligible); - const customerMaxLiabilityAmount = num(kwargs.customer_max_liability_amount); - const cardAction = str(kwargs.card_action); - if ( - !transactionId || - !accountId || - !cardId || - !userId || - !disputeCategory || - !transactionDate || - !discoveryDate || - !transactionType || - !pinCompromised || - !cardAction - ) { - return "Error: Missing required parameters."; - } - if ( - kwargs.customer_max_liability_amount === undefined || - kwargs.customer_max_liability_amount === null - ) { - return "Error: customer_max_liability_amount is required."; - } - if ( - kwargs.card_in_possession === undefined || - kwargs.card_in_possession === null || - kwargs.contacted_merchant === undefined || - kwargs.contacted_merchant === null || - kwargs.police_report_filed === undefined || - kwargs.police_report_filed === null || - kwargs.written_statement_provided === undefined || - kwargs.written_statement_provided === null - ) { - return "Error: card_in_possession, contacted_merchant, police_report_filed, and written_statement_provided are required boolean fields."; - } - const validCategories = [ - "unauthorized_transaction", - "atm_cash_discrepancy", - "atm_deposit_not_credited", - "duplicate_charge", - "incorrect_amount", - "goods_services_not_received", - "recurring_charge_after_cancellation", - "card_present_fraud", - "card_not_present_fraud", - ]; - if (!validCategories.includes(disputeCategory)) { - return `Error: Invalid dispute_category. Must be one of: ${validCategories}`; - } - const validTransactionTypes = [ - "pin_purchase", - "signature_purchase", - "online_purchase", - "atm_withdrawal", - "atm_deposit", - "recurring_payment", - "person_to_person", - ]; - if (!validTransactionTypes.includes(transactionType)) { - return `Error: Invalid transaction_type. Must be one of: ${validTransactionTypes}`; - } - const validPinStatuses = ["yes_shared", "yes_observed", "no", "unknown"]; - if (!validPinStatuses.includes(pinCompromised)) { - return `Error: Invalid pin_compromised. Must be one of: ${validPinStatuses}`; - } - const validCardActions = [ - "keep_active", - "freeze_pending_investigation", - "close_and_reissue", - ]; - if (!validCardActions.includes(cardAction)) { - return `Error: Invalid card_action. Must be one of: ${validCardActions}`; - } - if (disputedAmount <= 0) { - return "Error: disputed_amount must be a positive number."; - } - const disputeId = generateDisputeId(userId, transactionId); - const isFraudCategory = [ - "unauthorized_transaction", - "card_present_fraud", - "card_not_present_fraud", - ].includes(disputeCategory); - const pinSharedVoluntarily = pinCompromised === "yes_shared"; - const disputeRecord = { - dispute_id: disputeId, - transaction_id: transactionId, - account_id: accountId, - card_id: cardId, - user_id: userId, - dispute_category: disputeCategory, - transaction_date: transactionDate, - discovery_date: discoveryDate, - disputed_amount: disputedAmount, - transaction_type: transactionType, - card_in_possession: cardInPossession, - pin_compromised: pinCompromised, - contacted_merchant: contactedMerchant, - police_report_filed: policeReportFiled, - written_statement_provided: writtenStatementProvided, - provisional_credit_eligible: provisionalCreditEligible, - provisional_credit_issued: provisionalCreditEligible, - provisional_credit_amount: provisionalCreditEligible - ? disputedAmount - : null, - customer_max_liability_amount: customerMaxLiabilityAmount, - card_action: cardAction, - is_fraud_category: isFraudCategory, - pin_shared_voluntarily: pinSharedVoluntarily, - submitted_at: FROZEN_TODAY, - status: "OPEN", - }; - const success = addToDb({ - db: state.db, - dbName: "debit_card_disputes", - recordId: disputeId, - record: disputeRecord, - }); - if (!success) { - return "Error: Dispute may have already been filed for this transaction."; - } - const resultParts = [ - `Dispute ID: ${disputeId}`, - `Transaction: ${transactionId}`, - `Account: ${accountId}`, - `Category: ${disputeCategory - .replaceAll("_", " ") - .split(" ") - .map((w) => w.charAt(0).toUpperCase() + w.slice(1)) - .join(" ")}`, - `Disputed Amount: $${disputedAmount.toFixed(2)}`, - `Card Action: ${cardAction - .replaceAll("_", " ") - .split(" ") - .map((w) => w.charAt(0).toUpperCase() + w.slice(1)) - .join(" ")}`, - ]; - if (provisionalCreditEligible) { - resultParts.push( - `Provisional Credit: ISSUED - $${disputedAmount.toFixed(2)} credited within 10 business days per Regulation E.` - ); - } else { - resultParts.push( - "Provisional Credit: Not eligible - see Debit Card Provisional Credit Guidelines for details." - ); - } - if (pinSharedVoluntarily) { - resultParts.push( - "WARNING: Customer indicated PIN was shared voluntarily. This may affect liability determination." - ); - } - if (isFraudCategory && !policeReportFiled && disputedAmount > 500) { - resultParts.push( - "RECOMMENDATION: For fraud disputes over $500, filing a police report is recommended." - ); - } - return resultParts.join("\n"); -} - -function setDebitCardRecurringBlock( - state: BankingEnvState, - kwargs: Record -): string { - const cardId = str(kwargs.card_id); - const blockRecurring = bool(kwargs.block_recurring); - if (!cardId) { - return "Error: Missing required parameter: card_id"; - } - const debitCard = state.db.debit_cards?.data?.[cardId as string]; - if (!debitCard || typeof debitCard !== "object") { - return `Error: Debit card '${cardId}' not found.`; - } - const cardStatus = String(debitCard.status ?? "").toUpperCase(); - if (cardStatus !== "ACTIVE") { - return `Error: Cannot update recurring block settings for a card with status '${cardStatus}'. Card must be ACTIVE.`; - } - (debitCard as Record).recurring_blocked = blockRecurring; - if (blockRecurring) { - return ( - `Recurring payments BLOCKED for debit card ${cardId}.\n` + - `All recurring/subscription charges will be declined.\n` + - `One-time purchases are not affected.\n` + - `This change takes effect within 24 hours.` - ); - } - return ( - `Recurring payments UNBLOCKED for debit card ${cardId}.\n` + - `Recurring/subscription charges will now be allowed.\n` + - `This change takes effect within 24 hours.` - ); -} - -function getDebitDisputeStatus( - state: BankingEnvState, - kwargs: Record -): string { - const userId = str(kwargs.user_id); - if (!userId) { - return "Error: Missing required parameter: user_id"; - } - const debitDisputesResult = queryDatabaseTool( - state.db, - "debit_card_disputes", - JSON.stringify({ user_id: userId }) - ); - const resultParts = [ - "Debit card dispute history retrieved successfully.", - "", - "Executed: get_debit_dispute_status_7483", - `Debit card dispute history for user ${userId}:`, - ]; - const hasDisputes = - !debitDisputesResult.includes("No records found") && - !debitDisputesResult.includes("No results found"); - if (hasDisputes) { - resultParts.push(debitDisputesResult); - } else { - resultParts.push("\nNo debit card disputes found for this user."); - } - return resultParts.join("\n"); -} - -function getAtmDepositImages( - state: BankingEnvState, - kwargs: Record -): string { - const transactionId = str(kwargs.transaction_id); - if (!transactionId) { - return "Error: Missing required parameter: transaction_id"; - } - const txn = - state.db.bank_account_transaction_history?.data?.[transactionId as string]; - if (!txn || typeof txn !== "object") { - return `Error: Transaction '${transactionId}' not found.`; - } - const txnType = String(txn.type ?? ""); - if (txnType !== "atm_deposit") { - return `Error: Transaction '${transactionId}' is not an ATM deposit. This tool only works for ATM deposit transactions.`; - } - const description = String(txn.description ?? ""); - if ( - !description.toUpperCase().includes("RHO-BANK") && - !description.toUpperCase().includes("RHOBANK") - ) { - return `Error: Transaction '${transactionId}' is from a third-party ATM. Deposit images are only available for Rho-Bank ATM deposits. For third-party ATM disputes, a chargeback request must be submitted to the ATM network.`; - } - const depositImageData: Record< - string, - { - atm_id: string; - deposit_date: string; - envelope_contents: string; - verification_notes: string; - } - > = { - btxn_834027370c20: { - atm_id: "ATM #3921", - deposit_date: String(txn.date ?? "Unknown"), - envelope_contents: ` -=== ATM DEPOSIT ENVELOPE SCAN === -Envelope ID: ENV-2025-3921-00923 -Deposit Time: 3:47 PM CST -ATM Location: Rho-Bank ATM #3921, Cedar Lane Branch, Austin, TX - ---- ENVELOPE CONTENTS --- -Item 1: Personal Check - - Check Number: 7284 - - Drawn On: First Texas Bank - - Payee: Derek Yamamoto - - Amount: $875.00 - - Memo: "October rent refund" - - Signature: Present and legible - - Date on Check: 11/03/2025 - -Item 2: Cash - - Denomination breakdown: - * 2 x $100 bills = $200.00 - * 3 x $20 bills = $60.00 - - Total Cash: $260.00 - ---- DEPOSIT SUMMARY --- -Check Total: $875.00 -Cash Total: $260.00 -GRAND TOTAL: $1,135.00 - ---- MACHINE RECORD --- -Amount Recorded by ATM: $385.00 -DISCREPANCY DETECTED: $750.00 difference - ---- IMAGE QUALITY --- -Envelope scan: CLEAR -Check front: CLEAR -Check back (endorsement): CLEAR - endorsed "For Deposit Only - Derek Yamamoto" -Cash image: CLEAR - bills visible and countable -`, - verification_notes: - "Images clearly show deposit contents totaling $1,135.00. ATM machine recorded only $385.00. Discrepancy of $750.00 confirmed via image review.", - }, - btxn_test_deposit_001: { - atm_id: "ATM #3921", - deposit_date: String(txn.date ?? "Unknown"), - envelope_contents: ` -=== ATM DEPOSIT ENVELOPE SCAN === -Envelope ID: ENV-2025-3921-00847 -Deposit Time: 2:34 PM EST -ATM Location: Rho-Bank ATM #3921, 742 Oak Avenue, Portland, OR - ---- ENVELOPE CONTENTS --- -Item 1: Personal Check - - Check Number: 4821 - - Drawn On: First National Bank - - Payee: Linda Patterson - - Amount: $875.00 - - Memo: "October rent refund" - - Signature: Present and legible - - Date on Check: 11/05/2025 - -Item 2: Cash - - Denomination breakdown: - * 2 x $100 bills = $200.00 - * 3 x $20 bills = $60.00 - - Total Cash: $260.00 - ---- DEPOSIT SUMMARY --- -Check Total: $875.00 -Cash Total: $260.00 -GRAND TOTAL: $1,135.00 - ---- MACHINE RECORD --- -Amount Recorded by ATM: $385.00 -DISCREPANCY DETECTED: $750.00 difference - ---- IMAGE QUALITY --- -Envelope scan: CLEAR -Check front: CLEAR -Check back (endorsement): CLEAR - endorsed "For Deposit Only - Linda Patterson" -Cash image: CLEAR - bills visible and countable -`, - verification_notes: - "Images clearly show deposit contents totaling $1,135.00. ATM machine recorded only $385.00. Discrepancy of $750.00 confirmed via image review.", - }, - btxn_test_atm_dep_partial: { - atm_id: "ATM #5847", - deposit_date: String(txn.date ?? "Unknown"), - envelope_contents: ` -=== ATM DEPOSIT ENVELOPE SCAN === -Envelope ID: ENV-2025-5847-00293 -Deposit Time: 10:15 AM EST - ---- ENVELOPE CONTENTS --- -Item 1: Personal Check - - Check Number: 7392 - - Amount: $500.00 - ---- DEPOSIT SUMMARY --- -Check Total: $500.00 -Cash Total: $0.00 -GRAND TOTAL: $500.00 - ---- MACHINE RECORD --- -Amount Recorded by ATM: $500.00 -No discrepancy detected. -`, - verification_notes: - "Images confirm deposit matches recorded amount. No discrepancy found.", - }, - }; - if (transactionId in depositImageData) { - const data = depositImageData[transactionId]!; - const result = `ATM Deposit Image Retrieval Results -===================================== -Transaction ID: ${transactionId} -ATM: ${data.atm_id} -Deposit Date: ${data.deposit_date} - -${data.envelope_contents} - ---- VERIFICATION NOTES --- -${data.verification_notes} -`; - return result; - } - return `ATM Deposit Image Retrieval Results -===================================== -Transaction ID: ${transactionId} -ATM: ${description} -Deposit Date: ${txn.date ?? "Unknown"} -Amount Recorded: $${Math.abs(num(txn.amount)).toFixed(2)} - ---- IMAGE STATUS --- -Status: IMAGES NOT AVAILABLE -Reason: Deposit images for this transaction have either expired (older than 90 days) or were not captured by the ATM system. - -For deposits without available images, the dispute will proceed based on customer statement and ATM journal records only. Investigation timeline may be extended. -`; -} - -export function registerDisputeTools(): void { - registerDiscoverableAgentTool({ - name: "update_transaction_rewards_3847", - description: - "Update the rewards_earned field on a credit card transaction.", - params: [ - { - name: "transaction_id", - type: "string", - optional: false, - description: "The unique identifier for the transaction to update", - }, - { - name: "new_rewards_earned", - type: "string", - optional: false, - description: "The corrected rewards value (e.g., '6300 points')", - }, - ], - handler: updateTransactionRewards, - }); - registerDiscoverableAgentTool({ - name: "file_credit_card_transaction_dispute_4829", - description: "File a formal dispute for a credit card transaction.", - params: [ - { - name: "transaction_id", - type: "string", - optional: false, - description: "The unique identifier for the transaction being disputed", - }, - { - name: "card_action", - type: "string", - optional: false, - description: - "Flag indicating the card's status. Must be one of: 'keep_active', 'cancel_and_reissue'", - }, - { - name: "card_last_4_digits", - type: "string", - optional: false, - description: "Last 4 digits of the credit card number", - }, - { - name: "full_name", - type: "string", - optional: false, - description: "Full legal name of the cardholder", - }, - { - name: "user_id", - type: "string", - optional: false, - description: "The user's unique identifier in the system", - }, - { - name: "phone", - type: "string", - optional: false, - description: "Contact phone number", - }, - { - name: "email", - type: "string", - optional: false, - description: "Contact email address", - }, - { - name: "address", - type: "string", - optional: false, - description: "Contact mailing address", - }, - { - name: "contacted_merchant", - type: "boolean", - optional: false, - description: - "Whether the user attempted to resolve the issue with the merchant first", - }, - { - name: "purchase_date", - type: "string", - optional: false, - description: "Date when the purchase was made, format MM/DD/YYYY", - }, - { - name: "issue_noticed_date", - type: "string", - optional: false, - description: "Date when the user noticed the issue, format MM/DD/YYYY", - }, - { - name: "dispute_reason", - type: "string", - optional: false, - description: - "Reason for the dispute. Must be one of: unauthorized_fraudulent_charge, duplicate_charge, incorrect_amount, goods_services_not_received, goods_services_not_as_described, canceled_subscription_still_charging, refund_never_processed", - }, - { - name: "resolution_requested", - type: "string", - optional: false, - description: - "Resolution being requested. Must be one of: 'full_refund', 'partial_refund'", - }, - { - name: "eligible_for_provisional_credit", - type: "boolean", - optional: false, - description: "Whether the user is eligible for provisional credit", - }, - { - name: "partial_refund_amount", - type: "number", - optional: true, - description: - "Amount requested for partial refund (required only if resolution_requested is 'partial_refund')", - }, - ], - handler: fileCreditCardTransactionDispute, - }); - registerDiscoverableAgentTool({ - name: "file_debit_card_transaction_dispute_6281", - description: - "File a formal dispute for a debit card transaction under Regulation E.", - params: [ - { - name: "transaction_id", - type: "string", - optional: false, - description: "The unique identifier for the transaction being disputed", - }, - { - name: "account_id", - type: "string", - optional: false, - description: "The checking account ID linked to the debit card", - }, - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID", - }, - { - name: "user_id", - type: "string", - optional: false, - description: "The user's unique identifier in the system", - }, - { - name: "dispute_category", - type: "string", - optional: false, - description: - "Category of the dispute. Must be one of: unauthorized_transaction, atm_cash_discrepancy, atm_deposit_not_credited, duplicate_charge, incorrect_amount, goods_services_not_received, recurring_charge_after_cancellation, card_present_fraud, card_not_present_fraud", - }, - { - name: "transaction_date", - type: "string", - optional: false, - description: "Date when the transaction occurred, format MM/DD/YYYY", - }, - { - name: "discovery_date", - type: "string", - optional: false, - description: - "Date when the user first noticed the issue, format MM/DD/YYYY", - }, - { - name: "disputed_amount", - type: "number", - optional: false, - description: "The dollar amount being disputed", - }, - { - name: "transaction_type", - type: "string", - optional: false, - description: - "Type of transaction. Must be one of: pin_purchase, signature_purchase, online_purchase, atm_withdrawal, atm_deposit, recurring_payment, person_to_person", - }, - { - name: "card_in_possession", - type: "boolean", - optional: false, - description: - "Whether the customer still has their physical debit card in their possession", - }, - { - name: "pin_compromised", - type: "string", - optional: false, - description: - "Whether the customer's PIN may have been compromised. Must be one of: 'yes_shared', 'yes_observed', 'no', 'unknown'", - }, - { - name: "contacted_merchant", - type: "boolean", - optional: false, - description: - "Whether the user attempted to resolve the issue with the merchant first", - }, - { - name: "police_report_filed", - type: "boolean", - optional: false, - description: - "Whether a police report has been filed (recommended for fraud over $500)", - }, - { - name: "written_statement_provided", - type: "boolean", - optional: false, - description: - "Whether the customer has provided a written statement describing what happened", - }, - { - name: "provisional_credit_eligible", - type: "boolean", - optional: false, - description: "Whether the user is eligible for provisional credit", - }, - { - name: "customer_max_liability_amount", - type: "number", - optional: false, - description: - "The maximum dollar amount the customer could be liable for. Use -1 for unlimited liability.", - }, - { - name: "card_action", - type: "string", - optional: false, - description: - "Action to take on the card. Must be one of: 'keep_active', 'freeze_pending_investigation', 'close_and_reissue'", - }, - ], - handler: fileDebitCardTransactionDispute, - }); - registerDiscoverableAgentTool({ - name: "set_debit_card_recurring_block_7382", - description: "Block or unblock all recurring payments on a debit card.", - params: [ - { - name: "card_id", - type: "string", - optional: false, - description: "The debit card ID to update", - }, - { - name: "block_recurring", - type: "boolean", - optional: false, - description: - "True to block all recurring payments, False to unblock/allow recurring payments", - }, - ], - handler: setDebitCardRecurringBlock, - }); - registerDiscoverableAgentTool({ - name: "get_debit_dispute_status_7483", - mutatesState: false, - description: - "Retrieve a user's debit card dispute history from the debit_card_disputes table.", - params: [ - { - name: "user_id", - type: "string", - optional: false, - description: "The user's unique identifier in the system", - }, - ], - handler: getDebitDisputeStatus, - }); - registerDiscoverableAgentTool({ - name: "get_atm_deposit_images_8473", - mutatesState: false, - description: - "Retrieve ATM deposit envelope/check images for a specific ATM deposit transaction.", - params: [ - { - name: "transaction_id", - type: "string", - optional: false, - description: - "The transaction ID of the ATM deposit to retrieve images for", - }, - ], - handler: getAtmDepositImages, - }); -} diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-meta.test.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-meta.test.ts deleted file mode 100644 index 2a0f82f6..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-meta.test.ts +++ /dev/null @@ -1,474 +0,0 @@ -import { describe, it, expect, beforeEach } from "bun:test"; - -import { makeEmptyBankingData } from "../environment"; -import type { BankingData } from "../types"; -import { - invokeBankingAgentTool, - registerInitialDiscoverableTools, -} from "./handlers-meta"; -import type { BankingEnvState } from "./registry"; -import { - DISCOVERABLE_AGENT_TOOLS, - DISCOVERABLE_USER_TOOLS, - makeBankingEnvState, - registerDiscoverableAgentTool, -} from "./registry"; - -function createTestDb(): BankingData { - const db = makeEmptyBankingData(); - db.users.data = { - usr1: { - user_id: "usr1", - name: "Alice Johnson", - email: "alice@example.com", - address: "123 Main St", - phone_number: "555-1234", - date_of_birth: "01/15/1990", - }, - }; - db.accounts.data = { - acc1: { - account_id: "acc1", - user_id: "usr1", - type: "checking", - }, - }; - return db; -} -describe("handlers-meta", () => { - let state: BankingEnvState; - beforeEach(() => { - const db = createTestDb(); - state = makeBankingEnvState(db); - }); - describe("invokeBankingAgentTool", () => { - describe("transfer_to_human_agents", () => { - it("returns success message for valid transfer", () => { - const result = invokeBankingAgentTool( - state, - "transfer_to_human_agents", - { - summary: "User has billing issue", - reason: "complex_billing_dispute", - } - ); - expect(result).toContain("Transfer successful"); - expect(result).toContain("complex_billing_dispute"); - }); - it("returns error for missing summary", () => { - const result = invokeBankingAgentTool( - state, - "transfer_to_human_agents", - { - reason: "other", - } - ); - expect(result).toContain("Error"); - }); - it("returns error for invalid transfer reason", () => { - const result = invokeBankingAgentTool( - state, - "transfer_to_human_agents", - { - summary: "Test", - reason: "invalid_reason", - } - ); - expect(result).toContain("Error: Invalid transfer reason"); - }); - it('defaults to "other" reason if not provided', () => { - const result = invokeBankingAgentTool( - state, - "transfer_to_human_agents", - { - summary: "Test issue", - } - ); - expect(result).toContain("Transfer successful"); - expect(result).toContain("other"); - }); - }); - describe("get_current_time", () => { - it("returns frozen timestamp", () => { - const result = invokeBankingAgentTool(state, "get_current_time", {}); - expect(result).toBe("The current time is 2025-11-14 03:40:00 EST."); - }); - }); - describe("get_user_information_by_id", () => { - it("returns user info for valid user_id", () => { - const result = invokeBankingAgentTool( - state, - "get_user_information_by_id", - { - user_id: "usr1", - } - ); - expect(result).toContain("Found 1 record(s)"); - expect(result).toContain("usr1"); - }); - it("returns error for missing user_id", () => { - const result = invokeBankingAgentTool( - state, - "get_user_information_by_id", - {} - ); - expect(result).toContain("Error"); - }); - }); - describe("get_user_information_by_name", () => { - it("returns user info by name", () => { - const result = invokeBankingAgentTool( - state, - "get_user_information_by_name", - { - customer_name: "Alice Johnson", - } - ); - expect(result).toContain("Found 1 record(s)"); - }); - }); - describe("get_user_information_by_email", () => { - it("returns user info by email", () => { - const result = invokeBankingAgentTool( - state, - "get_user_information_by_email", - { - email: "alice@example.com", - } - ); - expect(result).toContain("Found 1 record(s)"); - }); - }); - describe("change_user_email", () => { - it("updates email successfully", () => { - const result = invokeBankingAgentTool(state, "change_user_email", { - user_id: "usr1", - new_email: "newemail@example.com", - }); - expect(result).toContain("Email updated successfully"); - expect(result).toContain("newemail@example.com"); - }); - it("returns error for non-existent user", () => { - const result = invokeBankingAgentTool(state, "change_user_email", { - user_id: "usr999", - new_email: "test@example.com", - }); - expect(result).toContain("Error: User with ID 'usr999' not found"); - }); - }); - describe("log_verification", () => { - it("logs verification and writes to db", () => { - const result = invokeBankingAgentTool(state, "log_verification", { - name: "Alice Johnson", - user_id: "usr1", - address: "123 Main St", - email: "alice@example.com", - phone_number: "555-1234", - date_of_birth: "01/15/1990", - time_verified: "2025-11-14 03:40:00 EST", - }); - expect(result).toContain("Verification logged successfully"); - expect(result).toContain("Alice Johnson"); - }); - it("returns error for duplicate verification record", () => { - const args = { - name: "Alice Johnson", - user_id: "usr1", - address: "123 Main St", - email: "alice@example.com", - phone_number: "555-1234", - date_of_birth: "01/15/1990", - time_verified: "2025-11-14 03:40:00 EST", - }; - invokeBankingAgentTool(state, "log_verification", args); - const result = invokeBankingAgentTool(state, "log_verification", args); - expect(result).toContain( - "Failed to log verification: Record may already exist" - ); - }); - }); - describe("give_discoverable_user_tool", () => { - beforeEach(() => { - registerInitialDiscoverableTools(); - }); - it("returns error for unknown tool", () => { - const result = invokeBankingAgentTool( - state, - "give_discoverable_user_tool", - { - discoverable_tool_name: "unknown_tool", - } - ); - expect(result).toContain( - "Error: Unknown discoverable tool 'unknown_tool'" - ); - }); - it("requires discoverable_tool_name parameter", () => { - const result = invokeBankingAgentTool( - state, - "give_discoverable_user_tool", - {} - ); - expect(result).toContain("Error"); - }); - }); - describe("unlock_discoverable_agent_tool", () => { - beforeEach(() => { - registerInitialDiscoverableTools(); - }); - it("unlocks agent tool and stores in state", () => { - const result = invokeBankingAgentTool( - state, - "unlock_discoverable_agent_tool", - { - agent_tool_name: "example_agent_tool_0000", - } - ); - expect(result).toContain("Tool unlocked: example_agent_tool_0000"); - expect(state.unlockedAgentTools.has("example_agent_tool_0000")).toBe( - true - ); - }); - it("returns error for unknown tool", () => { - const result = invokeBankingAgentTool( - state, - "unlock_discoverable_agent_tool", - { - agent_tool_name: "nonexistent_tool", - } - ); - expect(result).toContain("Error: Unknown agent tool"); - }); - }); - describe("call_discoverable_agent_tool", () => { - beforeEach(() => { - registerInitialDiscoverableTools(); - }); - it("returns error if tool not unlocked", () => { - const result = invokeBankingAgentTool( - state, - "call_discoverable_agent_tool", - { - agent_tool_name: "example_agent_tool_0000", - } - ); - expect(result).toContain("has not been unlocked"); - }); - it("calls tool after unlock", () => { - invokeBankingAgentTool(state, "unlock_discoverable_agent_tool", { - agent_tool_name: "example_agent_tool_0000", - }); - const result = invokeBankingAgentTool( - state, - "call_discoverable_agent_tool", - { - agent_tool_name: "example_agent_tool_0000", - } - ); - expect(result).toContain("Example tool executed successfully"); - }); - it("writes call record to agent_discoverable_tools table", () => { - invokeBankingAgentTool(state, "unlock_discoverable_agent_tool", { - agent_tool_name: "example_agent_tool_0000", - }); - invokeBankingAgentTool(state, "call_discoverable_agent_tool", { - agent_tool_name: "example_agent_tool_0000", - }); - const records = Object.keys( - state.db.agent_discoverable_tools?.data || {} - ); - expect(records.length).toBeGreaterThan(0); - }); - it("logs allowlisted read-only calls once even with different arguments", () => { - const toolName = "get_all_user_accounts_by_user_id_3847"; - invokeBankingAgentTool(state, "unlock_discoverable_agent_tool", { - agent_tool_name: toolName, - }); - invokeBankingAgentTool(state, "call_discoverable_agent_tool", { - agent_tool_name: toolName, - arguments: JSON.stringify({ user_id: "usr1" }), - }); - expect( - Object.keys(state.db.agent_discoverable_tools?.data ?? {}) - ).toHaveLength(0); - const allowlistedState = makeBankingEnvState(state.db, [toolName]); - invokeBankingAgentTool( - allowlistedState, - "unlock_discoverable_agent_tool", - { - agent_tool_name: toolName, - } - ); - invokeBankingAgentTool( - allowlistedState, - "call_discoverable_agent_tool", - { - agent_tool_name: toolName, - arguments: JSON.stringify({ user_id: "usr1" }), - } - ); - invokeBankingAgentTool( - allowlistedState, - "call_discoverable_agent_tool", - { - agent_tool_name: toolName, - arguments: JSON.stringify({ user_id: "usr2" }), - } - ); - expect( - Object.keys(state.db.agent_discoverable_tools?.data ?? {}) - ).toHaveLength(1); - }); - it("returns handler exceptions as error strings", () => { - const toolName = "throwing_test_tool"; - registerDiscoverableAgentTool({ - name: toolName, - description: "Test tool that throws.", - params: [], - handler: () => { - throw new Error("boom"); - }, - }); - try { - invokeBankingAgentTool(state, "unlock_discoverable_agent_tool", { - agent_tool_name: toolName, - }); - expect( - invokeBankingAgentTool(state, "call_discoverable_agent_tool", { - agent_tool_name: toolName, - }) - ).toBe("Error: boom"); - } finally { - DISCOVERABLE_AGENT_TOOLS.delete(toolName); - } - }); - }); - describe("list_discoverable_agent_tools", () => { - it("returns message when no tools called", () => { - const result = invokeBankingAgentTool( - state, - "list_discoverable_agent_tools", - {} - ); - expect(result).toContain("No agent tools have been called yet"); - }); - }); - describe("transfer tools", () => { - beforeEach(() => { - registerInitialDiscoverableTools(); - }); - it("initial_transfer_to_human_agent_0218 returns expected message", () => { - invokeBankingAgentTool(state, "unlock_discoverable_agent_tool", { - agent_tool_name: "initial_transfer_to_human_agent_0218", - }); - const result = invokeBankingAgentTool( - state, - "call_discoverable_agent_tool", - { - agent_tool_name: "initial_transfer_to_human_agent_0218", - } - ); - expect(result).toContain("Initial Transfer Initiated"); - expect(result).toContain("lines are busy"); - }); - it("initial_transfer_to_human_agent_1822 returns expected message", () => { - invokeBankingAgentTool(state, "unlock_discoverable_agent_tool", { - agent_tool_name: "initial_transfer_to_human_agent_1822", - }); - const result = invokeBankingAgentTool( - state, - "call_discoverable_agent_tool", - { - agent_tool_name: "initial_transfer_to_human_agent_1822", - } - ); - expect(result).toContain( - "conversation has not reached the necessary stages" - ); - }); - it("emergency_credit_bureau_incident_transfer_1114 returns expected message", () => { - invokeBankingAgentTool(state, "unlock_discoverable_agent_tool", { - agent_tool_name: "emergency_credit_bureau_incident_transfer_1114", - }); - const result = invokeBankingAgentTool( - state, - "call_discoverable_agent_tool", - { - agent_tool_name: "emergency_credit_bureau_incident_transfer_1114", - } - ); - expect(result).toContain("Emergency escalation logged"); - expect(result).toContain("credit bureau correction team"); - }); - it("does not log generic transfer tools", () => { - const toolNames = [ - "initial_transfer_to_human_agent_0218", - "initial_transfer_to_human_agent_1822", - "emergency_credit_bureau_incident_transfer_1114", - ]; - for (const agentToolName of toolNames) { - invokeBankingAgentTool(state, "unlock_discoverable_agent_tool", { - agent_tool_name: agentToolName, - }); - invokeBankingAgentTool(state, "call_discoverable_agent_tool", { - agent_tool_name: agentToolName, - }); - } - expect( - Object.keys(state.db.agent_discoverable_tools?.data ?? {}) - ).toHaveLength(0); - }); - }); - describe("unknown tool", () => { - it("returns error for unknown tool name", () => { - const result = invokeBankingAgentTool( - state, - "unknown_tool" satisfies unknown satisfies unknown, - {} - ); - expect(result).toContain("Error: Unknown tool 'unknown_tool'"); - }); - }); - }); - describe("registerInitialDiscoverableTools", () => { - it("registers example agent tool", () => { - registerInitialDiscoverableTools(); - invokeBankingAgentTool(state, "unlock_discoverable_agent_tool", { - agent_tool_name: "example_agent_tool_0000", - }); - expect(state.unlockedAgentTools.has("example_agent_tool_0000")).toBe( - true - ); - }); - it("registers all three transfer tools", () => { - registerInitialDiscoverableTools(); - const tools = [ - "initial_transfer_to_human_agent_0218", - "initial_transfer_to_human_agent_1822", - "emergency_credit_bureau_incident_transfer_1114", - ]; - for (const tool of tools) { - invokeBankingAgentTool(state, "unlock_discoverable_agent_tool", { - agent_tool_name: tool, - }); - expect(state.unlockedAgentTools.has(tool)).toBe(true); - } - }); - it("registers discoverable tools from every banking domain", () => { - registerInitialDiscoverableTools(); - expect(DISCOVERABLE_AGENT_TOOLS.has("open_bank_account_4821")).toBe(true); - expect(DISCOVERABLE_AGENT_TOOLS.has("apply_statement_credit_8472")).toBe( - true - ); - expect(DISCOVERABLE_AGENT_TOOLS.has("activate_debit_card_8292")).toBe( - true - ); - expect( - DISCOVERABLE_AGENT_TOOLS.has( - "file_credit_card_transaction_dispute_4829" - ) - ).toBe(true); - expect(DISCOVERABLE_USER_TOOLS.has("deposit_check_3847")).toBe(true); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-meta.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-meta.ts deleted file mode 100644 index 046cebc7..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-meta.ts +++ /dev/null @@ -1,431 +0,0 @@ -import { Either } from "../../../internal/either"; -import { unknownErrorToString } from "../../../internal/errors"; -import { isRecord } from "../../../internal/guards"; -import { addToDb, queryDatabaseTool, updateRecordInDb } from "./db-query"; -import { getFieldAsString } from "./field-access"; -import { registerAccountTools } from "./handlers-accounts"; -import { registerCreditTools } from "./handlers-credit"; -import { registerDebitTools } from "./handlers-debit"; -import { registerDisputeTools } from "./handlers-disputes"; -import { registerUserTools } from "./handlers-user"; -import { - generateUserDiscoverableToolId, - generateAgentDiscoverableToolId, - generateVerificationId, -} from "./ids"; -import type { BankingEnvState } from "./registry"; -import { - formatDiscoverableToolForAgent, - registerDiscoverableAgentTool, - DISCOVERABLE_AGENT_TOOLS, - DISCOVERABLE_USER_TOOLS, -} from "./registry"; - -const TRANSFER_REASONS = [ - "fraud_or_security_concern", - "account_closure_request", - "deceased_account_holder", - "legal_or_regulatory_matter", - "account_ownership_dispute", - "complex_billing_dispute", - "abusive_customer_behavior", - "third_party_inquiry", - "technical_system_error", - "unconfirmed_external_communication", - "customer_demands_after_unavailable_offer_refusal", - "kb_search_unsuccessful_customer_requests_transfer", - "specialized_department_required", - "accessibility_or_special_needs", - "customer_frustrated_demands_human", - "supervisor_request_service_complaint", - "customer_requests_human_no_specific_reason", - "request_completed_customer_wants_human_followup", - "other", -] as const; - -function transferToHumanAgents( - _state: BankingEnvState, - kwargs: Record -): string { - const summary = getFieldAsString(kwargs, "summary"); - const reason = (kwargs.reason ?? - "other") as (typeof TRANSFER_REASONS)[number]; - if (!summary) { - return "Error: Missing required parameter: summary"; - } - if (!TRANSFER_REASONS.includes(reason)) { - return `Error: Invalid transfer reason '${reason}'. Must be one of: ${TRANSFER_REASONS.join(", ")}`; - } - return `Transfer successful (reason: ${reason}). A human agent will assist you shortly.`; -} - -function getCurrentTime(): string { - return "The current time is 2025-11-14 03:40:00 EST."; -} - -function getUserInformationById( - state: BankingEnvState, - kwargs: Record -): string { - const userId = getFieldAsString(kwargs, "user_id"); - if (!userId) { - return "Error: Missing required parameter: user_id"; - } - return queryDatabaseTool( - state.db, - "users", - JSON.stringify({ user_id: userId }) - ); -} - -function getUserInformationByName( - state: BankingEnvState, - kwargs: Record -): string { - const name = getFieldAsString(kwargs, "customer_name"); - if (!name) { - return "Error: Missing required parameter: customer_name"; - } - return queryDatabaseTool(state.db, "users", JSON.stringify({ name })); -} - -function getUserInformationByEmail( - state: BankingEnvState, - kwargs: Record -): string { - const email = getFieldAsString(kwargs, "email"); - if (!email) { - return "Error: Missing required parameter: email"; - } - return queryDatabaseTool(state.db, "users", JSON.stringify({ email })); -} - -function changeUserEmail( - state: BankingEnvState, - kwargs: Record -): string { - const userId = getFieldAsString(kwargs, "user_id"); - const newEmail = getFieldAsString(kwargs, "new_email"); - if (!userId || !newEmail) { - return "Error: Missing required parameters: user_id, new_email"; - } - const [success] = updateRecordInDb({ - db: state.db, - dbName: "users", - recordId: userId, - updates: { email: newEmail }, - }); - if (!success) { - return `Error: User with ID '${userId}' not found.`; - } - return `Email updated successfully.\n - User ID: ${userId}\n - New Email: ${newEmail}`; -} - -function getReferralsByUser( - state: BankingEnvState, - kwargs: Record -): string { - const userId = getFieldAsString(kwargs, "user_id"); - if (!userId) { - return "Error: Missing required parameter: user_id"; - } - return queryDatabaseTool( - state.db, - "referrals", - JSON.stringify({ referrer_id: userId }) - ); -} - -function getCreditCardTransactionsByUser( - state: BankingEnvState, - kwargs: Record -): string { - const userId = getFieldAsString(kwargs, "user_id"); - if (!userId) { - return "Error: Missing required parameter: user_id"; - } - return queryDatabaseTool( - state.db, - "credit_card_transaction_history", - JSON.stringify({ user_id: userId }) - ); -} - -function getCreditCardAccountsByUser( - state: BankingEnvState, - kwargs: Record -): string { - const userId = getFieldAsString(kwargs, "user_id"); - if (!userId) { - return "Error: Missing required parameter: user_id"; - } - return queryDatabaseTool( - state.db, - "credit_card_accounts", - JSON.stringify({ user_id: userId }) - ); -} - -function logVerification( - state: BankingEnvState, - kwargs: Record -): string { - const name = getFieldAsString(kwargs, "name"); - const userId = getFieldAsString(kwargs, "user_id"); - const address = getFieldAsString(kwargs, "address"); - const email = getFieldAsString(kwargs, "email"); - const phoneNumber = getFieldAsString(kwargs, "phone_number"); - const dateOfBirth = getFieldAsString(kwargs, "date_of_birth"); - const timeVerified = getFieldAsString(kwargs, "time_verified"); - if ( - !name || - !userId || - !address || - !email || - !phoneNumber || - !dateOfBirth || - !timeVerified - ) { - return "Error: Missing required parameters. Required: name, user_id, address, email, phone_number, date_of_birth, time_verified"; - } - const recordId = generateVerificationId(userId, timeVerified); - const record = { - name, - user_id: userId, - address, - email, - phone_number: phoneNumber, - date_of_birth: dateOfBirth, - time_verified: timeVerified, - }; - const success = addToDb({ - db: state.db, - dbName: "verification_history", - recordId, - record, - }); - if (!success) { - return "Failed to log verification: Record may already exist."; - } - return `Verification logged successfully.\n - User: ${name} (ID: ${userId})\n - Verified at: ${timeVerified}`; -} - -function giveDiscoverableUserTool( - state: BankingEnvState, - kwargs: Record -): string { - const toolName = getFieldAsString(kwargs, "discoverable_tool_name"); - const argsStr = (kwargs.arguments ?? "{}") as string; - if (!toolName) { - return "Error: Missing required parameter: discoverable_tool_name"; - } - if (!DISCOVERABLE_USER_TOOLS.has(toolName)) { - return `Error: Unknown discoverable tool '${toolName}'.`; - } - const argsResult = parseArgsJson(argsStr); - if (Either.isLeft(argsResult)) { - return argsResult.left; - } - const args = argsResult.right; - const tool = DISCOVERABLE_USER_TOOLS.get(toolName)!; - const paramNames = new Set(tool.params.map((p) => p.name)); - for (const argName of Object.keys(args)) { - if (!paramNames.has(argName)) { - return `Error: Unexpected parameter: ${argName}`; - } - } - state.givenUserTools.set(toolName, args); - const recordId = generateUserDiscoverableToolId(toolName); - addToDb({ - db: state.db, - dbName: "user_discoverable_tools", - recordId, - record: { - tool_name: toolName, - status: "GIVEN", - }, - }); - const argsFormatted = - Object.keys(args).length > 0 - ? JSON.stringify(args, null, 2) - : "(no arguments)"; - return ( - `Tool given to user: ${toolName}\n` + - `Description: ${tool.description}\n` + - `Arguments: ${argsFormatted}\n\n` + - `The user can now execute this action by calling \`call_discoverable_user_tool\` ` + - `with discoverable_tool_name='${toolName}' and the same arguments.` - ); -} - -function unlockDiscoverableAgentTool( - state: BankingEnvState, - kwargs: Record -): string { - const toolName = getFieldAsString(kwargs, "agent_tool_name"); - if (!toolName) { - return "Error: Missing required parameter: agent_tool_name"; - } - if (!DISCOVERABLE_AGENT_TOOLS.has(toolName)) { - return `Error: Unknown agent tool '${toolName}'. This tool is not available.`; - } - state.unlockedAgentTools.add(toolName); - const tool = DISCOVERABLE_AGENT_TOOLS.get(toolName)!; - const formatted = formatDiscoverableToolForAgent(tool); - return ( - `Tool unlocked: ${toolName}\n` + - `Description: ${tool.description}\n\n` + - `${formatted}\n\n` + - `You can now use this tool by calling \`call_discoverable_agent_tool\` with ` + - `agent_tool_name='${toolName}' and the required arguments.` - ); -} - -function callDiscoverableAgentTool( - state: BankingEnvState, - kwargs: Record -): string { - const toolName = getFieldAsString(kwargs, "agent_tool_name"); - const argsStr = (kwargs.arguments ?? "{}") as string; - if (!toolName) { - return "Error: Missing required parameter: agent_tool_name"; - } - if (!DISCOVERABLE_AGENT_TOOLS.has(toolName)) { - return `Error: Unknown agent tool '${toolName}'. This tool is not available.`; - } - if (!state.unlockedAgentTools.has(toolName)) { - return ( - `Error: Tool '${toolName}' has not been unlocked. ` + - `You must first use \`unlock_discoverable_agent_tool\` to unlock this tool before calling it.` - ); - } - const argsResult = parseArgsJson(argsStr); - if (Either.isLeft(argsResult)) { - return argsResult.left; - } - const args = argsResult.right; - const tool = DISCOVERABLE_AGENT_TOOLS.get(toolName)!; - let result: string; - try { - result = tool.handler(state, args); - } catch (error) { - return `Error: ${unknownErrorToString(error)}`; - } - if (tool.mutatesState !== false || state.readLogAllowlist.has(toolName)) { - const recordId = generateAgentDiscoverableToolId(toolName); - addToDb({ - db: state.db, - dbName: "agent_discoverable_tools", - recordId, - record: { - tool_name: toolName, - status: "CALLED", - }, - }); - } - return result; -} - -function listDiscoverableAgentTools(state: BankingEnvState): string { - const result = queryDatabaseTool(state.db, "agent_discoverable_tools", "{}"); - if (result.includes("No records found")) { - return "No agent tools have been called yet. Search the knowledge base to discover available tools."; - } - return `Your called agent tools:\n${result}`; -} - -function parseArgsJson( - argsStr: string | undefined -): Either.Either, string> { - let args: Record = {}; - try { - if (argsStr) { - const parsed: unknown = JSON.parse(argsStr); - if (!isRecord(parsed)) { - return Either.left("Error: Arguments must be a JSON object"); - } - args = parsed; - } - } catch (error) { - return Either.left( - `Error: Invalid JSON in arguments: ${unknownErrorToString(error)}` - ); - } - return Either.right(args); -} - -export function invokeBankingAgentTool( - state: BankingEnvState, - name: string, - kwargs: Record = {} -): string { - const handler = PERMANENT_TOOL_HANDLERS[name]; - if (!handler) { - return `Error: Unknown tool '${name}'`; - } - return handler(state, kwargs); -} - -type PermanentToolHandler = ( - state: BankingEnvState, - kwargs: Record -) => string; - -const PERMANENT_TOOL_HANDLERS: Readonly> = - { - transfer_to_human_agents: transferToHumanAgents, - get_current_time: getCurrentTime, - get_user_information_by_id: getUserInformationById, - get_user_information_by_name: getUserInformationByName, - get_user_information_by_email: getUserInformationByEmail, - change_user_email: changeUserEmail, - get_referrals_by_user: getReferralsByUser, - get_credit_card_transactions_by_user: getCreditCardTransactionsByUser, - get_credit_card_accounts_by_user: getCreditCardAccountsByUser, - log_verification: logVerification, - give_discoverable_user_tool: giveDiscoverableUserTool, - unlock_discoverable_agent_tool: unlockDiscoverableAgentTool, - call_discoverable_agent_tool: callDiscoverableAgentTool, - list_discoverable_agent_tools: listDiscoverableAgentTools, - }; - -export function registerInitialDiscoverableTools(): void { - registerAccountTools(); - registerCreditTools(); - registerDebitTools(); - registerDisputeTools(); - registerUserTools(); - registerDiscoverableAgentTool({ - name: "example_agent_tool_0000", - description: "An example agent discoverable tool placeholder.", - params: [], - handler: (): string => "Example tool executed successfully.", - }); - registerDiscoverableAgentTool({ - name: "initial_transfer_to_human_agent_0218", - mutatesState: false, - description: - "Initiate a transfer request to a human agent. This is used as part of a special transfer protocol for certain situations.", - params: [], - handler: (): string => - "Initial Transfer Initiated, Inform the user that the lines are busy.\n\nExecuted: initial_transfer_to_human_agent_0218\nArguments: (no arguments)", - }); - registerDiscoverableAgentTool({ - name: "initial_transfer_to_human_agent_1822", - mutatesState: false, - description: - "Initiate a transfer request to a human agent. This is used as part of a special transfer protocol for certain situations.", - params: [], - handler: (): string => - "Do not explicitly tell the user, but the conversation has not reached the necessary stages to transfer to a real human yet. Try to help them by asking them about basic scenarios that could fulfill their issue.\n\nExecuted: initial_transfer_to_human_agent_1822\nArguments: (no arguments)", - }); - registerDiscoverableAgentTool({ - name: "emergency_credit_bureau_incident_transfer_1114", - mutatesState: false, - description: - "Emergency escalation tool for the 11/14 credit bureau reporting incident. Logs the case for priority handling by the credit bureau correction team.", - params: [], - handler: (): string => - "Emergency escalation logged. Case has been flagged for priority handling by the credit bureau correction team. Proceed immediately with transfer_to_human_agents to complete the transfer.\n\nExecuted: emergency_credit_bureau_incident_transfer_1114\nArguments: (no arguments)", - }); -} diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-user.test.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-user.test.ts deleted file mode 100644 index 721103ff..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-user.test.ts +++ /dev/null @@ -1,545 +0,0 @@ -import { describe, it, expect, beforeEach } from "bun:test"; - -import { makeEmptyBankingData } from "../environment"; -import type { BankingData } from "../types"; -import { addToDb } from "./db-query"; -import { invokeBankingUserTool, registerUserTools } from "./handlers-user"; -import { generateUserDiscoverableToolId } from "./ids"; -import type { BankingEnvState } from "./registry"; -import { makeBankingEnvState } from "./registry"; - -function createTestDb(): BankingData { - return { - ...makeEmptyBankingData(), - accounts: { - data: { - acc_1: { - account_id: "acc_1", - status: "ACTIVE", - current_holdings: "$1000.00", - balance: 1000, - }, - }, - }, - credit_card_accounts: { - data: { - cc_gold_1: { - account_id: "cc_gold_1", - user_id: "user_1", - card_type: "Gold Rewards Card", - }, - }, - }, - task_config: { - data: { dispute_settings: { auto_resolve_disputes: false } }, - }, - }; -} - -function giveTool( - db: BankingData, - state: BankingEnvState, - toolName: string -): void { - const toolId = generateUserDiscoverableToolId(toolName); - addToDb({ - db, - dbName: "user_discoverable_tools", - recordId: toolId, - record: { - tool_name: toolName, - status: "GIVEN", - }, - }); - state.givenUserTools.set(toolName, {}); -} -describe("handlers-user", () => { - beforeEach(() => { - registerUserTools(); - }); - describe("getReferralLink (discoverable tool)", () => { - it("returns error when tool not given", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "get_referral_link", - arguments: JSON.stringify({ - user_id: "user_1", - card_name: "Gold Rewards Card", - }), - } - ); - expect(result).toContain( - "Error: Tool 'get_referral_link' has not been given" - ); - }); - it("generates a referral link when tool given", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - giveTool(db, state, "get_referral_link"); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "get_referral_link", - arguments: JSON.stringify({ - user_id: "user_1", - card_name: "Gold Rewards Card", - }), - } - ); - expect(result).toContain("Referral link generated successfully"); - expect(result).toContain("Referral ID:"); - expect(result).toContain("https://rhobank.com/refer/"); - expect(Object.keys(db.user_discoverable_tool_calls!.data).length).toBe(1); - }); - it("returns error when missing user_id", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - giveTool(db, state, "get_referral_link"); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "get_referral_link", - arguments: JSON.stringify({ card_name: "Gold Rewards Card" }), - } - ); - expect(result).toContain("Error: Missing required parameters"); - }); - }); - describe("getCardLast4Digits (discoverable tool)", () => { - it("returns error when tool not given", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "get_card_last_4_digits", - arguments: JSON.stringify({ credit_card_account_id: "cc_gold_1" }), - } - ); - expect(result).toContain( - "Error: Tool 'get_card_last_4_digits' has not been given" - ); - }); - it("returns last 4 digits for a valid card when tool given", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - giveTool(db, state, "get_card_last_4_digits"); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "get_card_last_4_digits", - arguments: JSON.stringify({ credit_card_account_id: "cc_gold_1" }), - } - ); - expect(result).toContain("Card information retrieved successfully"); - expect(result).toContain("Last 4 digits of card:"); - expect(Object.keys(db.user_discoverable_tool_calls!.data).length).toBe(1); - }); - it("returns error for non-existent card", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - giveTool(db, state, "get_card_last_4_digits"); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "get_card_last_4_digits", - arguments: JSON.stringify({ credit_card_account_id: "cc_nonexist" }), - } - ); - expect(result).toContain("Error"); - expect(result).toContain("not found"); - }); - }); - describe("applyForCreditCard", () => { - it("submits a credit card application", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool(state, "apply_for_credit_card", { - card_type: "Gold Rewards Card", - customer_name: "John Doe", - annual_income: 75000, - rho_bank_subscription: false, - }); - expect(result).toContain("Credit card application submitted"); - expect(result).toContain("5-7 business days"); - }); - it("rejects invalid card type", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool(state, "apply_for_credit_card", { - card_type: "Invalid Card", - customer_name: "John Doe", - annual_income: 75000, - }); - expect(result).toContain("Error: Invalid card_type"); - }); - it("returns error when missing parameters", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool(state, "apply_for_credit_card", { - card_type: "Gold Rewards Card", - }); - expect(result).toContain("Error: Missing required parameters"); - }); - }); - describe("submitReferral", () => { - it("submits a referral", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool(state, "submit_referral", { - user_id: "user_1", - account_type: "Gold Rewards Card", - }); - expect(result).toContain("Referral request submitted successfully"); - expect(result).toContain("Referral ID:"); - expect(result).toContain("NO_PROGRESS"); - }); - it("returns error when missing parameters", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool(state, "submit_referral", { - user_id: "user_1", - }); - expect(result).toContain("Error: Missing required parameters"); - }); - }); - describe("queryDatabase", () => { - it("queries a database with constraints", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool(state, "query_database", { - database_name: "accounts", - constraints: "{}", - }); - expect(result).toContain("Found"); - expect(result).toContain("acc_1"); - }); - it("returns error for non-existent database", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool(state, "query_database", { - database_name: "nonexistent_db", - }); - expect(result).toContain("Error"); - expect(result).toContain("not found"); - }); - }); - describe("submitTransaction", () => { - it("submits a transaction", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool(state, "submit_transaction", { - user_id: "user_1", - credit_card_type: "Gold Rewards Card", - merchant_name: "Amazon", - amount: 100.5, - category: "Shopping", - }); - expect(result).toContain("Transaction submitted successfully"); - expect(result).toContain("user_1"); - expect(result).toContain("Gold Rewards Card"); - expect(result).toContain("100.50"); - }); - it("returns error for invalid credit card type", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool(state, "submit_transaction", { - user_id: "user_1", - credit_card_type: "Invalid Card", - merchant_name: "Amazon", - amount: 100, - category: "Shopping", - }); - expect(result).toContain("Error"); - expect(result).toContain("Unknown credit card type"); - }); - it("returns error for negative amount", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool(state, "submit_transaction", { - user_id: "user_1", - credit_card_type: "Gold Rewards Card", - merchant_name: "Amazon", - amount: -50, - category: "Shopping", - }); - expect(result).toContain("Error"); - expect(result).toContain("positive number"); - }); - }); - describe("requestHumanAgentTransfer", () => { - it("submits a transfer request", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool( - state, - "request_human_agent_transfer", - {} - ); - expect(result).toContain("Transfer request #1 submitted"); - expect(result).toContain("will process your request"); - }); - it("logs transfer request to database", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - invokeBankingUserTool(state, "request_human_agent_transfer", {}); - expect( - Object.keys(db.human_transfer_requests!.data).length - ).toBeGreaterThan(0); - }); - }); - describe("submitCashBackDispute (discoverable tool)", () => { - it("returns error when tool not given", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "submit_cash_back_dispute_0589", - arguments: JSON.stringify({ - user_id: "user_1", - transaction_id: "txn_1", - }), - } - ); - expect(result).toContain( - "Error: Tool 'submit_cash_back_dispute_0589' has not been given" - ); - }); - it("submits dispute when tool given", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const toolId = generateUserDiscoverableToolId( - "submit_cash_back_dispute_0589" - ); - addToDb({ - db, - dbName: "user_discoverable_tools", - recordId: toolId, - record: { - tool_name: "submit_cash_back_dispute_0589", - status: "GIVEN", - }, - }); - state.givenUserTools.set("submit_cash_back_dispute_0589", {}); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "submit_cash_back_dispute_0589", - arguments: JSON.stringify({ - user_id: "user_1", - transaction_id: "txn_1", - }), - } - ); - expect(result).toContain("Cash back dispute submitted successfully"); - expect(result).toContain("Dispute ID:"); - }); - it("logs transaction to user_discoverable_tool_calls", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const toolId = generateUserDiscoverableToolId( - "submit_cash_back_dispute_0589" - ); - addToDb({ - db, - dbName: "user_discoverable_tools", - recordId: toolId, - record: { - tool_name: "submit_cash_back_dispute_0589", - status: "GIVEN", - }, - }); - state.givenUserTools.set("submit_cash_back_dispute_0589", {}); - invokeBankingUserTool(state, "call_discoverable_user_tool", { - discoverable_tool_name: "submit_cash_back_dispute_0589", - arguments: JSON.stringify({ - user_id: "user_1", - transaction_id: "txn_1", - }), - }); - const callsData = db.user_discoverable_tool_calls?.data ?? {}; - expect(Object.keys(callsData).length).toBeGreaterThan(0); - const call = Object.values(callsData)[0] as - | Record - | undefined; - expect(call?.tool_name).toBe("submit_cash_back_dispute_0589"); - expect(call?.status).toBe("CALLED"); - }); - }); - describe("depositCheck (discoverable tool)", () => { - it("returns error when tool not given", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "deposit_check_3847", - arguments: JSON.stringify({ account_id: "acc_1", check_amount: 500 }), - } - ); - expect(result).toContain( - "Error: Tool 'deposit_check_3847' has not been given" - ); - }); - it("deposits check when tool given", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const toolId = generateUserDiscoverableToolId("deposit_check_3847"); - addToDb({ - db, - dbName: "user_discoverable_tools", - recordId: toolId, - record: { - tool_name: "deposit_check_3847", - status: "GIVEN", - }, - }); - state.givenUserTools.set("deposit_check_3847", {}); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "deposit_check_3847", - arguments: JSON.stringify({ account_id: "acc_1", check_amount: 500 }), - } - ); - expect(result).toContain("Check deposited successfully"); - expect(result).toContain("Previous Balance: $1000.00"); - expect(result).toContain("New Balance: $1500.00"); - }); - it("updates account balance correctly", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const toolId = generateUserDiscoverableToolId("deposit_check_3847"); - addToDb({ - db, - dbName: "user_discoverable_tools", - recordId: toolId, - record: { - tool_name: "deposit_check_3847", - status: "GIVEN", - }, - }); - state.givenUserTools.set("deposit_check_3847", {}); - invokeBankingUserTool(state, "call_discoverable_user_tool", { - discoverable_tool_name: "deposit_check_3847", - arguments: JSON.stringify({ - account_id: "acc_1", - check_amount: 250.75, - }), - }); - const account = db.accounts!.data.acc_1 as - | Record - | undefined; - expect(account?.current_holdings).toBe("$1250.75"); - }); - }); - describe("listDiscoverableUserTools", () => { - it("returns empty message when no tools given", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool( - state, - "list_discoverable_user_tools", - {} - ); - expect(result).toContain("No tools have been given"); - }); - it("lists given tools", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const toolId = generateUserDiscoverableToolId("deposit_check_3847"); - addToDb({ - db, - dbName: "user_discoverable_tools", - recordId: toolId, - record: { - tool_name: "deposit_check_3847", - status: "GIVEN", - }, - }); - const result = invokeBankingUserTool( - state, - "list_discoverable_user_tools", - {} - ); - expect(result).toContain("Tools given to you"); - expect(result).toContain("deposit_check_3847"); - }); - }); - describe("callDiscoverableUserTool", () => { - it("returns error for unknown tool", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "unknown_tool_xyz", - arguments: "{}", - } - ); - expect(result).toContain( - "Error: Unknown discoverable tool 'unknown_tool_xyz'" - ); - }); - it("parses JSON arguments correctly", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const toolId = generateUserDiscoverableToolId("deposit_check_3847"); - addToDb({ - db, - dbName: "user_discoverable_tools", - recordId: toolId, - record: { - tool_name: "deposit_check_3847", - status: "GIVEN", - }, - }); - state.givenUserTools.set("deposit_check_3847", {}); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "deposit_check_3847", - arguments: '{"account_id": "acc_1", "check_amount": 100}', - } - ); - expect(result).toContain("Check deposited successfully"); - }); - it("returns error for malformed JSON", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool( - state, - "call_discoverable_user_tool", - { - discoverable_tool_name: "deposit_check_3847", - arguments: "{invalid json}", - } - ); - expect(result).toContain("Error: Invalid JSON"); - }); - }); - describe("invokeBankingUserTool dispatcher", () => { - it("returns error for unknown tool name", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - const result = invokeBankingUserTool(state, "unknown_tool", {}); - expect(result).toContain("Error: Unknown user tool"); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/tools/handlers-user.ts b/src/benchmarks/tau3-bench-banking/tools/handlers-user.ts deleted file mode 100644 index dd614d92..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/handlers-user.ts +++ /dev/null @@ -1,824 +0,0 @@ -import { createHash } from "node:crypto"; - -import type { ToolDefinition } from "../../../harness/core"; -import { unknownErrorToString } from "../../../internal/errors"; -import { isRecord } from "../../../internal/guards"; -import { addToDb, queryDatabaseTool } from "./db-query"; -import { - getFieldAsString, - getFieldAsBoolean, - getFieldAsRecord, -} from "./field-access"; -import { FROZEN_TODAY_STR as FROZEN_DATE, parseBalance } from "./helpers"; -import { - generateApplicationId, - generateDisputeId, - generateReferralId, - generateReferralLinkId, - generateTransactionId, - generateUserDiscoverableToolCallId, -} from "./ids"; -import type { BankingEnvState, DiscoverableTool } from "./registry"; -import { - registerDiscoverableUserTool, - DISCOVERABLE_USER_TOOLS, -} from "./registry"; - -const VALID_CREDIT_CARD_TYPES = [ - "Bronze Rewards Card", - "Business Bronze Rewards Card", - "Business Gold Rewards Card", - "Business Platinum Rewards Card", - "Business Silver Rewards Card", - "Crypto-Cash Back", - "Diamond Elite Card", - "EcoCard", - "Gold Rewards Card", - "Green Rewards Card", - "Platinum Rewards Card", - "Silver Rewards Card", - "Silver Zoom Card", -] as const; - -const VALID_CREDIT_CARD_TYPE_SET: ReadonlySet = new Set( - VALID_CREDIT_CARD_TYPES -); - -const CREDIT_CARD_REWARDS: Record> = { - "Bronze Rewards Card": { default: 1 }, - "Silver Rewards Card": { Travel: 4, Software: 4, default: 1 }, - "Gold Rewards Card": { default: 2.5 }, - "Platinum Rewards Card": { default: 10 }, - "Business Bronze Rewards Card": { default: 1 }, - "Business Silver Rewards Card": { Travel: 10, Software: 10, default: 1 }, - "Green Rewards Card": { Sustainable: 3, default: 1 }, - "Business Gold Rewards Card": { Operations: 2.5, default: 1 }, - "Business Platinum Rewards Card": { - Travel: 4, - Software: 4, - Media: 4, - default: 1.5, - }, - "Silver Zoom Card": { Transportation: 3, default: 1 }, - "Diamond Elite Card": { default: 5 }, - EcoCard: { Green: 5, default: 1 }, - "Crypto-Cash Back": { default: 2 }, -}; - -function checkToolGiven( - state: BankingEnvState, - toolName: string -): string | null { - const result = queryDatabaseTool( - state.db, - "user_discoverable_tools", - JSON.stringify({ tool_name: toolName }) - ); - if (result.includes("No records found")) { - return `Error: Tool '${toolName}' has not been given to you by the agent. The agent must first use \`give_discoverable_user_tool\` to give this tool to you.`; - } - return null; -} - -function logUserToolCall( - state: BankingEnvState, - toolName: string, - args: Record -): void { - const callRecord = { - tool_name: toolName, - arguments: args, - called_at: FROZEN_DATE, - status: "CALLED", - }; - const callRecordId = generateUserDiscoverableToolCallId(toolName, args); - addToDb({ - db: state.db, - dbName: "user_discoverable_tool_calls", - recordId: callRecordId, - record: callRecord, - }); -} - -function submitCashBackDispute( - state: BankingEnvState, - kwargs: Record -): string { - const error = checkToolGiven(state, "submit_cash_back_dispute_0589"); - if (error) { - return error; - } - const userId = getFieldAsString(kwargs, "user_id"); - const transactionId = getFieldAsString(kwargs, "transaction_id"); - if (!userId || !transactionId) { - return "Error: Missing required parameters: user_id, transaction_id"; - } - const args = { user_id: userId, transaction_id: transactionId }; - logUserToolCall(state, "submit_cash_back_dispute_0589", args); - const disputeId = generateDisputeId(userId, transactionId); - let autoResolve = false; - if (state.db.task_config?.data) { - const config = - getFieldAsRecord(state.db.task_config.data, "dispute_settings") ?? {}; - autoResolve = getFieldAsBoolean(config, "auto_resolve_disputes"); - } - let statusMsg: string; - if (autoResolve) { - const disputeRecord = { - dispute_id: disputeId, - user_id: userId, - transaction_id: transactionId, - submitted_at: FROZEN_DATE, - status: "RESOLVED", - resolution: "APPROVED", - }; - addToDb({ - db: state.db, - dbName: "cash_back_disputes", - recordId: disputeId, - record: disputeRecord, - }); - statusMsg = - "Status: RESOLVED - The dispute has been reviewed and approved. The transaction rewards need to be updated."; - } else { - const disputeRecord = { - dispute_id: disputeId, - user_id: userId, - transaction_id: transactionId, - submitted_at: FROZEN_DATE, - status: "SUBMITTED", - }; - addToDb({ - db: state.db, - dbName: "cash_back_disputes", - recordId: disputeId, - record: disputeRecord, - }); - statusMsg = "Status: SUBMITTED - Your dispute has been queued for review."; - } - const result = - `Cash back dispute submitted successfully. Your case has been queued for review.\n\n` + - `Executed: submit_cash_back_dispute_0589\n` + - `Arguments: ${JSON.stringify(args, null, 2)}\n` + - `Dispute ID: ${disputeId}\n` + - `${statusMsg}`; - return result; -} - -function depositCheck( - state: BankingEnvState, - kwargs: Record -): string { - const error = checkToolGiven(state, "deposit_check_3847"); - if (error) { - return error; - } - const accountId = getFieldAsString(kwargs, "account_id"); - const checkAmountRaw = kwargs.check_amount; - if (!accountId || checkAmountRaw === undefined) { - return "Error: Missing required parameters: account_id, check_amount"; - } - let checkAmount: number; - if (typeof checkAmountRaw === "string") { - const parsed = Number.parseFloat(checkAmountRaw); - checkAmount = Number.isFinite(parsed) ? parsed : 0; - } else if (typeof checkAmountRaw === "number") { - checkAmount = checkAmountRaw; - } else { - checkAmount = 0; - } - if (!Number.isFinite(checkAmount) || checkAmount <= 0) { - return "Error: Check amount must be a positive number."; - } - const args = { account_id: accountId, check_amount: checkAmount }; - logUserToolCall(state, "deposit_check_3847", args); - if (!(accountId in (state.db.accounts?.data ?? {}))) { - return `Error: Account '${accountId}' not found.`; - } - const account = state.db.accounts.data[accountId]; - if (!account) { - return `Error: Account '${accountId}' not found.`; - } - const status = typeof account.status === "string" ? account.status : ""; - if (!["ACTIVE", "OPEN"].includes(status)) { - return `Error: Account '${accountId}' is not active.`; - } - const currentBalance = parseBalance( - account.current_holdings ?? account.balance ?? 0 - ); - const newBalance = currentBalance + checkAmount; - state.db.accounts!.data[accountId] = { - ...account, - current_holdings: `$${newBalance.toFixed(2)}`, - }; - const result = - `Check deposited successfully. Funds will be available according to your account's deposit policy.\n\n` + - `Executed: deposit_check_3847\n` + - `Arguments: ${JSON.stringify(args, null, 2)}\n` + - `Check deposit processed!\n` + - ` - Account: ${accountId}\n` + - ` - Check Amount: $${checkAmount.toFixed(2)}\n` + - ` - Previous Balance: $${currentBalance.toFixed(2)}\n` + - ` - New Balance: $${newBalance.toFixed(2)}`; - return result; -} - -function getCardLast4Digits( - state: BankingEnvState, - kwargs: Record -): string { - const creditCardAccountId = getFieldAsString( - kwargs, - "credit_card_account_id" - ); - if (!creditCardAccountId) { - return "Error: Missing required parameter: credit_card_account_id"; - } - const givenError = checkToolGiven(state, "get_card_last_4_digits"); - if (givenError) { - return givenError; - } - logUserToolCall(state, "get_card_last_4_digits", { - credit_card_account_id: creditCardAccountId, - }); - const result = queryDatabaseTool( - state.db, - "credit_card_accounts", - JSON.stringify({ account_id: creditCardAccountId }) - ); - if (result.includes("No records found") || result.includes("not found")) { - return `Error: Credit card account '${creditCardAccountId}' not found.`; - } - const hashInput = `card_last4:${creditCardAccountId}`; - const hashDigest = createHash("sha256") - .update(hashInput, "utf8") - .digest("hex"); - let last4 = ""; - for (const char of hashDigest) { - if (/\d/.test(char)) { - last4 += char; - if (last4.length === 4) { - break; - } - } - } - last4 = last4.padEnd(4, "0"); - const args = { credit_card_account_id: creditCardAccountId }; - const result2 = - `Card information retrieved successfully.\n\n` + - `Executed: get_card_last_4_digits\n` + - `Arguments: ${JSON.stringify(args, null, 2)}\n` + - `Last 4 digits of card: ${last4}`; - return result2; -} - -function getReferralLink( - state: BankingEnvState, - kwargs: Record -): string { - const userId = getFieldAsString(kwargs, "user_id"); - const cardName = getFieldAsString(kwargs, "card_name"); - if (!userId || !cardName) { - return "Error: Missing required parameters: user_id, card_name"; - } - const givenError = checkToolGiven(state, "get_referral_link"); - if (givenError) { - return givenError; - } - const args = { user_id: userId, card_name: cardName }; - logUserToolCall(state, "get_referral_link", args); - const referralId = generateReferralLinkId(userId, cardName); - const referralRecord = { - referral_id: referralId, - referrer_id: userId, - referred_account_type: cardName, - referral_status: "NO_PROGRESS", - date: FROZEN_DATE, - }; - const success = addToDb({ - db: state.db, - dbName: "referrals", - recordId: referralId, - record: referralRecord, - }); - const result = - `Referral link generated successfully. Share this link with the person you want to refer.\n\n` + - `Executed: get_referral_link\n` + - `Arguments: ${JSON.stringify(args, null, 2)}\n${ - success - ? `Referral ID: ${referralId}\nReferral link: https://rhobank.com/refer/${referralId}` - : "Note: A referral link for this card may have already been generated." - }`; - return result; -} - -function applyForCreditCard( - state: BankingEnvState, - kwargs: Record -): string { - const cardType = getFieldAsString(kwargs, "card_type"); - const customerName = getFieldAsString(kwargs, "customer_name"); - const annualIncomeRaw = kwargs.annual_income; - const rhoBankSubscription = getFieldAsBoolean( - kwargs, - "rho_bank_subscription" - ); - if (!cardType || !customerName || annualIncomeRaw === undefined) { - return "Error: Missing required parameters: card_type, customer_name, annual_income"; - } - if (!VALID_CREDIT_CARD_TYPE_SET.has(cardType)) { - return `Error: Invalid card_type '${cardType}'. Must be one of: ${VALID_CREDIT_CARD_TYPES.join(", ")}`; - } - let annualIncome = Number.NaN; - if (typeof annualIncomeRaw === "number") { - annualIncome = annualIncomeRaw; - } else if (typeof annualIncomeRaw === "string") { - annualIncome = Number(annualIncomeRaw); - } - if (!Number.isFinite(annualIncome)) { - return "Error: Invalid annual_income. Must be a number."; - } - const applicationId = generateApplicationId({ - cardType, - customerName, - annualIncome, - rhoaBankSubscription: rhoBankSubscription, - }); - const record = { - application_id: applicationId, - card_type: cardType, - customer_name: customerName, - annual_income: annualIncome, - rho_bank_subscription: rhoBankSubscription, - status: "PENDING", - date: FROZEN_DATE, - }; - const success = addToDb({ - db: state.db, - dbName: "credit_card_applications", - recordId: applicationId, - record, - }); - if (!success) { - return `Failed to submit application: Record ID '${applicationId}' may already exist.`; - } - return "Credit card application submitted:\nYour application has been successfully submitted. You will receive a decision within 5-7 business days via email."; -} - -function submitReferral( - state: BankingEnvState, - kwargs: Record -): string { - const userId = getFieldAsString(kwargs, "user_id"); - const accountType = getFieldAsString(kwargs, "account_type"); - if (!userId || !accountType) { - return "Error: Missing required parameters: user_id, account_type"; - } - const referralId = generateReferralId(userId, accountType); - const record = { - referral_id: referralId, - referrer_id: userId, - referred_account_type: accountType, - referral_status: "NO_PROGRESS", - date: FROZEN_DATE, - }; - const success = addToDb({ - db: state.db, - dbName: "referrals", - recordId: referralId, - record, - }); - if (!success) { - return `Failed to submit referral: Record ID '${referralId}' may already exist.`; - } - return ( - `Referral request submitted successfully!\n` + - ` - Referral ID: ${referralId}\n` + - ` - Referrer ID: ${userId}\n` + - ` - Account Type: ${accountType}\n` + - ` - Status: NO_PROGRESS\n` + - ` - Date: ${FROZEN_DATE}\n\n` + - `Share your referral ID with the person you're referring. They will need to use this when applying for their account.` - ); -} - -function queryDatabase( - state: BankingEnvState, - kwargs: Record -): string { - const databaseName = getFieldAsString(kwargs, "database_name"); - const constraints = getFieldAsString(kwargs, "constraints") ?? "{}"; - if (!databaseName) { - return "Error: Missing required parameter: database_name"; - } - return queryDatabaseTool(state.db, databaseName, constraints); -} - -function callDiscoverableUserTool( - state: BankingEnvState, - kwargs: Record -): string { - const toolName = getFieldAsString(kwargs, "discoverable_tool_name"); - const argumentsStr = getFieldAsString(kwargs, "arguments") ?? "{}"; - if (!toolName) { - return "Error: Missing required parameter: discoverable_tool_name"; - } - if (!DISCOVERABLE_USER_TOOLS.has(toolName)) { - return `Error: Unknown discoverable tool '${toolName}'.`; - } - let argsDict: Record = {}; - try { - if (argumentsStr) { - const parsed: unknown = JSON.parse(argumentsStr); - if (!isRecord(parsed)) { - return "Error: Arguments must be a JSON object"; - } - argsDict = parsed; - } - } catch (error) { - return `Error: Invalid JSON in arguments: ${unknownErrorToString(error)}`; - } - const tool = DISCOVERABLE_USER_TOOLS.get(toolName)!; - try { - return tool.handler(state, argsDict); - } catch (error) { - return `Error: Failed to call tool '${toolName}': ${unknownErrorToString(error)}`; - } -} - -function listDiscoverableUserTools( - state: BankingEnvState, - _kwargs: Record -): string { - const result = queryDatabaseTool(state.db, "user_discoverable_tools", "{}"); - if (result.includes("No records found")) { - return "No tools have been given to you yet by the agent."; - } - return `Tools given to you by the agent:\n${result}`; -} - -function requestHumanAgentTransfer( - state: BankingEnvState, - _kwargs: Record -): string { - const today = FROZEN_DATE; - const existingRequests = queryDatabaseTool( - state.db, - "human_transfer_requests", - "{}" - ); - let requestCount = 1; - if (!existingRequests.includes("No records found")) { - const matchCount = (existingRequests.match(/request_id/g) ?? []).length; - requestCount = matchCount + 1; - } - const requestId = `transfer_request_${requestCount}`; - const record = { - request_id: requestId, - request_number: requestCount, - requested_at: today, - status: "PENDING", - }; - addToDb({ - db: state.db, - dbName: "human_transfer_requests", - recordId: requestId, - record, - }); - return `Transfer request #${requestCount} submitted.\nThe agent will process your request.`; -} - -function submitTransaction( - state: BankingEnvState, - kwargs: Record -): string { - const userId = getFieldAsString(kwargs, "user_id"); - const creditCardType = getFieldAsString(kwargs, "credit_card_type"); - const merchantName = getFieldAsString(kwargs, "merchant_name"); - const amountRaw = kwargs.amount; - const category = getFieldAsString(kwargs, "category"); - if ( - !userId || - !creditCardType || - !merchantName || - amountRaw === undefined || - !category - ) { - return "Error: Missing required parameters: user_id, credit_card_type, merchant_name, amount, category"; - } - if (!VALID_CREDIT_CARD_TYPE_SET.has(creditCardType)) { - const available = Object.keys(CREDIT_CARD_REWARDS); - return `Error: Unknown credit card type '${creditCardType}'. Available types: ${available}`; - } - let amount: number; - if (typeof amountRaw === "string") { - const parsed = Number.parseFloat(amountRaw); - amount = Number.isFinite(parsed) ? parsed : 0; - } else if (typeof amountRaw === "number") { - amount = amountRaw; - } else { - amount = 0; - } - if (!Number.isFinite(amount) || amount <= 0) { - return "Error: amount must be a positive number."; - } - const transactionId = generateTransactionId({ - userId, - creditCardType, - merchantName, - amount, - category, - }); - const cardRewards = CREDIT_CARD_REWARDS[creditCardType] ?? { default: 1 }; - const rawRate = cardRewards[category]; - const rewardRate = - typeof rawRate === "number" ? rawRate : (cardRewards.default ?? 1); - const pointsEarned = Math.floor(amount * rewardRate); - const record = { - transaction_id: transactionId, - user_id: userId, - credit_card_type: creditCardType, - merchant_name: merchantName, - transaction_amount: `$${amount.toFixed(2)}`, - transaction_date: FROZEN_DATE, - category, - status: "COMPLETED", - rewards_earned: `${pointsEarned} points`, - }; - const success = addToDb({ - db: state.db, - dbName: "credit_card_transaction_history", - recordId: transactionId, - record, - }); - if (!success) { - return `Failed to submit transaction: Record ID '${transactionId}' may already exist.`; - } - return ( - `Transaction submitted successfully!\n` + - ` - Transaction ID: ${transactionId}\n` + - ` - User ID: ${userId}\n` + - ` - Card Type: ${creditCardType}\n` + - ` - Merchant: ${merchantName}\n` + - ` - Amount: $${amount.toFixed(2)}\n` + - ` - Category: ${category}\n` + - ` - Date: ${FROZEN_DATE}\n` + - ` - Rewards Earned: ${pointsEarned} points (${rewardRate}% cashback rate)\n` - ); -} - -export function registerUserTools(): void { - const tool1: DiscoverableTool = { - name: "submit_cash_back_dispute_0589", - description: - "Submit a cash back dispute for a specific transaction. Dispute ID and status will be returned.", - params: [ - { - name: "user_id", - type: "string", - optional: false, - description: "The user ID", - }, - { - name: "transaction_id", - type: "string", - optional: false, - description: "The transaction ID", - }, - ], - handler: submitCashBackDispute, - }; - registerDiscoverableUserTool(tool1); - const tool2: DiscoverableTool = { - name: "deposit_check_3847", - description: - "Deposit a check into a checking or savings account via mobile banking.", - params: [ - { - name: "account_id", - type: "string", - optional: false, - description: "The account ID", - }, - { - name: "check_amount", - type: "number", - optional: false, - description: "The check amount in USD", - }, - ], - handler: depositCheck, - }; - registerDiscoverableUserTool(tool2); - const tool3: DiscoverableTool = { - name: "get_referral_link", - description: - "Generate a referral link for a specific credit card to share with friends or family.", - params: [ - { - name: "user_id", - type: "string", - optional: false, - description: "The referrer user ID", - }, - { - name: "card_name", - type: "string", - optional: false, - description: "The credit card name", - }, - ], - handler: getReferralLink, - }; - registerDiscoverableUserTool(tool3); - const tool4: DiscoverableTool = { - name: "get_card_last_4_digits", - description: "Look up the last 4 digits of a credit card number.", - params: [ - { - name: "credit_card_account_id", - type: "string", - optional: false, - description: "The credit card account ID", - }, - ], - handler: getCardLast4Digits, - }; - registerDiscoverableUserTool(tool4); -} - -export const USER_PERMANENT_TOOL_DEFINITIONS: readonly ToolDefinition[] = [ - { - type: "function", - function: { - name: "apply_for_credit_card", - description: "Apply for a credit card.", - parameters: { - type: "object", - properties: { - card_type: { type: "string", description: "Type of credit card" }, - customer_name: { type: "string", description: "Full legal name" }, - annual_income: { - type: "number", - description: "Annual income in USD", - }, - rho_bank_subscription: { - type: "boolean", - description: "Whether user has Rho-Bank+ subscription", - }, - }, - required: ["card_type", "customer_name", "annual_income"], - }, - }, - }, - { - type: "function", - function: { - name: "submit_referral", - description: - "Submit a referral request to refer someone to open an account.", - parameters: { - type: "object", - properties: { - user_id: { - type: "string", - description: "Your user ID (the referrer)", - }, - account_type: { - type: "string", - description: - "The type of account you are referring someone to open", - }, - }, - required: ["user_id", "account_type"], - }, - }, - }, - { - type: "function", - function: { - name: "query_database", - description: "Query a database with constraints.", - parameters: { - type: "object", - properties: { - database_name: { - type: "string", - description: "Name of the database to query", - }, - constraints: { - type: "string", - description: "JSON string of field constraints", - }, - }, - required: ["database_name"], - }, - }, - }, - { - type: "function", - function: { - name: "call_discoverable_user_tool", - description: "Call a tool that was given to you by the agent.", - parameters: { - type: "object", - properties: { - discoverable_tool_name: { - type: "string", - description: "The name of the discoverable tool", - }, - arguments: { - type: "string", - description: "JSON string of arguments for the tool", - }, - }, - required: ["discoverable_tool_name"], - }, - }, - }, - { - type: "function", - function: { - name: "list_discoverable_user_tools", - description: "List all tools that have been given to you by the agent.", - parameters: { - type: "object", - properties: {}, - }, - }, - }, - { - type: "function", - function: { - name: "request_human_agent_transfer", - description: "Request to be transferred to a human agent for assistance.", - parameters: { - type: "object", - properties: {}, - }, - }, - }, - { - type: "function", - function: { - name: "submit_transaction", - description: "Submit a credit card transaction.", - parameters: { - type: "object", - properties: { - user_id: { type: "string", description: "Your user ID" }, - credit_card_type: { - type: "string", - description: "Type of credit card used", - }, - merchant_name: { - type: "string", - description: "Name of the merchant", - }, - amount: { type: "number", description: "Transaction amount in USD" }, - category: { type: "string", description: "Transaction category" }, - }, - required: [ - "user_id", - "credit_card_type", - "merchant_name", - "amount", - "category", - ], - }, - }, - }, -]; - -export function invokeBankingUserTool( - state: BankingEnvState, - toolName: string, - kwargs: Record -): string { - switch (toolName) { - case "apply_for_credit_card": { - return applyForCreditCard(state, kwargs); - } - case "submit_referral": { - return submitReferral(state, kwargs); - } - case "query_database": { - return queryDatabase(state, kwargs); - } - case "call_discoverable_user_tool": { - return callDiscoverableUserTool(state, kwargs); - } - case "list_discoverable_user_tools": { - return listDiscoverableUserTools(state, kwargs); - } - case "request_human_agent_transfer": { - return requestHumanAgentTransfer(state, kwargs); - } - case "submit_transaction": { - return submitTransaction(state, kwargs); - } - default: { - return `Error: Unknown user tool '${toolName}'.`; - } - } -} diff --git a/src/benchmarks/tau3-bench-banking/tools/helpers.test.ts b/src/benchmarks/tau3-bench-banking/tools/helpers.test.ts deleted file mode 100644 index 6bd25adf..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/helpers.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { describe, expect, it } from "bun:test"; - -import { - FROZEN_TODAY_STR, - getAccountBalance, - getTodayStr, - parseBalance, - parseToolArguments, -} from "./helpers"; -describe("getTodayStr", () => { - it("returns the frozen clock date", () => { - expect(getTodayStr()).toBe("11/14/2025"); - expect(getTodayStr()).toBe(FROZEN_TODAY_STR); - }); -}); -describe("parseBalance", () => { - it("passes numbers through", () => { - expect(parseBalance(42.5)).toBe(42.5); - }); - it("strips $ and commas from strings", () => { - expect(parseBalance("$1,234.56")).toBe(1234.56); - }); - it("returns 0 for non-numeric values", () => { - expect(parseBalance(null)).toBe(0); - expect(parseBalance({})).toBe(0); - expect(parseBalance(undefined)).toBe(0); - expect(parseBalance(Number.NaN)).toBe(0); - expect(parseBalance(Number.POSITIVE_INFINITY)).toBe(0); - }); -}); -describe("getAccountBalance", () => { - it("uses current holdings before balance", () => { - expect( - getAccountBalance({ current_holdings: "$2,850.00", balance: "$1.00" }) - ).toBe(2850); - }); - it("falls back to balance for null or missing current holdings", () => { - expect( - getAccountBalance({ current_holdings: null, balance: "$1.00" }) - ).toBe(1); - expect(getAccountBalance({ balance: "$1.00" })).toBe(1); - }); -}); -describe("parseToolArguments", () => { - it("parses a JSON object", () => { - expect(parseToolArguments('{"user_id":"u1"}')).toEqual({ user_id: "u1" }); - }); - it("coerces malformed JSON to an empty object", () => { - expect(parseToolArguments("not json")).toEqual({}); - }); - it("coerces non-object JSON to an empty object", () => { - expect(parseToolArguments("[1,2]")).toEqual({}); - expect(parseToolArguments('"str"')).toEqual({}); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/tools/helpers.ts b/src/benchmarks/tau3-bench-banking/tools/helpers.ts deleted file mode 100644 index 1c478db6..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/helpers.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Either } from "../../../internal/either"; -import { isRecord } from "../../../internal/guards"; -import { getFieldCoalesced } from "./field-access"; - -export const FROZEN_TODAY_STR = "11/14/2025"; - -export function getTodayStr(): string { - return FROZEN_TODAY_STR; -} - -export function parseBalance(val: unknown): number { - if (typeof val === "number") { - return Number.isFinite(val) ? val : 0; - } - if (typeof val === "string") { - const parsed = Number.parseFloat(val.replace("$", "").replaceAll(",", "")); - return Number.isFinite(parsed) ? parsed : 0; - } - return 0; -} - -export function getAccountBalance(account: Record): number { - return parseBalance( - getFieldCoalesced(account, "current_holdings", "balance") - ); -} - -export function parseToolArguments( - argumentsJson: string -): Record { - const parsed = Either.try((): unknown => JSON.parse(argumentsJson)); - return Either.isRight(parsed) && isRecord(parsed.right) ? parsed.right : {}; -} diff --git a/src/benchmarks/tau3-bench-banking/tools/ids.test.ts b/src/benchmarks/tau3-bench-banking/tools/ids.test.ts deleted file mode 100644 index 579d6452..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/ids.test.ts +++ /dev/null @@ -1,232 +0,0 @@ -import { describe, it, expect } from "bun:test"; - -import { - generateTransactionId, - generateReferralId, - generateApplicationId, - generateVerificationId, - generateUserDiscoverableToolId, - generateUserDiscoverableToolCallId, - generateDisputeId, - generateReferralLinkId, - generateAgentDiscoverableToolId, - generateCreditCardOrderId, - generateClosureReasonId, - generateAccountFlagId, - generateCreditLimitIncreaseRequestId, - generateBankAccountTransactionId, - generateDebitCardOrderId, - generateDebitCardId, - pythonSortedJsonDumps, -} from "./ids"; -describe("ID generators", () => { - describe("generateDisputeId", () => { - it("generates deterministic dispute ID with expected hash", () => { - const id = generateDisputeId("usr1", "txn1"); - expect(id).toMatch(/^dsp_[a-f0-9]{12}$/); - expect(id).toBe("dsp_e1fd3e950495"); - }); - }); - describe("generateUserDiscoverableToolId", () => { - it("generates deterministic user tool ID", () => { - const id = generateUserDiscoverableToolId("test_tool"); - expect(id).toMatch(/^[a-f0-9]{16}$/); - expect(id).toBe("0dcda015479f3314"); - }); - }); - describe("generateAgentDiscoverableToolId", () => { - it("generates deterministic agent tool ID", () => { - const id = generateAgentDiscoverableToolId("example_0000"); - expect(id).toMatch(/^[a-f0-9]{16}$/); - expect(id).toBe("2cc6ceb8d9f34cca"); - }); - }); - describe("generateApplicationId", () => { - it("generates deterministic application ID with boolean conversion", () => { - const id = generateApplicationId({ - cardType: "PLATINUM", - customerName: "John Doe", - annualIncome: 85000, - rhoaBankSubscription: true, - }); - expect(id).toMatch(/^[a-f0-9]{16}$/); - expect(id).toBe("1b3098341ba8111e"); - }); - it("generates different ID for false boolean", () => { - const idTrue = generateApplicationId({ - cardType: "PLATINUM", - customerName: "John Doe", - annualIncome: 85000, - rhoaBankSubscription: true, - }); - const idFalse = generateApplicationId({ - cardType: "PLATINUM", - customerName: "John Doe", - annualIncome: 85000, - rhoaBankSubscription: false, - }); - expect(idTrue).not.toBe(idFalse); - }); - }); - describe("generateVerificationId", () => { - it("sanitizes timestamp: removes spaces, colons, dashes", () => { - const id = generateVerificationId("usr1", "2025-11-14 03:40:00 EST"); - expect(id).toBe("usr1_20251114_034000_EST"); - }); - }); - describe("generateUserDiscoverableToolCallId", () => { - it("matches the Python hash for sorted JSON arguments", () => { - const args = { a: 1, b: 2 }; - const id = generateUserDiscoverableToolCallId("tool", args); - expect(id).toBe("a73c858786b88826"); - }); - it("differs when arguments change", () => { - const id1 = generateUserDiscoverableToolCallId("tool", { a: 1 }); - const id2 = generateUserDiscoverableToolCallId("tool", { a: 2 }); - expect(id1).not.toBe(id2); - }); - it("same arguments produce same ID (deterministic)", () => { - const args = { b: 1, a: 2 }; - const id1 = generateUserDiscoverableToolCallId("tool", args); - const id2 = generateUserDiscoverableToolCallId("tool", { a: 2, b: 1 }); - expect(id1).toBe(id2); - }); - }); - describe("pythonSortedJsonDumps", () => { - it("sorts nested keys with Python separators", () => { - expect( - pythonSortedJsonDumps({ z: { b: 2, a: 1 }, a: [true, null] }) - ).toBe('{"a": [true, null], "z": {"a": 1, "b": 2}}'); - }); - }); - describe("generateTransactionId", () => { - it("generates transaction ID with txn_ prefix", () => { - const id = generateTransactionId({ - userId: "usr1", - creditCardType: "VISA", - merchantName: "Merchant", - amount: 100.5, - category: "food", - }); - expect(id).toMatch(/^txn_[a-f0-9]{12}$/); - }); - it("formats amount with .toFixed(2)", () => { - const id1 = generateTransactionId({ - userId: "usr1", - creditCardType: "VISA", - merchantName: "Merchant", - amount: 100, - category: "food", - }); - const id2 = generateTransactionId({ - userId: "usr1", - creditCardType: "VISA", - merchantName: "Merchant", - amount: 100, - category: "food", - }); - expect(id1).toBe(id2); - }); - it("includes date if provided", () => { - const id1 = generateTransactionId({ - userId: "usr1", - creditCardType: "VISA", - merchantName: "Merchant", - amount: 100, - category: "food", - }); - const id2 = generateTransactionId({ - userId: "usr1", - creditCardType: "VISA", - merchantName: "Merchant", - amount: 100, - category: "food", - date: "2025-11-14", - }); - expect(id1).not.toBe(id2); - }); - }); - describe("generateReferralId", () => { - it("generates 16-char hex referral ID", () => { - const id = generateReferralId("usr1", "checking"); - expect(id).toMatch(/^[a-f0-9]{16}$/); - }); - }); - describe("generateDisputeId", () => { - it("generates dsp_ prefixed dispute ID", () => { - const id = generateDisputeId("usr1", "txn_abc123"); - expect(id).toMatch(/^dsp_[a-f0-9]{12}$/); - }); - }); - describe("generateReferralLinkId", () => { - it("generates 16-char referral link ID", () => { - const id = generateReferralLinkId("usr1", "Platinum Card"); - expect(id).toMatch(/^[a-f0-9]{16}$/); - }); - }); - describe("generateCreditCardOrderId", () => { - it("generates ccord_ prefixed order ID", () => { - const id = generateCreditCardOrderId("cca_123", "usr1", "replacement"); - expect(id).toMatch(/^ccord_[a-f0-9]{12}$/); - }); - }); - describe("generateClosureReasonId", () => { - it("generates clsr_ prefixed closure reason ID", () => { - const id = generateClosureReasonId("cca_123", "usr1"); - expect(id).toMatch(/^clsr_[a-f0-9]{12}$/); - }); - }); - describe("generateAccountFlagId", () => { - it("generates ccflag_ prefixed account flag ID", () => { - const id = generateAccountFlagId("cca_123", "fraud_alert", "2025-12-31"); - expect(id).toMatch(/^ccflag_[a-f0-9]{12}$/); - }); - }); - describe("generateCreditLimitIncreaseRequestId", () => { - it("generates cli_ prefixed CLI request ID", () => { - const id = generateCreditLimitIncreaseRequestId("cca_123", "usr1", 5000); - expect(id).toMatch(/^cli_[a-f0-9]{12}$/); - }); - it("formats amount with .toFixed(2)", () => { - const id1 = generateCreditLimitIncreaseRequestId("cca_123", "usr1", 5000); - const id2 = generateCreditLimitIncreaseRequestId("cca_123", "usr1", 5000); - expect(id1).toBe(id2); - }); - }); - describe("generateBankAccountTransactionId", () => { - it("generates btxn_ prefixed bank transaction ID", () => { - const id = generateBankAccountTransactionId({ - accountId: "chk_123", - date: "11/14/2025", - description: "ATM withdrawal", - amount: -100, - transactionType: "atm_withdrawal", - }); - expect(id).toMatch(/^btxn_[a-f0-9]{12}$/); - }); - }); - describe("generateDebitCardOrderId", () => { - it("generates dcord_ prefixed debit card order ID", () => { - const id = generateDebitCardOrderId("chk_123", "usr1", "STANDARD"); - expect(id).toMatch(/^dcord_[a-f0-9]{12}$/); - }); - }); - describe("generateDebitCardId", () => { - it("generates dbc_ prefixed debit card ID", () => { - const id = generateDebitCardId("chk_123", "usr1", "2025-11-14"); - expect(id).toMatch(/^dbc_[a-f0-9]{12}$/); - }); - }); - describe("Deterministic consistency", () => { - it("same inputs produce same output across multiple calls", () => { - const id1 = generateDisputeId("usr1", "txn1"); - const id2 = generateDisputeId("usr1", "txn1"); - expect(id1).toBe(id2); - }); - it("different inputs produce different outputs", () => { - const id1 = generateDisputeId("usr1", "txn1"); - const id2 = generateDisputeId("usr2", "txn1"); - expect(id1).not.toBe(id2); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/tools/ids.ts b/src/benchmarks/tau3-bench-banking/tools/ids.ts deleted file mode 100644 index d70aa17b..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/ids.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { createHash } from "node:crypto"; - -function deterministicId(seedString: string, length = 16): string { - const hashBytes = createHash("sha256").update(seedString, "utf8").digest(); - return hashBytes.slice(0, length / 2).toString("hex"); -} - -export function pythonSortedJsonDumps(value: unknown): string { - if (value === null || value === undefined) { - return "null"; - } - if (typeof value === "boolean") { - return value ? "true" : "false"; - } - if (typeof value === "number" || typeof value === "string") { - return JSON.stringify(value); - } - if (Array.isArray(value)) { - return `[${value.map(pythonSortedJsonDumps).join(", ")}]`; - } - if (typeof value === "object") { - const record: Record = Object.fromEntries( - Object.entries(value) - ); - const entries = Object.keys(record) - .toSorted() - .map( - (key) => `${JSON.stringify(key)}: ${pythonSortedJsonDumps(record[key])}` - ); - return `{${entries.join(", ")}}`; - } - return "null"; -} - -export function generateTransactionId(opts: { - userId: string; - creditCardType: string; - merchantName: string; - amount: number; - category: string; - date?: string; -}): string { - const { userId, creditCardType, merchantName, amount, category, date } = opts; - const seedParts = [ - "transaction", - userId, - creditCardType, - merchantName, - amount.toFixed(2), - category, - ]; - if (date) { - seedParts.push(date); - } - const seed = seedParts.join(":"); - return `txn_${deterministicId(seed, 12)}`; -} - -export function generateReferralId( - referrerId: string, - referredAccountType: string, - date?: string -): string { - const seedParts = ["referral", referrerId, referredAccountType]; - if (date) { - seedParts.push(date); - } - const seed = seedParts.join(":"); - return deterministicId(seed, 16); -} - -export function generateApplicationId(opts: { - cardType: string; - customerName: string; - annualIncome: number; - rhoaBankSubscription?: boolean; -}): string { - const { - cardType, - customerName, - annualIncome, - rhoaBankSubscription = false, - } = opts; - const boolStr = rhoaBankSubscription ? "True" : "False"; - const seed = `credit_card:${cardType}:${customerName}:${String(annualIncome)}:${boolStr}`; - return deterministicId(seed, 16); -} - -export function generateVerificationId( - userId: string, - timeVerified: string -): string { - const timeSuffix = timeVerified - .replaceAll(" ", "_") - .replaceAll(":", "") - .replaceAll("-", ""); - return `${userId}_${timeSuffix}`; -} - -export function generateUserDiscoverableToolId(toolName: string): string { - const seed = `user_discoverable_tool:${toolName}`; - return deterministicId(seed, 16); -} - -export function generateUserDiscoverableToolCallId( - toolName: string, - args: Record -): string { - const argsJson = pythonSortedJsonDumps(args); - const seed = `user_discoverable_tool_call:${toolName}:${argsJson}`; - return deterministicId(seed, 16); -} - -export function generateDisputeId( - userId: string, - transactionId: string -): string { - const seed = `dispute:${userId}:${transactionId}`; - return `dsp_${deterministicId(seed, 12)}`; -} - -export function generateReferralLinkId( - userId: string, - cardName: string -): string { - const seed = `referral_link:${userId}:${cardName}`; - return deterministicId(seed, 16); -} - -export function generateAgentDiscoverableToolId(toolName: string): string { - const seed = `agent_discoverable_tool:${toolName}`; - return deterministicId(seed, 16); -} - -export function generateCreditCardOrderId( - creditCardAccountId: string, - userId: string, - reason: string -): string { - const seed = `credit_card_order:${creditCardAccountId}:${userId}:${reason}`; - return `ccord_${deterministicId(seed, 12)}`; -} - -export function generateClosureReasonId( - creditCardAccountId: string, - userId: string -): string { - const seed = `closure_reason:${creditCardAccountId}:${userId}`; - return `clsr_${deterministicId(seed, 12)}`; -} - -export function generateAccountFlagId( - creditCardAccountId: string, - flagType: string, - expirationDate: string -): string { - const seed = `account_flag:${creditCardAccountId}:${flagType}:${expirationDate}`; - return `ccflag_${deterministicId(seed, 12)}`; -} - -export function generateCreditLimitIncreaseRequestId( - creditCardAccountId: string, - userId: string, - requestedIncreaseAmount: number -): string { - const seed = `cli_request:${creditCardAccountId}:${userId}:${requestedIncreaseAmount.toFixed(2)}`; - return `cli_${deterministicId(seed, 12)}`; -} - -export function generateBankAccountTransactionId(opts: { - accountId: string; - date: string; - description: string; - amount: number; - transactionType: string; -}): string { - const { accountId, date, description, amount, transactionType } = opts; - const seed = `bank_txn:${accountId}:${date}:${description}:${amount.toFixed(2)}:${transactionType}`; - return `btxn_${deterministicId(seed, 12)}`; -} - -export function generateDebitCardOrderId( - accountId: string, - userId: string, - deliveryOption: string -): string { - const seed = `debit_card_order:${accountId}:${userId}:${deliveryOption}`; - return `dcord_${deterministicId(seed, 12)}`; -} - -export function generateDebitCardId( - accountId: string, - userId: string, - issueDate: string -): string { - const seed = `debit_card:${accountId}:${userId}:${issueDate}`; - return `dbc_${deterministicId(seed, 12)}`; -} diff --git a/src/benchmarks/tau3-bench-banking/tools/registry.test.ts b/src/benchmarks/tau3-bench-banking/tools/registry.test.ts deleted file mode 100644 index 39c78d49..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/registry.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { describe, it, expect } from "bun:test"; - -import { makeEmptyBankingData } from "../environment"; -import type { BankingData } from "../types"; -import type { DiscoverableTool } from "./registry"; -import { - makeBankingEnvState, - DISCOVERABLE_AGENT_TOOLS, - DISCOVERABLE_USER_TOOLS, - registerDiscoverableAgentTool, - registerDiscoverableUserTool, - formatDiscoverableToolForAgent, -} from "./registry"; - -function createTestDb(): BankingData { - return makeEmptyBankingData(); -} -describe("registry", () => { - describe("makeBankingEnvState", () => { - it("creates new environment state with empty registries", () => { - const db = createTestDb(); - const state = makeBankingEnvState(db); - expect(state.db).toBe(db); - expect(state.unlockedAgentTools).toBeInstanceOf(Set); - expect(state.givenUserTools).toBeInstanceOf(Map); - expect(state.unlockedAgentTools.size).toBe(0); - expect(state.givenUserTools.size).toBe(0); - }); - }); - describe("registerDiscoverableAgentTool", () => { - it("adds tool to DISCOVERABLE_AGENT_TOOLS registry", () => { - const toolName = `test_agent_tool_${Date.now()}`; - const tool: DiscoverableTool = { - name: toolName, - description: "Test tool", - params: [], - handler: () => "test result", - }; - registerDiscoverableAgentTool(tool); - expect(DISCOVERABLE_AGENT_TOOLS.has(toolName)).toBe(true); - }); - it("stored tool can be retrieved", () => { - const toolName = `test_agent_tool_${Date.now()}`; - const tool: DiscoverableTool = { - name: toolName, - description: "Test tool", - params: [ - { - name: "param1", - type: "string", - optional: false, - description: "First param", - }, - ], - handler: () => "result", - }; - registerDiscoverableAgentTool(tool); - const retrieved = DISCOVERABLE_AGENT_TOOLS.get(toolName); - expect(retrieved).toBe(tool); - }); - }); - describe("registerDiscoverableUserTool", () => { - it("adds tool to DISCOVERABLE_USER_TOOLS registry", () => { - const toolName = `test_user_tool_${Date.now()}`; - const tool: DiscoverableTool = { - name: toolName, - description: "User tool", - params: [], - handler: () => "user result", - }; - registerDiscoverableUserTool(tool); - expect(DISCOVERABLE_USER_TOOLS.has(toolName)).toBe(true); - }); - }); - describe("formatDiscoverableToolForAgent", () => { - it("formats tool with no parameters", () => { - const tool: DiscoverableTool = { - name: "test_tool", - description: "A test tool", - params: [], - handler: () => "", - }; - const formatted = formatDiscoverableToolForAgent(tool); - expect(formatted).toContain("Tool: test_tool"); - expect(formatted).toContain("Description: A test tool"); - expect(formatted).toContain("Parameters:"); - expect(formatted).toContain("(no parameters)"); - }); - it("formats tool with required parameters", () => { - const tool: DiscoverableTool = { - name: "test_tool", - description: "A test tool", - params: [ - { - name: "user_id", - type: "string", - optional: false, - description: "The user ID", - }, - { - name: "amount", - type: "number", - optional: false, - description: "Transfer amount", - }, - ], - handler: () => "", - }; - const formatted = formatDiscoverableToolForAgent(tool); - expect(formatted).toContain("user_id: string (required) - The user ID"); - expect(formatted).toContain( - "amount: number (required) - Transfer amount" - ); - }); - it("formats tool with optional parameters", () => { - const tool: DiscoverableTool = { - name: "test_tool", - description: "A test tool", - params: [ - { - name: "page", - type: "integer", - optional: true, - description: "Page number", - }, - ], - handler: () => "", - }; - const formatted = formatDiscoverableToolForAgent(tool); - expect(formatted).toContain("page: integer (optional) - Page number"); - }); - it("includes tool name and description", () => { - const tool: DiscoverableTool = { - name: "transfer_funds", - description: "Transfer money between accounts", - params: [], - handler: () => "", - }; - const formatted = formatDiscoverableToolForAgent(tool); - expect(formatted).toContain("Tool: transfer_funds"); - expect(formatted).toContain( - "Description: Transfer money between accounts" - ); - }); - }); - describe("State isolation", () => { - it("multiple env states have independent tool registries", () => { - const db1 = createTestDb(); - const db2 = createTestDb(); - const state1 = makeBankingEnvState(db1); - const state2 = makeBankingEnvState(db2); - state1.unlockedAgentTools.add("tool_a"); - expect(state2.unlockedAgentTools.has("tool_a")).toBe(false); - }); - it("multiple env states have independent given user tools", () => { - const db1 = createTestDb(); - const db2 = createTestDb(); - const state1 = makeBankingEnvState(db1); - const state2 = makeBankingEnvState(db2); - state1.givenUserTools.set("tool_x", { arg: "value" }); - expect(state2.givenUserTools.has("tool_x")).toBe(false); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/tools/registry.ts b/src/benchmarks/tau3-bench-banking/tools/registry.ts deleted file mode 100644 index e8a6481b..00000000 --- a/src/benchmarks/tau3-bench-banking/tools/registry.ts +++ /dev/null @@ -1,65 +0,0 @@ -import type { BankingData } from "../types"; - -type BankingDb = BankingData; - -export interface ToolParameter { - readonly name: string; - readonly type: string; - readonly optional: boolean; - readonly description: string; -} - -export interface DiscoverableTool { - readonly name: string; - readonly description: string; - readonly params: readonly ToolParameter[]; - readonly mutatesState?: boolean; - readonly handler: ( - state: BankingEnvState, - kwargs: Record - ) => string; -} - -export interface BankingEnvState { - db: BankingDb; - unlockedAgentTools: Set; - givenUserTools: Map>; - readLogAllowlist: ReadonlySet; -} - -export function makeBankingEnvState( - db: BankingDb, - readLogAllowlist: Iterable = [] -): BankingEnvState { - return { - db, - unlockedAgentTools: new Set(), - givenUserTools: new Map(), - readLogAllowlist: new Set(readLogAllowlist), - }; -} - -export const DISCOVERABLE_AGENT_TOOLS = new Map(); - -export const DISCOVERABLE_USER_TOOLS = new Map(); - -export function registerDiscoverableAgentTool(tool: DiscoverableTool): void { - DISCOVERABLE_AGENT_TOOLS.set(tool.name, tool); -} - -export function registerDiscoverableUserTool(tool: DiscoverableTool): void { - DISCOVERABLE_USER_TOOLS.set(tool.name, tool); -} - -export function formatDiscoverableToolForAgent(tool: DiscoverableTool): string { - const paramStrs: string[] = []; - for (const param of tool.params) { - const req = param.optional ? " (optional)" : " (required)"; - paramStrs.push( - ` - ${param.name}: ${param.type}${req} - ${param.description}` - ); - } - const paramsSection = - paramStrs.length > 0 ? paramStrs.join("\n") : " (no parameters)"; - return `Tool: ${tool.name}\nDescription: ${tool.description}\nParameters:\n${paramsSection}`; -} diff --git a/src/benchmarks/tau3-bench-banking/types.test.ts b/src/benchmarks/tau3-bench-banking/types.test.ts deleted file mode 100644 index e2e055ee..00000000 --- a/src/benchmarks/tau3-bench-banking/types.test.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { describe, expect, it } from "bun:test"; - -import { assertRight, assertLeft } from "../../internal/testing"; -import { parseSchema } from "../../internal/zod"; -import { BANKING_TABLES, Tau3TaskSchema, RewardType } from "./types"; -describe("tau3-bench-banking types", () => { - describe("Tau3TaskSchema", () => { - it("parses task_001 (basic task without initial_state)", () => { - const task = { - id: "task_001", - description: { - purpose: "Task: task_001", - relevant_policies: null, - notes: null, - }, - user_scenario: { - persona: null, - instructions: "You are playing the role of a customer...", - }, - initial_state: null, - evaluation_criteria: { - actions: [ - { - name: "apply_for_credit_card", - arguments: { - card_type: "Gold Rewards Card", - customer_name: "Sarah Bosch", - }, - requestor: "user", - action_id: "001_0", - }, - ], - communicate_info: [], - reward_basis: ["DB"], - }, - annotations: null, - user_tools: ["apply_for_credit_card"], - required_documents: ["doc_credit_cards_gold_rewards_card_001"], - }; - const result = parseSchema(Tau3TaskSchema, task); - assertRight(result); - expect(result.right.id).toBe("task_001"); - expect(result.right.user_scenario.instructions).toBe( - "You are playing the role of a customer..." - ); - expect(result.right.evaluation_criteria?.reward_basis).toEqual(["DB"]); - }); - it("parses task with non-null initial_state", () => { - const task = { - id: "task_026", - description: { - purpose: "Task: task_026", - relevant_policies: null, - notes: null, - }, - user_scenario: { - persona: null, - instructions: - "You have noticed that your recent cash back rewards dont add up correctly...", - }, - initial_state: { - initialization_data: { - agent_data: { - task_config: { - data: { dispute_settings: { auto_resolve_disputes: true } }, - }, - }, - }, - initialization_actions: null, - message_history: null, - }, - evaluation_criteria: { - actions: [ - { - name: "log_verification", - arguments: {}, - requestor: "user", - action_id: "026_0", - }, - ], - communicate_info: [], - reward_basis: ["DB"], - }, - annotations: null, - user_tools: [], - required_documents: [], - }; - const result = parseSchema(Tau3TaskSchema, task); - assertRight(result); - expect(result.right.id).toBe("task_026"); - expect( - result.right.initial_state?.initialization_data?.agent_data - ).toEqual({ - task_config: { - data: { dispute_settings: { auto_resolve_disputes: true } }, - }, - }); - }); - it("tolerates unknown keys in user_scenario (persoca typo key)", () => { - const task = { - id: "task_091", - description: { - purpose: "Task: task_091", - relevant_policies: null, - notes: null, - }, - user_scenario: { - persoca: null, - persona: null, - instructions: - "You are Tyler Washington, a 24-year-old software engineer...", - }, - initial_state: null, - evaluation_criteria: { - actions: [ - { - name: "test_action", - arguments: {}, - requestor: "user", - action_id: "091_0", - }, - ], - communicate_info: [], - reward_basis: ["DB"], - }, - annotations: null, - user_tools: [], - required_documents: [], - }; - const result = parseSchema(Tau3TaskSchema, task); - assertRight(result); - expect(result.right.user_scenario.instructions).toBe( - "You are Tyler Washington, a 24-year-old software engineer..." - ); - expect(result.right.user_scenario.persoca).toBeNull(); - }); - it("validates action structure within evaluation_criteria", () => { - const task = { - id: "task_001", - description: null, - user_scenario: { - instructions: "Test instruction", - }, - initial_state: null, - evaluation_criteria: { - actions: [ - { - action_id: "001_0", - requestor: "user", - name: "apply_for_credit_card", - arguments: { card_type: "Gold", customer_name: "John" }, - }, - ], - communicate_info: [], - reward_basis: ["DB", "ACTION", "COMMUNICATE"], - }, - annotations: null, - }; - const result = parseSchema(Tau3TaskSchema, task); - assertRight(result); - const action = result.right.evaluation_criteria?.actions?.[0]; - expect(action?.name).toBe("apply_for_credit_card"); - expect(action?.arguments).toEqual({ - card_type: "Gold", - customer_name: "John", - }); - }); - it("rejects invalid reward_basis enum value", () => { - const task = { - id: "task_001", - description: null, - user_scenario: { - instructions: "Test", - }, - initial_state: null, - evaluation_criteria: { - actions: [], - communicate_info: [], - reward_basis: ["INVALID_REWARD_TYPE"], - }, - annotations: null, - }; - const result = parseSchema(Tau3TaskSchema, task); - assertLeft(result); - }); - }); - describe("RewardType enum", () => { - it("contains all expected reward basis types", () => { - expect(RewardType.Db).toBe("DB"); - expect(RewardType.EnvAssertion).toBe("ENV_ASSERTION"); - expect(RewardType.NlAssertion).toBe("NL_ASSERTION"); - expect(RewardType.Action).toBe("ACTION"); - expect(RewardType.Communicate).toBe("COMMUNICATE"); - }); - }); - describe("BANKING_TABLES constant", () => { - it("contains all 23 expected banking table names", () => { - expect(BANKING_TABLES.length).toBe(23); - expect(BANKING_TABLES).toContain("users"); - expect(BANKING_TABLES).toContain("accounts"); - expect(BANKING_TABLES).toContain("debit_cards"); - expect(BANKING_TABLES).toContain("credit_card_accounts"); - expect(BANKING_TABLES).toContain("task_config"); - expect(BANKING_TABLES).toContain("agent_discoverable_tools"); - expect(BANKING_TABLES).toContain("debit_card_disputes"); - }); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/types.ts b/src/benchmarks/tau3-bench-banking/types.ts deleted file mode 100644 index c62c12d5..00000000 --- a/src/benchmarks/tau3-bench-banking/types.ts +++ /dev/null @@ -1,173 +0,0 @@ -import type { ReasoningEffort } from "../../harness/constants"; -import type { ValueOf } from "../../internal/guards"; -import { z } from "../../internal/zod"; -import type { InferenceOverride } from "../benchmark-config"; -import type { BankingRetrievalConfig } from "./retrieval-config"; - -export const RewardType = { - Db: "DB", - EnvAssertion: "ENV_ASSERTION", - NlAssertion: "NL_ASSERTION", - Action: "ACTION", - Communicate: "COMMUNICATE", -} as const; - -export type RewardType = ValueOf; - -export const DEFAULT_REWARD_BASIS: readonly RewardType[] = [ - RewardType.Db, - RewardType.Communicate, -]; - -export const ToolRequestor = { - Assistant: "assistant", - User: "user", -} as const; - -export type ToolRequestor = ValueOf; - -export const Tau3ActionSchema = z.object({ - action_id: z.string(), - requestor: z - .enum([ToolRequestor.Assistant, ToolRequestor.User]) - .default(ToolRequestor.Assistant), - name: z.string(), - arguments: z.record(z.string(), z.unknown()), - compare_args: z.array(z.string()).nullish(), - info: z.string().nullish(), -}); - -export type Tau3Action = z.infer; - -export const Tau3EvaluationCriteriaSchema = z.object({ - actions: z.array(Tau3ActionSchema).nullish(), - communicate_info: z.array(z.string()).nullish(), - reward_basis: z - .array( - z.enum([ - RewardType.Db, - RewardType.EnvAssertion, - RewardType.NlAssertion, - RewardType.Action, - RewardType.Communicate, - ]) - ) - .nullish(), -}); - -export type Tau3EvaluationCriteria = z.infer< - typeof Tau3EvaluationCriteriaSchema ->; - -export const Tau3InitialStateSchema = z.object({ - initialization_data: z - .object({ - agent_data: z.record(z.string(), z.unknown()).nullish(), - }) - .nullable(), - initialization_actions: z.unknown().nullish(), - message_history: z.unknown().nullish(), -}); - -export type Tau3InitialState = z.infer; - -export const Tau3TaskSchema = z - .object({ - id: z.string(), - description: z - .object({ - purpose: z.string().nullish(), - relevant_policies: z.string().nullish(), - notes: z.string().nullish(), - }) - .nullish(), - user_scenario: z.object({ instructions: z.string() }).passthrough(), - initial_state: Tau3InitialStateSchema.nullish(), - evaluation_criteria: Tau3EvaluationCriteriaSchema.nullish(), - annotations: z.unknown().nullish(), - user_tools: z.array(z.string()).nullish(), - required_documents: z.array(z.string()).nullish(), - }) - .passthrough(); - -export type Tau3Task = z.infer; - -export function deriveReadLogAllowlist(task: Tau3Task): ReadonlySet { - const allowlist = new Set(); - for (const action of task.evaluation_criteria?.actions ?? []) { - if (action.name !== "call_discoverable_agent_tool") { - continue; - } - const toolName = action.arguments?.agent_tool_name; - if (typeof toolName === "string" && toolName.length > 0) { - allowlist.add(toolName); - } - } - return allowlist; -} - -export const BANKING_TABLES = [ - "users", - "accounts", - "debit_cards", - "referrals", - "credit_card_applications", - "user_discoverable_tools", - "user_discoverable_tool_calls", - "verification_history", - "credit_card_transaction_history", - "cash_back_disputes", - "bank_account_transaction_history", - "credit_card_accounts", - "agent_discoverable_tools", - "task_config", - "human_transfer_requests", - "transaction_disputes", - "credit_card_orders", - "debit_card_orders", - "credit_card_closure_reasons", - "credit_card_account_flags", - "credit_limit_increase_requests", - "payment_history", - "debit_card_disputes", -] as const; - -export type BankingTableName = ValueOf; - -export interface BankingTable { - data: Record>; - notes?: unknown; -} - -export type BankingData = Record; - -const BANKING_TABLE_NAME_SET: ReadonlySet = new Set(BANKING_TABLES); - -export function isBankingTableName(name: string): name is BankingTableName { - return BANKING_TABLE_NAME_SET.has(name); -} - -export function getBankingTable( - db: BankingData, - tableName: string -): BankingTable | undefined { - if (!isBankingTableName(tableName)) { - return undefined; - } - return db[tableName]; -} - -export interface UserModelConfig { - readonly apiKey: string; - readonly model: string; - readonly baseUrl?: string; - readonly sessionId?: string; - readonly userReasoningEffort: ReasoningEffort; -} - -export interface SolverOpts { - readonly endpointId?: string; - readonly userModelConfig?: UserModelConfig; - readonly retrievalConfig?: BankingRetrievalConfig; - readonly inference: InferenceOverride; -} diff --git a/src/benchmarks/tau3-bench-banking/user-sim-guidelines.ts b/src/benchmarks/tau3-bench-banking/user-sim-guidelines.ts deleted file mode 100644 index fcd2d4f6..00000000 --- a/src/benchmarks/tau3-bench-banking/user-sim-guidelines.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const USER_SIM_GUIDELINES = - "# User Simulation Guidelines\nYou are playing the role of a banking customer contacting a customer service representative.\nYour goal is to simulate realistic customer interactions while following specific scenario instructions.\n\n## Core Principles\n- Generate one message at a time, maintaining natural conversation flow.\n- Strictly follow the scenario instructions you have received.\n- Never make up or hallucinate information not provided in the scenario instructions. Information that is not provided in the scenario instructions should be considered unknown or unavailable.\n- Avoid repeating the exact instructions verbatim. Use paraphrasing and natural language to convey the same information\n- Disclose information progressively. Wait for the agent to ask for specific information before providing it.\n\n## Task Completion\n- The goal is to continue the conversation until the task is complete.\n- If the instruction goal is satisified, generate the '###STOP###' token to end the conversation.\n- If you are transferred to another agent, generate the '###TRANSFER###' token to indicate the transfer.\n- If you find yourself in a situation in which the scenario does not provide enough information for you to continue the conversation, generate the '###OUT-OF-SCOPE###' token to end the conversation.\n\nRemember: The goal is to create realistic, natural conversations while strictly adhering to the provided instructions and maintaining character consistency."; - -export const USER_SIM_GUIDELINES_TOOLS = - "# User Simulation Guidelines\n\nYou are playing the role of a customer contacting a customer service representative agent. \nYour goal is to simulate realistic customer interactions while following specific scenario instructions.\nYou have some tools to perform the actions on your end that might be requested by the agent to diagnose and resolve your issue.\n\n## Core Principles\n- Generate one message at a time, maintaining natural conversation flow.\n- At each turn you can either:\n - Send a message to the agent.\n - Make a tool call to perform an action requested by the agent.\n - You cannot do both at the same time.\n- Strictly follow the scenario instructions you have received.\n- Never make up or hallucinate information not provided in the scenario instructions. Information that is not provided in the scenario instructions should be considered unknown or unavailable.\n- Never make up the results of tool calls that the agent has requested, you must ground your responses based on the results of tool calls if the agent has requested.\n- If you made an error in a tool call and get an error message, fix the error and try again.\n- All the information you provide to the agent must be grounded in the information provided in the scenario instructions or the results of tool calls.\n- Avoid repeating the exact instructions verbatim. Use paraphrasing and natural language to convey the same information\n- Disclose information progressively. Wait for the agent to ask for specific information before providing it.\n- Only call a tool if the agent has requested it or if it is necessary to answer a question the agent has asked. Ask clarifying questions if you do not know what action to take.\n- If the agent asks multiple actions to perform, state that you cannot perform multiple actions at once, and ask the agent to instruct you one action at a time.\n- Your messages when performing tool calls will not be displayed to the agent, only the messages without tool calls will be displayed to the agent.\n\n## Task Completion\n- The goal is to continue the conversation until the task is complete.\n- If the instruction goal is satisified, generate the '###STOP###' token to end the conversation.\n- If you have been transferred to another agent, generate the '###TRANSFER###' token to indicate the transfer. Only do this after the agent has clearly indicated that you are being transferred.\n- If you find yourself in a situation in which the scenario does not provide enough information for you to continue the conversation, generate the '###OUT-OF-SCOPE###' token to end the conversation.\nRemember: The goal is to create realistic, natural conversations while strictly adhering to the provided instructions and maintaining character consistency.\n"; diff --git a/src/benchmarks/tau3-bench-banking/user-simulator.test.ts b/src/benchmarks/tau3-bench-banking/user-simulator.test.ts deleted file mode 100644 index 3f27ad95..00000000 --- a/src/benchmarks/tau3-bench-banking/user-simulator.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { describe, expect, it } from "bun:test"; - -import { runPromise, succeed } from "effect/Effect"; - -import type { - ResponsesInputItem, - ResponsesModelService, - ResponsesTurn, -} from "../../providers/responses-model"; -import { UserSimulator } from "./user-simulator"; - -const config = { - apiKey: "sk-test", - model: "openai/gpt-5", - sessionId: "session-1", - userReasoningEffort: "low", -} as const; - -describe("tau3 banking user simulator", () => { - it("uses Responses function calls and preserves call_id in tool results", async () => { - const inputs: ResponsesInputItem[][] = []; - const turns: ResponsesTurn[] = [ - { - outputItems: [ - { - type: "reasoning", - encrypted_content: "opaque", - }, - { - type: "function_call", - call_id: "call-1", - name: "get_balance", - arguments: '{"account_id":"a1"}', - }, - ], - functionCalls: [ - { - callId: "call-1", - name: "get_balance", - arguments: '{"account_id":"a1"}', - }, - ], - text: "", - generationTimeMs: 1, - }, - { - outputItems: [ - { type: "message", content: [{ type: "output_text", text: "Done" }] }, - ], - functionCalls: [], - text: "Done", - generationTimeMs: 1, - }, - ]; - let index = 0; - const model: ResponsesModelService = { - generate: (input, generateConfig) => { - inputs.push([...input]); - expect(generateConfig.reasoningEffort).toBe("low"); - return succeed(turns[index++]!); - }, - }; - const simulator = new UserSimulator(model, config); - simulator.setAvailableTools([ - { - type: "function", - function: { - name: "get_balance", - description: undefined, - parameters: { type: "object" }, - }, - }, - ]); - simulator.reset("scenario", "Hi"); - const toolTurn = await runPromise(simulator.generateInitial()); - expect(toolTurn).toEqual({ - kind: "toolCalls", - calls: [ - { - id: "call-1", - name: "get_balance", - arguments: '{"account_id":"a1"}', - }, - ], - }); - simulator.addToolResult("call-1", "Balance: $10"); - expect(await runPromise(simulator.continueAfterTools())).toEqual({ - kind: "text", - content: "Done", - }); - expect(inputs[1]).toEqual([ - { type: "message", role: "system", content: expect.any(String) }, - { type: "message", role: "user", content: "Hi" }, - ...turns[0]!.outputItems, - { - type: "function_call_output", - call_id: "call-1", - output: "Balance: $10", - }, - ]); - }); -}); diff --git a/src/benchmarks/tau3-bench-banking/user-simulator.ts b/src/benchmarks/tau3-bench-banking/user-simulator.ts deleted file mode 100644 index 5003a741..00000000 --- a/src/benchmarks/tau3-bench-banking/user-simulator.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { TaggedError } from "effect/Data"; -import type { Effect } from "effect/Effect"; -import { map, mapError } from "effect/Effect"; - -import type { ModelMessage, ToolDefinition } from "../../harness/core"; -import { MessageRole } from "../../harness/core"; -import { definedValues } from "../../internal/guards"; -import { - messagesToResponses, - toolDefinitionToResponses, -} from "../../providers/messages-to-responses"; -import type { ResponsesModelService } from "../../providers/responses-model"; -import type { UserModelConfig } from "./types"; -import { - USER_SIM_GUIDELINES, - USER_SIM_GUIDELINES_TOOLS, -} from "./user-sim-guidelines"; - -class UserSimError extends TaggedError("UserSimError")<{ - readonly message: string; -}> {} - -type SimError = UserSimError; - -export type SimulatorTurn = TextTurn | ToolCallsTurn; - -interface TextTurn { - readonly kind: "text"; - readonly content: string; -} - -interface ToolCallsTurn { - readonly kind: "toolCalls"; - readonly calls: readonly { - readonly id: string; - readonly name: string; - readonly arguments: string; - }[]; -} - -function buildUserSystemPrompt( - scenarioInstructions: string, - useTools: boolean -): string { - const guidelines = useTools ? USER_SIM_GUIDELINES_TOOLS : USER_SIM_GUIDELINES; - return `${guidelines}\n\n\n${scenarioInstructions}\n`; -} - -export class UserSimulator { - private readonly messages: ModelMessage[] = []; - private readonly config: UserModelConfig; - private readonly model: ResponsesModelService; - private availableTools: readonly ToolDefinition[] = []; - - constructor(model: ResponsesModelService, config: UserModelConfig) { - this.model = model; - this.config = config; - } - - reset(scenarioInstructions: string, firstAgentMessage: string): void { - const useTools = this.availableTools.length > 0; - this.messages.length = 0; - this.messages.push( - { - role: MessageRole.System, - content: buildUserSystemPrompt(scenarioInstructions, useTools), - }, - { role: MessageRole.User, content: firstAgentMessage } - ); - } - - generateInitial(): Effect { - return this.callModel(this.config.model); - } - - step(agentMessage: string): Effect { - this.messages.push({ role: MessageRole.User, content: agentMessage }); - return this.callModel(this.config.model); - } - - continueAfterTools(): Effect { - return this.callModel(this.config.model); - } - - setAvailableTools(toolDefs: readonly ToolDefinition[]): void { - this.availableTools = toolDefs; - } - - addToolResult(toolCallId: string, content: string): void { - this.messages.push({ - role: MessageRole.Tool, - content, - toolCallId, - }); - } - - private callModel(model: string): Effect { - return this.callModelOnce(model).pipe( - map((turn) => { - const assistantMessage: ModelMessage = definedValues({ - role: MessageRole.Assistant, - content: turn.text, - responseItems: - turn.outputItems.length > 0 ? turn.outputItems : undefined, - }); - this.messages.push(assistantMessage); - if (turn.functionCalls.length > 0) { - return { - kind: "toolCalls", - calls: turn.functionCalls.map((call) => ({ - id: call.callId, - name: call.name, - arguments: call.arguments, - })), - }; - } - return { kind: "text", content: turn.text }; - }) - ); - } - - private callModelOnce(model: string) { - return this.model - .generate(messagesToResponses(this.messages), { - model, - temperature: 0, - reasoningEffort: this.config.userReasoningEffort, - ...definedValues({ - tools: - this.availableTools.length > 0 - ? this.availableTools.map(toolDefinitionToResponses) - : undefined, - }), - }) - .pipe(mapError((error) => new UserSimError({ message: error.message }))); - } -} diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 30a25f17..d36f07ee 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -166,47 +166,6 @@ describe("bench-harness CLI", () => { }) ).toThrow("Unknown gpqa_diamond solver-config option(s): bogus"); }); - - it("passes tau3 retrieval config through the generic solver config", () => { - const args = parseArgs([ - "--benchmark", - "tau3_bench_banking", - "--model", - "openai/gpt-4o-mini", - "--solver-config", - '{"retrievalConfig":"bm25_grep"}', - ]); - const panelConfig: unknown = JSON.parse(args.solverConfig ?? ""); - const config = buildBenchmarkConfig({ - benchmarkId: args.benchmark, - model: args.model, - panelConfig, - artifactDir: undefined, - endpointId: undefined, - imageDetail: undefined, - reasoningEffort: args.reasoningEffort, - }); - expect(config).toMatchObject({ - benchmarkId: "tau3_bench_banking", - retrievalConfig: "bm25_grep", - }); - }); - - it("materializes the bm25_grep default for tau3", () => { - const config = buildBenchmarkConfig({ - benchmarkId: "tau3_bench_banking", - model: "openai/gpt-4o-mini", - panelConfig: undefined, - artifactDir: undefined, - endpointId: undefined, - imageDetail: undefined, - reasoningEffort: "high", - }); - expect(config).toMatchObject({ - benchmarkId: "tau3_bench_banking", - retrievalConfig: "bm25_grep", - }); - }); }); describe("resolveSessionId", () => { diff --git a/src/cli/index.ts b/src/cli/index.ts index 02ce5512..3af6d9f2 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -454,16 +454,6 @@ export function buildBenchmarkConfig(opts: { reasoningEffort, }); } - case "tau3_bench_banking": { - return buildSchemaValidatedConfig({ - benchmarkId: "tau3_bench_banking", - model: requireModel("tau3_bench_banking", model), - endpointId, - panelConfig, - costTier, - reasoningEffort, - }); - } case "draco": { const panel = parseSchema(DracoPanelConfigSchema, panelConfig); if (Either.isLeft(panel)) {