From a3d681491fadb27f7a838b647ee537b59a79df5b Mon Sep 17 00:00:00 2001 From: JerryChaox Date: Sun, 15 Nov 2026 01:35:49 +0800 Subject: [PATCH] fix(bfcl): keep the bfcl extra importable by capping pyarrow below 26 --- docs/en/benchmarks/bfcl_v3.md | 2 +- docs/en/benchmarks/bfcl_v4.md | 2 +- docs/en/third_party/bfcl_v3.md | 3 +-- docs/en/third_party/bfcl_v4.md | 3 +-- docs/zh/benchmarks/bfcl_v3.md | 2 +- docs/zh/benchmarks/bfcl_v4.md | 2 +- docs/zh/third_party/bfcl_v3.md | 3 +-- docs/zh/third_party/bfcl_v4.md | 3 +-- evalscope/benchmarks/_meta/bfcl_v3.json | 12 ++++++------ evalscope/benchmarks/_meta/bfcl_v4.json | 12 ++++++------ evalscope/benchmarks/bfcl/requirements.txt | 2 ++ evalscope/benchmarks/bfcl/v3/bfcl_v3_adapter.py | 2 +- evalscope/benchmarks/bfcl/v4/bfcl_v4_adapter.py | 2 +- 13 files changed, 24 insertions(+), 26 deletions(-) diff --git a/docs/en/benchmarks/bfcl_v3.md b/docs/en/benchmarks/bfcl_v3.md index 6ad183279..5f60bbce1 100644 --- a/docs/en/benchmarks/bfcl_v3.md +++ b/docs/en/benchmarks/bfcl_v3.md @@ -23,7 +23,7 @@ BFCL (Berkeley Function Calling Leaderboard) v3 is the first comprehensive and e ## Evaluation Notes -- Requires `pip install bfcl-eval==2025.10.27.1` before evaluation +- Requires `pip install 'evalscope[bfcl]'` before evaluation - Set `is_fc_model=True` for models with native function calling support - `underscore_to_dot` parameter controls function name formatting - See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v3.html) for detailed setup diff --git a/docs/en/benchmarks/bfcl_v4.md b/docs/en/benchmarks/bfcl_v4.md index 780fe417f..65970106b 100644 --- a/docs/en/benchmarks/bfcl_v4.md +++ b/docs/en/benchmarks/bfcl_v4.md @@ -22,7 +22,7 @@ BFCL-v4 (Berkeley Function-Calling Leaderboard V4) is a comprehensive benchmark ## Evaluation Notes -- **Installation Required**: `pip install bfcl-eval==2025.10.27.1` +- **Installation Required**: `pip install 'evalscope[bfcl]'` - Primary metric: **Accuracy** per category - Configure `is_fc_model` based on model capabilities - Optional: Set `SERPAPI_API_KEY` for web search tasks diff --git a/docs/en/third_party/bfcl_v3.md b/docs/en/third_party/bfcl_v3.md index 527717b5f..a3886c561 100644 --- a/docs/en/third_party/bfcl_v3.md +++ b/docs/en/third_party/bfcl_v3.md @@ -39,8 +39,7 @@ This benchmark consists of 17 tasks, covering a wide range of function-calling s Before running the evaluation, you need to install the following dependencies: ```bash -pip install evalscope # Install evalscope -pip install bfcl-eval==2025.10.27.1 # Install bfcl-eval +pip install 'evalscope[bfcl]' # Install evalscope with bfcl-eval and its compatible dependencies ``` ## Usage diff --git a/docs/en/third_party/bfcl_v4.md b/docs/en/third_party/bfcl_v4.md index 5768554c1..61aae8793 100644 --- a/docs/en/third_party/bfcl_v4.md +++ b/docs/en/third_party/bfcl_v4.md @@ -62,8 +62,7 @@ Before running the evaluation, you need to install the following dependencies: ```bash -pip install evalscope -pip install bfcl-eval==2025.10.27.1 +pip install 'evalscope[bfcl]' ``` ## Usage diff --git a/docs/zh/benchmarks/bfcl_v3.md b/docs/zh/benchmarks/bfcl_v3.md index 437292c91..5c38c57f7 100644 --- a/docs/zh/benchmarks/bfcl_v3.md +++ b/docs/zh/benchmarks/bfcl_v3.md @@ -23,7 +23,7 @@ BFCL(Berkeley Function Calling Leaderboard)v3 是首个全面且可执行的 ## 评估说明 -- 评估前需安装 `pip install bfcl-eval==2025.10.27.1` +- 评估前需安装 `pip install 'evalscope[bfcl]'` - 对于原生支持函数调用的模型,请设置 `is_fc_model=True` - `underscore_to_dot` 参数控制函数名称的格式化方式 - 详细设置请参阅 [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html) diff --git a/docs/zh/benchmarks/bfcl_v4.md b/docs/zh/benchmarks/bfcl_v4.md index 13f115290..3fc2aebe2 100644 --- a/docs/zh/benchmarks/bfcl_v4.md +++ b/docs/zh/benchmarks/bfcl_v4.md @@ -22,7 +22,7 @@ BFCL-v4(Berkeley Function-Calling Leaderboard V4)是一个全面的基准测 ## 评估说明 -- **安装要求**:`pip install bfcl-eval==2025.10.27.1` +- **安装要求**:`pip install 'evalscope[bfcl]'` - 主要指标:各类别下的 **准确率(Accuracy)** - 根据模型能力设置 `is_fc_model` 参数 - 可选:配置 `SERPAPI_API_KEY` 以启用网络搜索任务 diff --git a/docs/zh/third_party/bfcl_v3.md b/docs/zh/third_party/bfcl_v3.md index b19eded16..62aeeb674 100644 --- a/docs/zh/third_party/bfcl_v3.md +++ b/docs/zh/third_party/bfcl_v3.md @@ -39,8 +39,7 @@ Berkeley Function Calling Leaderboard (BFCL) 专门用于评估大型语言模 在运行评测之前需要安装以下依赖: ```bash -pip install evalscope # 安装 evalscope -pip install bfcl-eval==2025.10.27.1 # 安装 bfcl-eval +pip install 'evalscope[bfcl]' # 安装 evalscope 及 bfcl-eval 和与之兼容的依赖 ``` ## 使用方法 diff --git a/docs/zh/third_party/bfcl_v4.md b/docs/zh/third_party/bfcl_v4.md index be7464349..81e829ae8 100644 --- a/docs/zh/third_party/bfcl_v4.md +++ b/docs/zh/third_party/bfcl_v4.md @@ -61,8 +61,7 @@ SUBSET_LIST = [ 在运行评测之前需要安装以下依赖: ```bash -pip install evalscope -pip install bfcl-eval==2025.10.27.1 +pip install 'evalscope[bfcl]' ``` ## 使用方法 diff --git a/evalscope/benchmarks/_meta/bfcl_v3.json b/evalscope/benchmarks/_meta/bfcl_v3.json index 527733539..2ddca8187 100644 --- a/evalscope/benchmarks/_meta/bfcl_v3.json +++ b/evalscope/benchmarks/_meta/bfcl_v3.json @@ -32,7 +32,7 @@ "multi_turn_miss_param", "multi_turn_long_context" ], - "description": "\n## Overview\n\nBFCL (Berkeley Function Calling Leaderboard) v3 is the first comprehensive and executable function call evaluation benchmark for assessing LLMs' ability to invoke functions. It evaluates various forms of function calls, diverse scenarios, and executability.\n\n## Task Description\n\n- **Task Type**: Function Calling / Tool Use Evaluation\n- **Input**: User query with available function definitions\n- **Output**: Correct function call with parameters\n- **Categories**: AST_NON_LIVE, AST_LIVE, RELEVANCE, MULTI_TURN\n\n## Key Features\n\n- Comprehensive function call evaluation\n- Tests simple, multiple, and parallel function calls\n- Multi-language support (Python, Java, JavaScript)\n- Relevance detection (irrelevance handling)\n- Multi-turn conversation function calling\n- Live API endpoints for execution testing\n\n## Evaluation Notes\n\n- Requires `pip install bfcl-eval==2025.10.27.1` before evaluation\n- Set `is_fc_model=True` for models with native function calling support\n- `underscore_to_dot` parameter controls function name formatting\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v3.html) for detailed setup\n- Results broken down by category (AST, Relevance, Multi-turn)\n", + "description": "\n## Overview\n\nBFCL (Berkeley Function Calling Leaderboard) v3 is the first comprehensive and executable function call evaluation benchmark for assessing LLMs' ability to invoke functions. It evaluates various forms of function calls, diverse scenarios, and executability.\n\n## Task Description\n\n- **Task Type**: Function Calling / Tool Use Evaluation\n- **Input**: User query with available function definitions\n- **Output**: Correct function call with parameters\n- **Categories**: AST_NON_LIVE, AST_LIVE, RELEVANCE, MULTI_TURN\n\n## Key Features\n\n- Comprehensive function call evaluation\n- Tests simple, multiple, and parallel function calls\n- Multi-language support (Python, Java, JavaScript)\n- Relevance detection (irrelevance handling)\n- Multi-turn conversation function calling\n- Live API endpoints for execution testing\n\n## Evaluation Notes\n\n- Requires `pip install 'evalscope[bfcl]'` before evaluation\n- Set `is_fc_model=True` for models with native function calling support\n- `underscore_to_dot` parameter controls function name formatting\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v3.html) for detailed setup\n- Results broken down by category (AST, Relevance, Multi-turn)\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -329,11 +329,11 @@ "truncated": false }, "readme": { - "en": "# BFCL-v3\n\n\n## Overview\n\nBFCL (Berkeley Function Calling Leaderboard) v3 is the first comprehensive and executable function call evaluation benchmark for assessing LLMs' ability to invoke functions. It evaluates various forms of function calls, diverse scenarios, and executability.\n\n## Task Description\n\n- **Task Type**: Function Calling / Tool Use Evaluation\n- **Input**: User query with available function definitions\n- **Output**: Correct function call with parameters\n- **Categories**: AST_NON_LIVE, AST_LIVE, RELEVANCE, MULTI_TURN\n\n## Key Features\n\n- Comprehensive function call evaluation\n- Tests simple, multiple, and parallel function calls\n- Multi-language support (Python, Java, JavaScript)\n- Relevance detection (irrelevance handling)\n- Multi-turn conversation function calling\n- Live API endpoints for execution testing\n\n## Evaluation Notes\n\n- Requires `pip install bfcl-eval==2025.10.27.1` before evaluation\n- Set `is_fc_model=True` for models with native function calling support\n- `underscore_to_dot` parameter controls function name formatting\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v3.html) for detailed setup\n- Results broken down by category (AST, Relevance, Multi-turn)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `bfcl_v3` |\n| **Dataset ID** | [AI-ModelScope/bfcl_v3](https://modelscope.cn/datasets/AI-ModelScope/bfcl_v3/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `FunctionCalling` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 4,441 |\n| Prompt Length (Mean) | 291.25 chars |\n| Prompt Length (Min/Max) | 36 / 10805 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `simple` | 400 | 119.66 | 57 | 303 |\n| `multiple` | 200 | 120.75 | 65 | 229 |\n| `parallel` | 200 | 298.92 | 69 | 781 |\n| `parallel_multiple` | 200 | 432.17 | 88 | 1249 |\n| `java` | 100 | 228.93 | 138 | 531 |\n| `javascript` | 50 | 222.34 | 118 | 637 |\n| `live_simple` | 258 | 161.74 | 47 | 1247 |\n| `live_multiple` | 1,053 | 138.76 | 42 | 4828 |\n| `live_parallel` | 16 | 150.44 | 88 | 365 |\n| `live_parallel_multiple` | 24 | 217.67 | 72 | 650 |\n| `irrelevance` | 240 | 85.96 | 55 | 203 |\n| `live_relevance` | 18 | 263.83 | 71 | 1214 |\n| `live_irrelevance` | 882 | 188.26 | 36 | 10805 |\n| `multi_turn_base` | 200 | 803.33 | 106 | 1756 |\n| `multi_turn_miss_func` | 200 | 806.75 | 110 | 1760 |\n| `multi_turn_miss_param` | 200 | 858.2 | 167 | 1791 |\n| `multi_turn_long_context` | 200 | 803.29 | 106 | 1756 |\n\n## Sample Example\n\n**Subset**: `simple`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"15e84989\",\n \"content\": \"[[{\\\"role\\\": \\\"user\\\", \\\"content\\\": \\\"Find the area of a triangle with a base of 10 units and height of 5 units.\\\"}]]\"\n }\n ],\n \"target\": \"[{\\\"calculate_triangle_area\\\": {\\\"base\\\": [10], \\\"height\\\": [5], \\\"unit\\\": [\\\"units\\\", \\\"\\\"]}}]\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"simple\",\n \"metadata\": {\n \"id\": \"simple_0\",\n \"multi_turn\": false,\n \"functions\": [\n {\n \"name\": \"calculate_triangle_area\",\n \"description\": \"Calculate the area of a triangle given its base and height.\",\n \"parameters\": {\n \"type\": \"dict\",\n \"properties\": {\n \"base\": {\n \"type\": \"integer\",\n \"description\": \"The base of the triangle.\"\n },\n \"height\": {\n \"type\": \"integer\",\n \"description\": \"The height of the triangle.\"\n },\n \"unit\": {\n \"type\": \"string\",\n \"description\": \"The unit of measure (defaults to 'units' if not specified)\"\n }\n },\n \"required\": [\n \"base\",\n \"height\"\n ]\n }\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"calculate_triangle_area\",\n \"description\": \"Calculate the area of a triangle given its base and height. Note that the provided function is in Python 3 syntax.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"base\": {\n \"type\": \"integer\",\n \"description\": \"The base of the triangle.\"\n },\n \"height\": {\n \"type\": \"integer\",\n \"description\": \"The height of the triangle.\"\n },\n \"unit\": {\n \"type\": \"string\",\n \"description\": \"The unit of measure (defaults to 'units' if not specified)\"\n }\n },\n \"required\": [\n \"base\",\n \"height\"\n ]\n }\n }\n }\n ],\n \"missed_functions\": \"{}\",\n \"initial_config\": {},\n \"involved_classes\": [],\n \"turns\": [\n [\n {\n \"role\": \"user\",\n \"content\": \"Find the area of a triangle with a base of 10 units and height of 5 units.\"\n }\n ]\n ],\n \"language\": \"Python\",\n \"test_category\": \"simple\",\n \"subset\": \"simple\",\n \"ground_truth\": [\n {\n \"calculate_triangle_area\": {\n \"base\": [\n 10\n ],\n \"height\": [\n 5\n ],\n \"unit\": [\n \"units\",\n \"\"\n ]\n }\n }\n ],\n \"should_execute_tool_calls\": false,\n \"missing_functions\": {},\n \"is_fc_model\": true\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `underscore_to_dot` | `bool` | `True` | Convert underscores to dots in function names for evaluation. |\n| `is_fc_model` | `bool` | `True` | Indicates the evaluated model natively supports function calling. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bfcl_v3 \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bfcl_v3'],\n dataset_args={\n 'bfcl_v3': {\n # subset_list: ['simple', 'multiple', 'parallel'] # optional, evaluate specific subsets\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# BFCL-v3\n\n\n## 概述\n\nBFCL(Berkeley Function Calling Leaderboard)v3 是首个全面且可执行的函数调用评估基准,用于评估大语言模型(LLM)调用函数的能力。它评估了多种形式的函数调用、多样化的场景以及可执行性。\n\n## 任务描述\n\n- **任务类型**:函数调用 / 工具使用评估\n- **输入**:包含可用函数定义的用户查询\n- **输出**:带有正确参数的函数调用\n- **类别**:AST_NON_LIVE、AST_LIVE、RELEVANCE、MULTI_TURN\n\n## 主要特性\n\n- 全面的函数调用评估\n- 测试简单、多重和并行函数调用\n- 多语言支持(Python、Java、JavaScript)\n- 相关性检测(处理无关查询)\n- 多轮对话中的函数调用\n- 提供实时 API 端点用于执行测试\n\n## 评估说明\n\n- 评估前需安装 `pip install bfcl-eval==2025.10.27.1`\n- 对于原生支持函数调用的模型,请设置 `is_fc_model=True`\n- `underscore_to_dot` 参数控制函数名称的格式化方式\n- 详细设置请参阅 [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html)\n- 结果按类别(AST、相关性、多轮)细分\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `bfcl_v3` |\n| **数据集ID** | [AI-ModelScope/bfcl_v3](https://modelscope.cn/datasets/AI-ModelScope/bfcl_v3/summary) |\n| **论文** | N/A |\n| **标签** | `Agent`, `FunctionCalling` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 4,441 |\n| 提示词长度(平均) | 291.25 字符 |\n| 提示词长度(最小/最大) | 36 / 10805 字符 |\n\n**各子集统计数据:**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `simple` | 400 | 119.66 | 57 | 303 |\n| `multiple` | 200 | 120.75 | 65 | 229 |\n| `parallel` | 200 | 298.92 | 69 | 781 |\n| `parallel_multiple` | 200 | 432.17 | 88 | 1249 |\n| `java` | 100 | 228.93 | 138 | 531 |\n| `javascript` | 50 | 222.34 | 118 | 637 |\n| `live_simple` | 258 | 161.74 | 47 | 1247 |\n| `live_multiple` | 1,053 | 138.76 | 42 | 4828 |\n| `live_parallel` | 16 | 150.44 | 88 | 365 |\n| `live_parallel_multiple` | 24 | 217.67 | 72 | 650 |\n| `irrelevance` | 240 | 85.96 | 55 | 203 |\n| `live_relevance` | 18 | 263.83 | 71 | 1214 |\n| `live_irrelevance` | 882 | 188.26 | 36 | 10805 |\n| `multi_turn_base` | 200 | 803.33 | 106 | 1756 |\n| `multi_turn_miss_func` | 200 | 806.75 | 110 | 1760 |\n| `multi_turn_miss_param` | 200 | 858.2 | 167 | 1791 |\n| `multi_turn_long_context` | 200 | 803.29 | 106 | 1756 |\n\n## 样例示例\n\n**子集**:`simple`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"15e84989\",\n \"content\": \"[[{\\\"role\\\": \\\"user\\\", \\\"content\\\": \\\"Find the area of a triangle with a base of 10 units and height of 5 units.\\\"}]]\"\n }\n ],\n \"target\": \"[{\\\"calculate_triangle_area\\\": {\\\"base\\\": [10], \\\"height\\\": [5], \\\"unit\\\": [\\\"units\\\", \\\"\\\"]}}]\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"simple\",\n \"metadata\": {\n \"id\": \"simple_0\",\n \"multi_turn\": false,\n \"functions\": [\n {\n \"name\": \"calculate_triangle_area\",\n \"description\": \"Calculate the area of a triangle given its base and height.\",\n \"parameters\": {\n \"type\": \"dict\",\n \"properties\": {\n \"base\": {\n \"type\": \"integer\",\n \"description\": \"The base of the triangle.\"\n },\n \"height\": {\n \"type\": \"integer\",\n \"description\": \"The height of the triangle.\"\n },\n \"unit\": {\n \"type\": \"string\",\n \"description\": \"The unit of measure (defaults to 'units' if not specified)\"\n }\n },\n \"required\": [\n \"base\",\n \"height\"\n ]\n }\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"calculate_triangle_area\",\n \"description\": \"Calculate the area of a triangle given its base and height. Note that the provided function is in Python 3 syntax.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"base\": {\n \"type\": \"integer\",\n \"description\": \"The base of the triangle.\"\n },\n \"height\": {\n \"type\": \"integer\",\n \"description\": \"The height of the triangle.\"\n },\n \"unit\": {\n \"type\": \"string\",\n \"description\": \"The unit of measure (defaults to 'units' if not specified)\"\n }\n },\n \"required\": [\n \"base\",\n \"height\"\n ]\n }\n }\n }\n ],\n \"missed_functions\": \"{}\",\n \"initial_config\": {},\n \"involved_classes\": [],\n \"turns\": [\n [\n {\n \"role\": \"user\",\n \"content\": \"Find the area of a triangle with a base of 10 units and height of 5 units.\"\n }\n ]\n ],\n \"language\": \"Python\",\n \"test_category\": \"simple\",\n \"subset\": \"simple\",\n \"ground_truth\": [\n {\n \"calculate_triangle_area\": {\n \"base\": [\n 10\n ],\n \"height\": [\n 5\n ],\n \"unit\": [\n \"units\",\n \"\"\n ]\n }\n }\n ],\n \"should_execute_tool_calls\": false,\n \"missing_functions\": {},\n \"is_fc_model\": true\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `underscore_to_dot` | `bool` | `True` | 在评估时将函数名中的下划线转换为点号。 |\n| `is_fc_model` | `bool` | `True` | 表示被评估的模型原生支持函数调用。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bfcl_v3 \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bfcl_v3'],\n dataset_args={\n 'bfcl_v3': {\n # subset_list: ['simple', 'multiple', 'parallel'] # 可选,评估特定子集\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "ab3e4011072d6057e3f368019bc7c90c", + "en": "# BFCL-v3\n\n\n## Overview\n\nBFCL (Berkeley Function Calling Leaderboard) v3 is the first comprehensive and executable function call evaluation benchmark for assessing LLMs' ability to invoke functions. It evaluates various forms of function calls, diverse scenarios, and executability.\n\n## Task Description\n\n- **Task Type**: Function Calling / Tool Use Evaluation\n- **Input**: User query with available function definitions\n- **Output**: Correct function call with parameters\n- **Categories**: AST_NON_LIVE, AST_LIVE, RELEVANCE, MULTI_TURN\n\n## Key Features\n\n- Comprehensive function call evaluation\n- Tests simple, multiple, and parallel function calls\n- Multi-language support (Python, Java, JavaScript)\n- Relevance detection (irrelevance handling)\n- Multi-turn conversation function calling\n- Live API endpoints for execution testing\n\n## Evaluation Notes\n\n- Requires `pip install 'evalscope[bfcl]'` before evaluation\n- Set `is_fc_model=True` for models with native function calling support\n- `underscore_to_dot` parameter controls function name formatting\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v3.html) for detailed setup\n- Results broken down by category (AST, Relevance, Multi-turn)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `bfcl_v3` |\n| **Dataset ID** | [AI-ModelScope/bfcl_v3](https://modelscope.cn/datasets/AI-ModelScope/bfcl_v3/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `FunctionCalling` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 4,441 |\n| Prompt Length (Mean) | 291.25 chars |\n| Prompt Length (Min/Max) | 36 / 10805 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `simple` | 400 | 119.66 | 57 | 303 |\n| `multiple` | 200 | 120.75 | 65 | 229 |\n| `parallel` | 200 | 298.92 | 69 | 781 |\n| `parallel_multiple` | 200 | 432.17 | 88 | 1249 |\n| `java` | 100 | 228.93 | 138 | 531 |\n| `javascript` | 50 | 222.34 | 118 | 637 |\n| `live_simple` | 258 | 161.74 | 47 | 1247 |\n| `live_multiple` | 1,053 | 138.76 | 42 | 4828 |\n| `live_parallel` | 16 | 150.44 | 88 | 365 |\n| `live_parallel_multiple` | 24 | 217.67 | 72 | 650 |\n| `irrelevance` | 240 | 85.96 | 55 | 203 |\n| `live_relevance` | 18 | 263.83 | 71 | 1214 |\n| `live_irrelevance` | 882 | 188.26 | 36 | 10805 |\n| `multi_turn_base` | 200 | 803.33 | 106 | 1756 |\n| `multi_turn_miss_func` | 200 | 806.75 | 110 | 1760 |\n| `multi_turn_miss_param` | 200 | 858.2 | 167 | 1791 |\n| `multi_turn_long_context` | 200 | 803.29 | 106 | 1756 |\n\n## Sample Example\n\n**Subset**: `simple`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"15e84989\",\n \"content\": \"[[{\\\"role\\\": \\\"user\\\", \\\"content\\\": \\\"Find the area of a triangle with a base of 10 units and height of 5 units.\\\"}]]\"\n }\n ],\n \"target\": \"[{\\\"calculate_triangle_area\\\": {\\\"base\\\": [10], \\\"height\\\": [5], \\\"unit\\\": [\\\"units\\\", \\\"\\\"]}}]\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"simple\",\n \"metadata\": {\n \"id\": \"simple_0\",\n \"multi_turn\": false,\n \"functions\": [\n {\n \"name\": \"calculate_triangle_area\",\n \"description\": \"Calculate the area of a triangle given its base and height.\",\n \"parameters\": {\n \"type\": \"dict\",\n \"properties\": {\n \"base\": {\n \"type\": \"integer\",\n \"description\": \"The base of the triangle.\"\n },\n \"height\": {\n \"type\": \"integer\",\n \"description\": \"The height of the triangle.\"\n },\n \"unit\": {\n \"type\": \"string\",\n \"description\": \"The unit of measure (defaults to 'units' if not specified)\"\n }\n },\n \"required\": [\n \"base\",\n \"height\"\n ]\n }\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"calculate_triangle_area\",\n \"description\": \"Calculate the area of a triangle given its base and height. Note that the provided function is in Python 3 syntax.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"base\": {\n \"type\": \"integer\",\n \"description\": \"The base of the triangle.\"\n },\n \"height\": {\n \"type\": \"integer\",\n \"description\": \"The height of the triangle.\"\n },\n \"unit\": {\n \"type\": \"string\",\n \"description\": \"The unit of measure (defaults to 'units' if not specified)\"\n }\n },\n \"required\": [\n \"base\",\n \"height\"\n ]\n }\n }\n }\n ],\n \"missed_functions\": \"{}\",\n \"initial_config\": {},\n \"involved_classes\": [],\n \"turns\": [\n [\n {\n \"role\": \"user\",\n \"content\": \"Find the area of a triangle with a base of 10 units and height of 5 units.\"\n }\n ]\n ],\n \"language\": \"Python\",\n \"test_category\": \"simple\",\n \"subset\": \"simple\",\n \"ground_truth\": [\n {\n \"calculate_triangle_area\": {\n \"base\": [\n 10\n ],\n \"height\": [\n 5\n ],\n \"unit\": [\n \"units\",\n \"\"\n ]\n }\n }\n ],\n \"should_execute_tool_calls\": false,\n \"missing_functions\": {},\n \"is_fc_model\": true\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `underscore_to_dot` | `bool` | `True` | Convert underscores to dots in function names for evaluation. |\n| `is_fc_model` | `bool` | `True` | Indicates the evaluated model natively supports function calling. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bfcl_v3 \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bfcl_v3'],\n dataset_args={\n 'bfcl_v3': {\n # subset_list: ['simple', 'multiple', 'parallel'] # optional, evaluate specific subsets\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# BFCL-v3\n\n\n## 概述\n\nBFCL(Berkeley Function Calling Leaderboard)v3 是首个全面且可执行的函数调用评估基准,用于评估大语言模型(LLM)调用函数的能力。它评估了多种形式的函数调用、多样化的场景以及可执行性。\n\n## 任务描述\n\n- **任务类型**:函数调用 / 工具使用评估\n- **输入**:包含可用函数定义的用户查询\n- **输出**:带有正确参数的函数调用\n- **类别**:AST_NON_LIVE、AST_LIVE、RELEVANCE、MULTI_TURN\n\n## 主要特性\n\n- 全面的函数调用评估\n- 测试简单、多重和并行函数调用\n- 多语言支持(Python、Java、JavaScript)\n- 相关性检测(处理无关查询)\n- 多轮对话中的函数调用\n- 提供实时 API 端点用于执行测试\n\n## 评估说明\n\n- 评估前需安装 `pip install 'evalscope[bfcl]'`\n- 对于原生支持函数调用的模型,请设置 `is_fc_model=True`\n- `underscore_to_dot` 参数控制函数名称的格式化方式\n- 详细设置请参阅 [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html)\n- 结果按类别(AST、相关性、多轮)细分\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `bfcl_v3` |\n| **数据集ID** | [AI-ModelScope/bfcl_v3](https://modelscope.cn/datasets/AI-ModelScope/bfcl_v3/summary) |\n| **论文** | N/A |\n| **标签** | `Agent`, `FunctionCalling` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 4,441 |\n| 提示词长度(平均) | 291.25 字符 |\n| 提示词长度(最小/最大) | 36 / 10805 字符 |\n\n**各子集统计数据:**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `simple` | 400 | 119.66 | 57 | 303 |\n| `multiple` | 200 | 120.75 | 65 | 229 |\n| `parallel` | 200 | 298.92 | 69 | 781 |\n| `parallel_multiple` | 200 | 432.17 | 88 | 1249 |\n| `java` | 100 | 228.93 | 138 | 531 |\n| `javascript` | 50 | 222.34 | 118 | 637 |\n| `live_simple` | 258 | 161.74 | 47 | 1247 |\n| `live_multiple` | 1,053 | 138.76 | 42 | 4828 |\n| `live_parallel` | 16 | 150.44 | 88 | 365 |\n| `live_parallel_multiple` | 24 | 217.67 | 72 | 650 |\n| `irrelevance` | 240 | 85.96 | 55 | 203 |\n| `live_relevance` | 18 | 263.83 | 71 | 1214 |\n| `live_irrelevance` | 882 | 188.26 | 36 | 10805 |\n| `multi_turn_base` | 200 | 803.33 | 106 | 1756 |\n| `multi_turn_miss_func` | 200 | 806.75 | 110 | 1760 |\n| `multi_turn_miss_param` | 200 | 858.2 | 167 | 1791 |\n| `multi_turn_long_context` | 200 | 803.29 | 106 | 1756 |\n\n## 样例示例\n\n**子集**:`simple`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"15e84989\",\n \"content\": \"[[{\\\"role\\\": \\\"user\\\", \\\"content\\\": \\\"Find the area of a triangle with a base of 10 units and height of 5 units.\\\"}]]\"\n }\n ],\n \"target\": \"[{\\\"calculate_triangle_area\\\": {\\\"base\\\": [10], \\\"height\\\": [5], \\\"unit\\\": [\\\"units\\\", \\\"\\\"]}}]\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"simple\",\n \"metadata\": {\n \"id\": \"simple_0\",\n \"multi_turn\": false,\n \"functions\": [\n {\n \"name\": \"calculate_triangle_area\",\n \"description\": \"Calculate the area of a triangle given its base and height.\",\n \"parameters\": {\n \"type\": \"dict\",\n \"properties\": {\n \"base\": {\n \"type\": \"integer\",\n \"description\": \"The base of the triangle.\"\n },\n \"height\": {\n \"type\": \"integer\",\n \"description\": \"The height of the triangle.\"\n },\n \"unit\": {\n \"type\": \"string\",\n \"description\": \"The unit of measure (defaults to 'units' if not specified)\"\n }\n },\n \"required\": [\n \"base\",\n \"height\"\n ]\n }\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"calculate_triangle_area\",\n \"description\": \"Calculate the area of a triangle given its base and height. Note that the provided function is in Python 3 syntax.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"base\": {\n \"type\": \"integer\",\n \"description\": \"The base of the triangle.\"\n },\n \"height\": {\n \"type\": \"integer\",\n \"description\": \"The height of the triangle.\"\n },\n \"unit\": {\n \"type\": \"string\",\n \"description\": \"The unit of measure (defaults to 'units' if not specified)\"\n }\n },\n \"required\": [\n \"base\",\n \"height\"\n ]\n }\n }\n }\n ],\n \"missed_functions\": \"{}\",\n \"initial_config\": {},\n \"involved_classes\": [],\n \"turns\": [\n [\n {\n \"role\": \"user\",\n \"content\": \"Find the area of a triangle with a base of 10 units and height of 5 units.\"\n }\n ]\n ],\n \"language\": \"Python\",\n \"test_category\": \"simple\",\n \"subset\": \"simple\",\n \"ground_truth\": [\n {\n \"calculate_triangle_area\": {\n \"base\": [\n 10\n ],\n \"height\": [\n 5\n ],\n \"unit\": [\n \"units\",\n \"\"\n ]\n }\n }\n ],\n \"should_execute_tool_calls\": false,\n \"missing_functions\": {},\n \"is_fc_model\": true\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `underscore_to_dot` | `bool` | `True` | 在评估时将函数名中的下划线转换为点号。 |\n| `is_fc_model` | `bool` | `True` | 表示被评估的模型原生支持函数调用。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bfcl_v3 \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bfcl_v3'],\n dataset_args={\n 'bfcl_v3': {\n # subset_list: ['simple', 'multiple', 'parallel'] # 可选,评估特定子集\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "2dd608360477a73e329abaff443664c6", "needs_translation": false }, - "updated_at": "2026-08-11T15:30:53.581856", - "translation_updated_at": "2026-01-28T15:56:15Z" + "updated_at": "2026-11-15T01:27:01.683609", + "translation_updated_at": "2026-11-15T01:27:13" } diff --git a/evalscope/benchmarks/_meta/bfcl_v4.json b/evalscope/benchmarks/_meta/bfcl_v4.json index e26e28774..6308f1d81 100644 --- a/evalscope/benchmarks/_meta/bfcl_v4.json +++ b/evalscope/benchmarks/_meta/bfcl_v4.json @@ -37,7 +37,7 @@ "web_search_base", "web_search_no_snippet" ], - "description": "\n## Overview\n\nBFCL-v4 (Berkeley Function-Calling Leaderboard V4) is a comprehensive benchmark for evaluating agentic function-calling capabilities of LLMs. It tests web search, memory operations, and format sensitivity as building blocks for agentic applications.\n\n## Task Description\n\n- **Task Type**: Agentic Function Calling Evaluation\n- **Input**: Scenarios requiring function calls for web search, memory, etc.\n- **Output**: Correct function calls with proper arguments\n- **Capabilities**: Web search, memory read/write, format handling\n\n## Key Features\n\n- Holistic agentic evaluation framework\n- Tests building blocks for LLM agents (search, memory, formatting)\n- Multiple scoring categories for detailed analysis\n- Supports both FC models and non-FC models\n- Optional web search with SerpAPI integration\n\n## Evaluation Notes\n\n- **Installation Required**: `pip install bfcl-eval==2025.10.27.1`\n- Primary metric: **Accuracy** per category\n- Configure `is_fc_model` based on model capabilities\n- Optional: Set `SERPAPI_API_KEY` for web search tasks\n- [Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v4.html)\n", + "description": "\n## Overview\n\nBFCL-v4 (Berkeley Function-Calling Leaderboard V4) is a comprehensive benchmark for evaluating agentic function-calling capabilities of LLMs. It tests web search, memory operations, and format sensitivity as building blocks for agentic applications.\n\n## Task Description\n\n- **Task Type**: Agentic Function Calling Evaluation\n- **Input**: Scenarios requiring function calls for web search, memory, etc.\n- **Output**: Correct function calls with proper arguments\n- **Capabilities**: Web search, memory read/write, format handling\n\n## Key Features\n\n- Holistic agentic evaluation framework\n- Tests building blocks for LLM agents (search, memory, formatting)\n- Multiple scoring categories for detailed analysis\n- Supports both FC models and non-FC models\n- Optional web search with SerpAPI integration\n\n## Evaluation Notes\n\n- **Installation Required**: `pip install 'evalscope[bfcl]'`\n- Primary metric: **Accuracy** per category\n- Configure `is_fc_model` based on model capabilities\n- Optional: Set `SERPAPI_API_KEY` for web search tasks\n- [Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v4.html)\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -325,11 +325,11 @@ "truncated": false }, "readme": { - "en": "# BFCL-v4\n\n\n## Overview\n\nBFCL-v4 (Berkeley Function-Calling Leaderboard V4) is a comprehensive benchmark for evaluating agentic function-calling capabilities of LLMs. It tests web search, memory operations, and format sensitivity as building blocks for agentic applications.\n\n## Task Description\n\n- **Task Type**: Agentic Function Calling Evaluation\n- **Input**: Scenarios requiring function calls for web search, memory, etc.\n- **Output**: Correct function calls with proper arguments\n- **Capabilities**: Web search, memory read/write, format handling\n\n## Key Features\n\n- Holistic agentic evaluation framework\n- Tests building blocks for LLM agents (search, memory, formatting)\n- Multiple scoring categories for detailed analysis\n- Supports both FC models and non-FC models\n- Optional web search with SerpAPI integration\n\n## Evaluation Notes\n\n- **Installation Required**: `pip install bfcl-eval==2025.10.27.1`\n- Primary metric: **Accuracy** per category\n- Configure `is_fc_model` based on model capabilities\n- Optional: Set `SERPAPI_API_KEY` for web search tasks\n- [Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v4.html)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `bfcl_v4` |\n| **Dataset ID** | [berkeley-function-call-leaderboard](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `FunctionCalling` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 5,106 |\n| Prompt Length (Mean) | 350.4 chars |\n| Prompt Length (Min/Max) | 36 / 10805 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `irrelevance` | 240 | 85.98 | 55 | 203 |\n| `live_irrelevance` | 884 | 198.72 | 36 | 10805 |\n| `live_multiple` | 1,053 | 149.16 | 42 | 4828 |\n| `live_parallel` | 16 | 163.88 | 88 | 365 |\n| `live_parallel_multiple` | 24 | 217.67 | 72 | 650 |\n| `live_relevance` | 16 | 385.94 | 71 | 1830 |\n| `live_simple` | 258 | 171.91 | 47 | 1247 |\n| `memory_kv` | 155 | 646.46 | 585 | 826 |\n| `memory_rec_sum` | 155 | 646.46 | 585 | 826 |\n| `memory_vector` | 155 | 646.46 | 585 | 826 |\n| `multi_turn_base` | 200 | 808.43 | 106 | 1756 |\n| `multi_turn_long_context` | 200 | 808.41 | 106 | 1756 |\n| `multi_turn_miss_func` | 200 | 811.88 | 110 | 1760 |\n| `multi_turn_miss_param` | 200 | 863.88 | 167 | 1791 |\n| `multiple` | 200 | 120.75 | 65 | 229 |\n| `parallel` | 200 | 297.93 | 69 | 781 |\n| `parallel_multiple` | 200 | 432.17 | 88 | 1249 |\n| `simple_java` | 100 | 228.93 | 138 | 531 |\n| `simple_javascript` | 50 | 222.34 | 118 | 637 |\n| `simple_python` | 400 | 119.66 | 57 | 303 |\n| `web_search_base` | 100 | 831.01 | 748 | 963 |\n| `web_search_no_snippet` | 100 | 831.01 | 748 | 963 |\n\n## Sample Example\n\n**Subset**: `irrelevance`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4e05e910\",\n \"content\": \"[[{\\\"role\\\": \\\"user\\\", \\\"content\\\": \\\"Calculate the area of a triangle given the base is 10 meters and height is 5 meters.\\\"}]]\"\n }\n ],\n \"target\": \"{}\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"id\": \"irrelevance_0\",\n \"question\": [\n [\n {\n \"role\": \"user\",\n \"content\": \"Calculate the area of a triangle given the base is 10 meters and height is 5 meters.\"\n }\n ]\n ],\n \"function\": [\n {\n \"name\": \"determine_body_mass_index\",\n \"description\": \"Calculate body mass index given weight and height.\",\n \"parameters\": {\n \"type\": \"dict\",\n \"properties\": {\n \"weight\": {\n \"type\": \"float\",\n \"description\": \"Weight of the individual in kilograms.\"\n },\n \"height\": {\n \"type\": \"float\",\n \"description\": \"Height of the individual in meters.\"\n }\n },\n \"required\": [\n \"weight\",\n \"height\"\n ]\n }\n }\n ],\n \"category\": \"irrelevance\",\n \"ground_truth\": {}\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `underscore_to_dot` | `bool` | `True` | Convert underscores to dots in function names for evaluation. |\n| `is_fc_model` | `bool` | `True` | Indicates the evaluated model natively supports function calling. |\n| `SERPAPI_API_KEY` | `str | null` | `None` | SerpAPI key enabling web-search capability in BFCL V4. Null disables web search. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bfcl_v4 \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bfcl_v4'],\n dataset_args={\n 'bfcl_v4': {\n # subset_list: ['irrelevance', 'live_irrelevance', 'live_multiple'] # optional, evaluate specific subsets\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# BFCL-v4\n\n\n## 概述\n\nBFCL-v4(Berkeley Function-Calling Leaderboard V4)是一个全面的基准测试,用于评估大语言模型(LLM)在智能体场景下的函数调用能力。它通过测试网络搜索、内存操作和格式敏感性等核心模块,为构建智能体应用提供基础能力评估。\n\n## 任务描述\n\n- **任务类型**:智能体函数调用评估\n- **输入**:需要调用函数完成网络搜索、内存操作等任务的场景\n- **输出**:带有正确参数的函数调用\n- **能力范围**:网络搜索、内存读写、格式处理\n\n## 核心特性\n\n- 全面的智能体评估框架\n- 测试 LLM 智能体的核心构建模块(搜索、内存、格式化)\n- 多维度评分类别,支持细粒度分析\n- 同时支持原生函数调用(FC)模型和非 FC 模型\n- 可选集成 SerpAPI 实现网络搜索功能\n\n## 评估说明\n\n- **安装要求**:`pip install bfcl-eval==2025.10.27.1`\n- 主要指标:各类别下的 **准确率(Accuracy)**\n- 根据模型能力设置 `is_fc_model` 参数\n- 可选:配置 `SERPAPI_API_KEY` 以启用网络搜索任务\n- [使用示例](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v4.html)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `bfcl_v4` |\n| **数据集ID** | [berkeley-function-call-leaderboard](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) |\n| **论文** | N/A |\n| **标签** | `Agent`, `FunctionCalling` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 5,106 |\n| 提示词长度(平均) | 350.4 字符 |\n| 提示词长度(最小/最大) | 36 / 10805 字符 |\n\n**各子集统计数据:**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `irrelevance` | 240 | 85.98 | 55 | 203 |\n| `live_irrelevance` | 884 | 198.72 | 36 | 10805 |\n| `live_multiple` | 1,053 | 149.16 | 42 | 4828 |\n| `live_parallel` | 16 | 163.88 | 88 | 365 |\n| `live_parallel_multiple` | 24 | 217.67 | 72 | 650 |\n| `live_relevance` | 16 | 385.94 | 71 | 1830 |\n| `live_simple` | 258 | 171.91 | 47 | 1247 |\n| `memory_kv` | 155 | 646.46 | 585 | 826 |\n| `memory_rec_sum` | 155 | 646.46 | 585 | 826 |\n| `memory_vector` | 155 | 646.46 | 585 | 826 |\n| `multi_turn_base` | 200 | 808.43 | 106 | 1756 |\n| `multi_turn_long_context` | 200 | 808.41 | 106 | 1756 |\n| `multi_turn_miss_func` | 200 | 811.88 | 110 | 1760 |\n| `multi_turn_miss_param` | 200 | 863.88 | 167 | 1791 |\n| `multiple` | 200 | 120.75 | 65 | 229 |\n| `parallel` | 200 | 297.93 | 69 | 781 |\n| `parallel_multiple` | 200 | 432.17 | 88 | 1249 |\n| `simple_java` | 100 | 228.93 | 138 | 531 |\n| `simple_javascript` | 50 | 222.34 | 118 | 637 |\n| `simple_python` | 400 | 119.66 | 57 | 303 |\n| `web_search_base` | 100 | 831.01 | 748 | 963 |\n| `web_search_no_snippet` | 100 | 831.01 | 748 | 963 |\n\n## 样例示例\n\n**子集**:`irrelevance`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4e05e910\",\n \"content\": \"[[{\\\"role\\\": \\\"user\\\", \\\"content\\\": \\\"Calculate the area of a triangle given the base is 10 meters and height is 5 meters.\\\"}]]\"\n }\n ],\n \"target\": \"{}\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"id\": \"irrelevance_0\",\n \"question\": [\n [\n {\n \"role\": \"user\",\n \"content\": \"Calculate the area of a triangle given the base is 10 meters and height is 5 meters.\"\n }\n ]\n ],\n \"function\": [\n {\n \"name\": \"determine_body_mass_index\",\n \"description\": \"Calculate body mass index given weight and height.\",\n \"parameters\": {\n \"type\": \"dict\",\n \"properties\": {\n \"weight\": {\n \"type\": \"float\",\n \"description\": \"Weight of the individual in kilograms.\"\n },\n \"height\": {\n \"type\": \"float\",\n \"description\": \"Height of the individual in meters.\"\n }\n },\n \"required\": [\n \"weight\",\n \"height\"\n ]\n }\n }\n ],\n \"category\": \"irrelevance\",\n \"ground_truth\": {}\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `underscore_to_dot` | `bool` | `True` | 在评估时将函数名中的下划线转换为点号。 |\n| `is_fc_model` | `bool` | `True` | 指示被评估模型是否原生支持函数调用。 |\n| `SERPAPI_API_KEY` | `str | null` | `None` | 启用 BFCL V4 中网络搜索功能的 SerpAPI 密钥。设为 null 则禁用网络搜索。 |\n\n## 使用方法\n\n### 通过 CLI 使用\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bfcl_v4 \\\n --limit 10 # 正式评估时请移除此行\n```\n\n### 通过 Python 使用\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bfcl_v4'],\n dataset_args={\n 'bfcl_v4': {\n # subset_list: ['irrelevance', 'live_irrelevance', 'live_multiple'] # 可选,仅评估指定子集\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请移除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "728115e036ffb4bcf8e8b17787c6881f", + "en": "# BFCL-v4\n\n\n## Overview\n\nBFCL-v4 (Berkeley Function-Calling Leaderboard V4) is a comprehensive benchmark for evaluating agentic function-calling capabilities of LLMs. It tests web search, memory operations, and format sensitivity as building blocks for agentic applications.\n\n## Task Description\n\n- **Task Type**: Agentic Function Calling Evaluation\n- **Input**: Scenarios requiring function calls for web search, memory, etc.\n- **Output**: Correct function calls with proper arguments\n- **Capabilities**: Web search, memory read/write, format handling\n\n## Key Features\n\n- Holistic agentic evaluation framework\n- Tests building blocks for LLM agents (search, memory, formatting)\n- Multiple scoring categories for detailed analysis\n- Supports both FC models and non-FC models\n- Optional web search with SerpAPI integration\n\n## Evaluation Notes\n\n- **Installation Required**: `pip install 'evalscope[bfcl]'`\n- Primary metric: **Accuracy** per category\n- Configure `is_fc_model` based on model capabilities\n- Optional: Set `SERPAPI_API_KEY` for web search tasks\n- [Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v4.html)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `bfcl_v4` |\n| **Dataset ID** | [berkeley-function-call-leaderboard](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `FunctionCalling` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 5,106 |\n| Prompt Length (Mean) | 350.4 chars |\n| Prompt Length (Min/Max) | 36 / 10805 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `irrelevance` | 240 | 85.98 | 55 | 203 |\n| `live_irrelevance` | 884 | 198.72 | 36 | 10805 |\n| `live_multiple` | 1,053 | 149.16 | 42 | 4828 |\n| `live_parallel` | 16 | 163.88 | 88 | 365 |\n| `live_parallel_multiple` | 24 | 217.67 | 72 | 650 |\n| `live_relevance` | 16 | 385.94 | 71 | 1830 |\n| `live_simple` | 258 | 171.91 | 47 | 1247 |\n| `memory_kv` | 155 | 646.46 | 585 | 826 |\n| `memory_rec_sum` | 155 | 646.46 | 585 | 826 |\n| `memory_vector` | 155 | 646.46 | 585 | 826 |\n| `multi_turn_base` | 200 | 808.43 | 106 | 1756 |\n| `multi_turn_long_context` | 200 | 808.41 | 106 | 1756 |\n| `multi_turn_miss_func` | 200 | 811.88 | 110 | 1760 |\n| `multi_turn_miss_param` | 200 | 863.88 | 167 | 1791 |\n| `multiple` | 200 | 120.75 | 65 | 229 |\n| `parallel` | 200 | 297.93 | 69 | 781 |\n| `parallel_multiple` | 200 | 432.17 | 88 | 1249 |\n| `simple_java` | 100 | 228.93 | 138 | 531 |\n| `simple_javascript` | 50 | 222.34 | 118 | 637 |\n| `simple_python` | 400 | 119.66 | 57 | 303 |\n| `web_search_base` | 100 | 831.01 | 748 | 963 |\n| `web_search_no_snippet` | 100 | 831.01 | 748 | 963 |\n\n## Sample Example\n\n**Subset**: `irrelevance`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4e05e910\",\n \"content\": \"[[{\\\"role\\\": \\\"user\\\", \\\"content\\\": \\\"Calculate the area of a triangle given the base is 10 meters and height is 5 meters.\\\"}]]\"\n }\n ],\n \"target\": \"{}\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"id\": \"irrelevance_0\",\n \"question\": [\n [\n {\n \"role\": \"user\",\n \"content\": \"Calculate the area of a triangle given the base is 10 meters and height is 5 meters.\"\n }\n ]\n ],\n \"function\": [\n {\n \"name\": \"determine_body_mass_index\",\n \"description\": \"Calculate body mass index given weight and height.\",\n \"parameters\": {\n \"type\": \"dict\",\n \"properties\": {\n \"weight\": {\n \"type\": \"float\",\n \"description\": \"Weight of the individual in kilograms.\"\n },\n \"height\": {\n \"type\": \"float\",\n \"description\": \"Height of the individual in meters.\"\n }\n },\n \"required\": [\n \"weight\",\n \"height\"\n ]\n }\n }\n ],\n \"category\": \"irrelevance\",\n \"ground_truth\": {}\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `underscore_to_dot` | `bool` | `True` | Convert underscores to dots in function names for evaluation. |\n| `is_fc_model` | `bool` | `True` | Indicates the evaluated model natively supports function calling. |\n| `SERPAPI_API_KEY` | `str | null` | `None` | SerpAPI key enabling web-search capability in BFCL V4. Null disables web search. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bfcl_v4 \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bfcl_v4'],\n dataset_args={\n 'bfcl_v4': {\n # subset_list: ['irrelevance', 'live_irrelevance', 'live_multiple'] # optional, evaluate specific subsets\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# BFCL-v4\n\n\n## 概述\n\nBFCL-v4(Berkeley Function-Calling Leaderboard V4)是一个全面的基准测试,用于评估大语言模型(LLM)在智能体场景下的函数调用能力。它通过测试网络搜索、内存操作和格式敏感性等核心模块,为构建智能体应用提供基础能力评估。\n\n## 任务描述\n\n- **任务类型**:智能体函数调用评估\n- **输入**:需要调用函数完成网络搜索、内存操作等任务的场景\n- **输出**:带有正确参数的函数调用\n- **能力范围**:网络搜索、内存读写、格式处理\n\n## 核心特性\n\n- 全面的智能体评估框架\n- 测试 LLM 智能体的核心构建模块(搜索、内存、格式化)\n- 多维度评分类别,支持细粒度分析\n- 同时支持原生函数调用(FC)模型和非 FC 模型\n- 可选集成 SerpAPI 实现网络搜索功能\n\n## 评估说明\n\n- **安装要求**:`pip install 'evalscope[bfcl]'`\n- 主要指标:各类别下的 **准确率(Accuracy)**\n- 根据模型能力设置 `is_fc_model` 参数\n- 可选:配置 `SERPAPI_API_KEY` 以启用网络搜索任务\n- [使用示例](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v4.html)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `bfcl_v4` |\n| **数据集ID** | [berkeley-function-call-leaderboard](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) |\n| **论文** | N/A |\n| **标签** | `Agent`, `FunctionCalling` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 5,106 |\n| 提示词长度(平均) | 350.4 字符 |\n| 提示词长度(最小/最大) | 36 / 10805 字符 |\n\n**各子集统计数据:**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `irrelevance` | 240 | 85.98 | 55 | 203 |\n| `live_irrelevance` | 884 | 198.72 | 36 | 10805 |\n| `live_multiple` | 1,053 | 149.16 | 42 | 4828 |\n| `live_parallel` | 16 | 163.88 | 88 | 365 |\n| `live_parallel_multiple` | 24 | 217.67 | 72 | 650 |\n| `live_relevance` | 16 | 385.94 | 71 | 1830 |\n| `live_simple` | 258 | 171.91 | 47 | 1247 |\n| `memory_kv` | 155 | 646.46 | 585 | 826 |\n| `memory_rec_sum` | 155 | 646.46 | 585 | 826 |\n| `memory_vector` | 155 | 646.46 | 585 | 826 |\n| `multi_turn_base` | 200 | 808.43 | 106 | 1756 |\n| `multi_turn_long_context` | 200 | 808.41 | 106 | 1756 |\n| `multi_turn_miss_func` | 200 | 811.88 | 110 | 1760 |\n| `multi_turn_miss_param` | 200 | 863.88 | 167 | 1791 |\n| `multiple` | 200 | 120.75 | 65 | 229 |\n| `parallel` | 200 | 297.93 | 69 | 781 |\n| `parallel_multiple` | 200 | 432.17 | 88 | 1249 |\n| `simple_java` | 100 | 228.93 | 138 | 531 |\n| `simple_javascript` | 50 | 222.34 | 118 | 637 |\n| `simple_python` | 400 | 119.66 | 57 | 303 |\n| `web_search_base` | 100 | 831.01 | 748 | 963 |\n| `web_search_no_snippet` | 100 | 831.01 | 748 | 963 |\n\n## 样例示例\n\n**子集**:`irrelevance`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4e05e910\",\n \"content\": \"[[{\\\"role\\\": \\\"user\\\", \\\"content\\\": \\\"Calculate the area of a triangle given the base is 10 meters and height is 5 meters.\\\"}]]\"\n }\n ],\n \"target\": \"{}\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"id\": \"irrelevance_0\",\n \"question\": [\n [\n {\n \"role\": \"user\",\n \"content\": \"Calculate the area of a triangle given the base is 10 meters and height is 5 meters.\"\n }\n ]\n ],\n \"function\": [\n {\n \"name\": \"determine_body_mass_index\",\n \"description\": \"Calculate body mass index given weight and height.\",\n \"parameters\": {\n \"type\": \"dict\",\n \"properties\": {\n \"weight\": {\n \"type\": \"float\",\n \"description\": \"Weight of the individual in kilograms.\"\n },\n \"height\": {\n \"type\": \"float\",\n \"description\": \"Height of the individual in meters.\"\n }\n },\n \"required\": [\n \"weight\",\n \"height\"\n ]\n }\n }\n ],\n \"category\": \"irrelevance\",\n \"ground_truth\": {}\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `underscore_to_dot` | `bool` | `True` | 在评估时将函数名中的下划线转换为点号。 |\n| `is_fc_model` | `bool` | `True` | 指示被评估模型是否原生支持函数调用。 |\n| `SERPAPI_API_KEY` | `str | null` | `None` | 启用 BFCL V4 中网络搜索功能的 SerpAPI 密钥。设为 null 则禁用网络搜索。 |\n\n## 使用方法\n\n### 通过 CLI 使用\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bfcl_v4 \\\n --limit 10 # 正式评估时请移除此行\n```\n\n### 通过 Python 使用\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bfcl_v4'],\n dataset_args={\n 'bfcl_v4': {\n # subset_list: ['irrelevance', 'live_irrelevance', 'live_multiple'] # 可选,仅评估指定子集\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请移除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "88abe1d8df85cb7cb25133e1bf546407", "needs_translation": false }, - "updated_at": "2026-08-11T15:30:53.581862", - "translation_updated_at": "2026-01-28T15:56:15Z" + "updated_at": "2026-11-15T01:27:01.683894", + "translation_updated_at": "2026-11-15T01:27:13" } diff --git a/evalscope/benchmarks/bfcl/requirements.txt b/evalscope/benchmarks/bfcl/requirements.txt index 35cdb6f20..c7a21cce7 100644 --- a/evalscope/benchmarks/bfcl/requirements.txt +++ b/evalscope/benchmarks/bfcl/requirements.txt @@ -1 +1,3 @@ bfcl-eval==2025.10.27.1 +# bfcl-eval pins numpy==1.26.4; pyarrow 26 needs NumPy 2 at import time without declaring it. +pyarrow<26 diff --git a/evalscope/benchmarks/bfcl/v3/bfcl_v3_adapter.py b/evalscope/benchmarks/bfcl/v3/bfcl_v3_adapter.py index 15b9c6b69..63b28f822 100644 --- a/evalscope/benchmarks/bfcl/v3/bfcl_v3_adapter.py +++ b/evalscope/benchmarks/bfcl/v3/bfcl_v3_adapter.py @@ -71,7 +71,7 @@ ## Evaluation Notes -- Requires `pip install bfcl-eval==2025.10.27.1` before evaluation +- Requires `pip install 'evalscope[bfcl]'` before evaluation - Set `is_fc_model=True` for models with native function calling support - `underscore_to_dot` parameter controls function name formatting - See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v3.html) for detailed setup diff --git a/evalscope/benchmarks/bfcl/v4/bfcl_v4_adapter.py b/evalscope/benchmarks/bfcl/v4/bfcl_v4_adapter.py index 9b9228052..e9fc6c0bf 100644 --- a/evalscope/benchmarks/bfcl/v4/bfcl_v4_adapter.py +++ b/evalscope/benchmarks/bfcl/v4/bfcl_v4_adapter.py @@ -69,7 +69,7 @@ def _normalize_openai_base_url(api_url: Optional[str]) -> str: ## Evaluation Notes -- **Installation Required**: `pip install bfcl-eval==2025.10.27.1` +- **Installation Required**: `pip install 'evalscope[bfcl]'` - Primary metric: **Accuracy** per category - Configure `is_fc_model` based on model capabilities - Optional: Set `SERPAPI_API_KEY` for web search tasks