From 9ccf7e34b127ae3f45441b955a7937e5e2741c18 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 14:52:35 +0800 Subject: [PATCH 01/28] feat(core): let a task declare that it requires native function calling `function_tools` is already a capability key, and passing `tools` projects an intent for it, but `TaskRequirements` had no slot to demand it -- so a task whose whole premise is the tools API could only fail at its first request, after a result directory existed. Add the fourth member of the existing capability-declaration set; both validation paths pick it up unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/core/models/requirements.py | 3 +++ sieval/core/tasks/task.py | 2 ++ tests/unit/core/models/test_requirements.py | 16 ++++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/sieval/core/models/requirements.py b/sieval/core/models/requirements.py index f7f2786a..abf81c17 100644 --- a/sieval/core/models/requirements.py +++ b/sieval/core/models/requirements.py @@ -48,6 +48,7 @@ class TaskRequirements: input_scoring: bool = False sampled_logprobs: bool = False min_top_logprobs: int | None = None + function_tools: bool = False def __post_init__(self) -> None: if self.input is not None and not isinstance(self.input, InputKind): @@ -67,6 +68,8 @@ def __post_init__(self) -> None: raise TypeError("input_scoring must be a boolean") if not isinstance(self.sampled_logprobs, bool): raise TypeError("sampled_logprobs must be a boolean") + if not isinstance(self.function_tools, bool): + raise TypeError("function_tools must be a boolean") minimum = self.min_top_logprobs if minimum is not None: diff --git a/sieval/core/tasks/task.py b/sieval/core/tasks/task.py index 0121bd90..bf99e097 100644 --- a/sieval/core/tasks/task.py +++ b/sieval/core/tasks/task.py @@ -65,6 +65,8 @@ def _required_capabilities(requires: TaskRequirements) -> frozenset[str]: capabilities.add("sampled_logprobs") if requires.min_top_logprobs is not None: capabilities.add("top_logprobs") + if requires.function_tools: + capabilities.add("function_tools") return frozenset(capabilities) diff --git a/tests/unit/core/models/test_requirements.py b/tests/unit/core/models/test_requirements.py index 1c3abc8e..9c621f9c 100644 --- a/tests/unit/core/models/test_requirements.py +++ b/tests/unit/core/models/test_requirements.py @@ -101,6 +101,22 @@ def test_invalid_requirement_values_are_rejected( with pytest.raises((TypeError, ValueError), match=message): TaskRequirements(**cast(Any, kwargs)) + def test_function_tools_defaults_off_and_requires_no_capability(self) -> None: + from sieval.core.tasks.task import _required_capabilities + + assert TaskRequirements().function_tools is False + assert "function_tools" not in _required_capabilities(TaskRequirements()) + + def test_function_tools_flag_demands_the_capability(self) -> None: + from sieval.core.tasks.task import _required_capabilities + + requires = TaskRequirements(function_tools=True) + assert _required_capabilities(requires) == frozenset({"function_tools"}) + + def test_function_tools_rejects_a_non_bool(self) -> None: + with pytest.raises(TypeError, match="function_tools must be a boolean"): + TaskRequirements(function_tools=cast(Any, "yes")) + class TestNormalizedBindings: def test_all_bindings_are_frozen_and_hold_only_setup_values(self) -> None: From aae9199fb4790b0ad1bb1a3f1f38d781f8fb3315 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 17:05:37 +0800 Subject: [PATCH 02/28] feat(community): vendor the BFCL v3 AST checker from gorilla v1.3 Apache-2.0 upstream, so the grader is copied rather than reimplemented. The only edit is rewriting `bfcl_eval.*` imports to relative ones; each module's docstring records upstream's blob SHA and that deviation, and a sha256 test pins the files as committed so a later edit cannot widen it quietly. `ast_checker` reads a `MODEL_CONFIG_MAPPING` global to answer one question -- may a function name contain a dot on this transport. That is a property of the protocol, not of a model, so sieval supplies the global instead of carrying upstream's 2044-line model registry. Co-Authored-By: Claude Opus 5 --- sieval/community/bfcl_v3/__init__.py | 63 ++ sieval/community/bfcl_v3/_model_config.py | 48 ++ sieval/community/bfcl_v3/_tables.py | 43 ++ sieval/community/bfcl_v3/aggregate.py | 65 ++ sieval/community/bfcl_v3/ast_checker.py | 646 ++++++++++++++++++ sieval/community/bfcl_v3/output_checks.py | 49 ++ sieval/community/bfcl_v3/parser.py | 114 ++++ sieval/community/bfcl_v3/prompts.py | 32 + sieval/community/bfcl_v3/tool_convert.py | 223 ++++++ .../bfcl_v3/type_convertor/__init__.py | 0 .../type_convertor/java_type_converter.py | 417 +++++++++++ .../type_convertor/js_type_converter.py | 321 +++++++++ sieval/community/bfcl_v3/type_mappings.py | 100 +++ tests/unit/community/bfcl_v3/test_identity.py | 54 ++ 14 files changed, 2175 insertions(+) create mode 100644 sieval/community/bfcl_v3/__init__.py create mode 100644 sieval/community/bfcl_v3/_model_config.py create mode 100644 sieval/community/bfcl_v3/_tables.py create mode 100644 sieval/community/bfcl_v3/aggregate.py create mode 100644 sieval/community/bfcl_v3/ast_checker.py create mode 100644 sieval/community/bfcl_v3/output_checks.py create mode 100644 sieval/community/bfcl_v3/parser.py create mode 100644 sieval/community/bfcl_v3/prompts.py create mode 100644 sieval/community/bfcl_v3/tool_convert.py create mode 100644 sieval/community/bfcl_v3/type_convertor/__init__.py create mode 100644 sieval/community/bfcl_v3/type_convertor/java_type_converter.py create mode 100644 sieval/community/bfcl_v3/type_convertor/js_type_converter.py create mode 100644 sieval/community/bfcl_v3/type_mappings.py create mode 100644 tests/unit/community/bfcl_v3/test_identity.py diff --git a/sieval/community/bfcl_v3/__init__.py b/sieval/community/bfcl_v3/__init__.py new file mode 100644 index 00000000..87e55803 --- /dev/null +++ b/sieval/community/bfcl_v3/__init__.py @@ -0,0 +1,63 @@ +"""BFCL v3 (single-turn) grader, vendored from ShishirPatil/gorilla. + +Source: https://github.com/ShishirPatil/gorilla +Revision: v1.3 (ea13468e4423454d0c213704fb87cf7cb3990433) +License: Apache-2.0 + +Upstream's evaluation surface is `bfcl_eval`, an installable package that +imports its own submodules by absolute path (`bfcl_eval.constants...`, +`bfcl_eval.model_handler...`). A vendored copy cannot keep those imports, since +`bfcl_eval` itself is not installed here -- rewriting them to relative imports +within this package is the only edit made to any vendored file. Each vendored +module's docstring records upstream's blob SHA at the pin and the exact rewrite +applied, so a reviewer can re-derive the copy from a fresh checkout. + +Whole-file vendored modules: `ast_checker.py`, `type_mappings.py`, +`type_convertor/java_type_converter.py`, `type_convertor/js_type_converter.py`. + +Partial vendored modules (named subsets of a larger upstream file -- see each +module's own docstring for exactly which symbols were taken and why): +`parser.py`, `tool_convert.py`, `output_checks.py`, `aggregate.py`. `prompts.py` +is `constants/default_prompts.py` copied whole. + +First-party modules: `_model_config.py` supplies the `MODEL_CONFIG_MAPPING` +global `ast_checker.convert_func_name` reads, since upstream's own 2044-line +registry has no entries for sieval's models. `_tables.py` holds the BFCL v3 +category tables (row counts, which categories have no gold answer file, which +are non-Python) shared by the dataset loader and the tasks -- neither of which +may reach into the other's private module, so the tables live here instead. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from ._model_config import set_underscore_to_dot +from ._tables import ( + CALL_EXPECTED, + GOLDLESS_CATEGORIES, + LANGUAGE_BY_CATEGORY, + LIVE_COUNTS, + NON_LIVE_COUNTS, +) +from .aggregate import calculate_unweighted_accuracy, calculate_weighted_accuracy +from .ast_checker import ast_checker +from .output_checks import is_empty_output, is_function_calling_format_output +from .parser import ast_parse +from .prompts import DEFAULT_SYSTEM_PROMPT +from .tool_convert import convert_to_tool + +__all__ = [ + "ast_checker", + "ast_parse", + "convert_to_tool", + "is_function_calling_format_output", + "is_empty_output", + "calculate_weighted_accuracy", + "calculate_unweighted_accuracy", + "DEFAULT_SYSTEM_PROMPT", + "set_underscore_to_dot", + "NON_LIVE_COUNTS", + "LIVE_COUNTS", + "GOLDLESS_CATEGORIES", + "CALL_EXPECTED", + "LANGUAGE_BY_CATEGORY", +] diff --git a/sieval/community/bfcl_v3/_model_config.py b/sieval/community/bfcl_v3/_model_config.py new file mode 100644 index 00000000..35ba731f --- /dev/null +++ b/sieval/community/bfcl_v3/_model_config.py @@ -0,0 +1,48 @@ +"""The `MODEL_CONFIG_MAPPING` global upstream's `ast_checker` expects. + +`convert_func_name` reads `MODEL_CONFIG_MAPPING[model].underscore_to_dot` to +decide whether a gold function name's dots must be rewritten to underscores +before comparison. Upstream carries a 2044-line registry of its own evaluated +models to answer that one question. sieval's models are not in it, so a verbatim +copy would `KeyError`. + +Rather than delete the call (removing a helper is as much a deviation as adding +one), supply the global the file expects. The flag is a property of the +*transport*, not of a model: it is True exactly when the function schemas +travelled through an API whose tool names cannot contain `.`. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class _ModelEntry: + underscore_to_dot: bool + + +class _ModelConfigMapping: + """Answers for any model name, because the flag does not depend on one.""" + + def __init__(self) -> None: + self._underscore_to_dot = False + + def set_underscore_to_dot(self, enabled: bool) -> None: + self._underscore_to_dot = enabled + + def __getitem__(self, _model_name: str) -> _ModelEntry: + return _ModelEntry(underscore_to_dot=self._underscore_to_dot) + + +MODEL_CONFIG_MAPPING = _ModelConfigMapping() + + +def set_underscore_to_dot(enabled: bool) -> None: + """Set the flag for the protocol about to be graded. + + Prompt protocol → False: schemas reach the model verbatim, dots survive. + FC protocol → True: `convert_to_tool` rewrote `.`→`_` on the way out, so + gold must be rewritten to match. + """ + MODEL_CONFIG_MAPPING.set_underscore_to_dot(enabled) diff --git a/sieval/community/bfcl_v3/_tables.py b/sieval/community/bfcl_v3/_tables.py new file mode 100644 index 00000000..db7f7be8 --- /dev/null +++ b/sieval/community/bfcl_v3/_tables.py @@ -0,0 +1,43 @@ +"""BFCL v3 category tables, shared by the dataset loader and the tasks. + +Held here rather than in either caller because both need them and neither may +import the other's private module. Keeping them out of the vendored files is +also what lets those stay byte-identical. + +Row counts are ours, measured at the pinned revision; the category membership is +upstream's `TEST_COLLECTION_MAPPING` at v1.3. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +NON_LIVE_COUNTS: dict[str, int] = { + "simple": 400, + "multiple": 200, + "parallel": 200, + "parallel_multiple": 200, + "java": 100, + "javascript": 50, + "irrelevance": 240, +} + +LIVE_COUNTS: dict[str, int] = { + "live_simple": 258, + "live_multiple": 1053, + "live_parallel": 16, + "live_parallel_multiple": 24, + "live_irrelevance": 882, + "live_relevance": 18, +} + +#: Upstream ships no `possible_answer/` file for these: the correct outcome is +#: about whether a call was produced, not about which call it was. +GOLDLESS_CATEGORIES = frozenset( + {"irrelevance", "live_irrelevance", "live_relevance"} +) + +#: Of the goldless three, the one where producing a call is CORRECT. +CALL_EXPECTED = frozenset({"live_relevance"}) + +#: Only these two are not Python. `language` selects the type converter inside +#: `ast_checker` -- it is NOT the parser's language (see `_base.py`). +LANGUAGE_BY_CATEGORY = {"java": "Java", "javascript": "JavaScript"} diff --git a/sieval/community/bfcl_v3/aggregate.py b/sieval/community/bfcl_v3/aggregate.py new file mode 100644 index 00000000..a0a2347c --- /dev/null +++ b/sieval/community/bfcl_v3/aggregate.py @@ -0,0 +1,65 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/eval_checker/eval_runner_helper.py +Upstream blob: 1de6462be8be9cc91d5a0e95c69e2c6fd6fc1a0d +License: Apache-2.0 + +Symbols taken (bodies unchanged): `calculate_weighted_accuracy`, +`calculate_unweighted_accuracy`. This is a partial extraction, not a +whole-file copy -- the rest of `eval_runner_helper.py` (leaderboard CSV +generation, cost/latency aggregation, pandas/numpy-based reporting) is out of +scope for this port. + +Only deviation: neither function needs any import, so there was nothing to +rewrite. +""" + + +def calculate_weighted_accuracy(accuracy_dict_list, display_na_if_category_missing=True): + has_na = False + total_count = 0 + total_accuracy = 0 + for accuracy_dict in accuracy_dict_list: + accuracy = accuracy_dict["accuracy"] + count = accuracy_dict["total_count"] + if accuracy_dict["display_accuracy"] == "N/A": + has_na = True + + total_count += count + total_accuracy += accuracy * count + + result = {"accuracy": total_accuracy / total_count, "total_count": total_count} + + if has_na and display_na_if_category_missing: + result["display_accuracy"] = "N/A" + else: + result["display_accuracy"] = result["accuracy"] + + return result + + +def calculate_unweighted_accuracy(accuracy_dict_list, display_na_if_category_missing=True): + has_na = False + total_count = 0 + total_accuracy = 0 + for accuracy_dict in accuracy_dict_list: + accuracy = accuracy_dict["accuracy"] + count = accuracy_dict["total_count"] + if accuracy_dict["display_accuracy"] == "N/A": + # If a category is not being evaluated, it will still be considered 0 in the overall score calculation. + has_na = True + + total_count += count + total_accuracy += accuracy + + result = { + "accuracy": total_accuracy / len(accuracy_dict_list), + "total_count": total_count, + } + + if has_na and display_na_if_category_missing: + result["display_accuracy"] = "N/A" + else: + result["display_accuracy"] = result["accuracy"] + + return result diff --git a/sieval/community/bfcl_v3/ast_checker.py b/sieval/community/bfcl_v3/ast_checker.py new file mode 100644 index 00000000..0782092e --- /dev/null +++ b/sieval/community/bfcl_v3/ast_checker.py @@ -0,0 +1,646 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/eval_checker/ast_eval/ast_checker.py +Upstream blob: 214f3ba55c012691761913b6aca68ab7bc4aa38b +License: Apache-2.0 + +Only deviation: absolute `bfcl_eval.*` imports rewritten to relative ones +(`.type_mappings`, `.type_convertor.*`, `._model_config`). No other edit. +""" + +from ._model_config import MODEL_CONFIG_MAPPING +from .type_mappings import ( + JAVA_TYPE_CONVERSION, + JS_TYPE_CONVERSION, +) +from .type_convertor.java_type_converter import java_type_converter +from .type_convertor.js_type_converter import js_type_converter +import re + +#### Constants #### +PYTHON_TYPE_MAPPING = { + "string": str, + "integer": int, + "float": float, + "boolean": bool, + "array": list, + "tuple": list, + "dict": dict, + "any": str, +} + +# This is the list of types that we need to recursively check its values +PYTHON_NESTED_TYPE_CHECK_LIST = ["array", "tuple"] + + +NESTED_CONVERSION_TYPE_LIST = ["Array", "ArrayList", "array"] + + +#### Main function #### +def ast_checker( + func_description, model_output, possible_answer, language, test_category, model_name +): + if "parallel" in test_category: + return parallel_function_checker_no_order( + func_description, model_output, possible_answer, language, model_name + ) + + elif "multiple" in test_category: + return multiple_function_checker( + func_description, model_output, possible_answer, language, model_name + ) + + else: + if len(model_output) != 1: + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "simple_function_checker:wrong_count", + } + + return simple_function_checker( + func_description[0], model_output[0], possible_answer[0], language, model_name + ) + + +#### Helper functions for AST #### +def find_description(func_descriptions, name): + if type(func_descriptions) == list: + for func_description in func_descriptions: + if func_description["name"] == name: + return func_description + return None + else: + # it is a dict, there is only one function + return func_descriptions + + +def get_possible_answer_type(possible_answer: list): + for answer in possible_answer: + if answer != "": # Optional parameter + return type(answer) + return None + + +def convert_func_name(function_name, model_name: str): + model_name_escaped = model_name.replace("_", "/") + if "." in function_name: + if MODEL_CONFIG_MAPPING[model_name_escaped].underscore_to_dot: + # OAI does not support "." in the function name so we replace it with "_". ^[a-zA-Z0-9_-]{1,64}$ is the regex for the name. + # This happens for OpenAI, Mistral, and Google models + return re.sub(r"\.", "_", function_name) + return function_name + + +def type_checker( + param: str, + value, + possible_answer: list, + expected_type_description: str, + expected_type_converted, + nested_type_converted, +): + # NOTE: This type checker only supports nested type checking for one level deep. + # We didn't implement recursive type checking for nested types, as it's not needed for the current use case and it's very complex. + + result = { + "valid": True, + "error": [], + "is_variable": False, + "error_type": "type_error:simple", + } + + is_variable = False + # check for the case where a variable is used instead of a actual value. + # use the type in possible_answer as the expected type + possible_answer_type = get_possible_answer_type(possible_answer) + # if possible_answer only contains optional parameters, we can't determine the type + if possible_answer_type != None: + # we are being precise here. + # in fact, possible_answer_type should always be string, as that's how we treat varibale in possible_answer + if possible_answer_type != expected_type_converted: + is_variable = True + + # value is the same type as in function description + if type(value) == expected_type_converted: + # We don't need to do recursive check for simple types + if nested_type_converted == None: + result["is_variable"] = is_variable + return result + else: + for possible_answer_item in possible_answer: + flag = True # Each parameter should match to at least one possible answer type. + # Here, we assume that each item should be the same type. We could also relax it. + if type(possible_answer_item) == list: + for value_item in value: + checker_result = type_checker( + param, + value_item, + possible_answer_item, + str(nested_type_converted), + nested_type_converted, + None, + ) + if not checker_result["valid"]: + flag = False + break + + if flag: + return {"valid": True, "error": [], "is_variable": is_variable} + + result["valid"] = False + result["error"] = [ + f"Nested type checking failed for parameter {repr(param)}. Expected outer type {expected_type_description} with inner type {str(nested_type_converted)}. Parameter value: {repr(value)}." + ] + result["error_type"] = "type_error:nested" + + # value is not as expected, check for the case where a variable is used instead of a actual value + # use the type in possible_answer as the expected type + possible_answer_type = get_possible_answer_type(possible_answer) + # if possible_answer only contains optional parameters, we can't determine the type + if possible_answer_type != None: + # we are being precise here. + # in fact, possible_answer_type should always be string, as that's how we treat varibale in possible_answer + if type(value) == possible_answer_type: + result["is_variable"] = True + return result + + result["valid"] = False + result["error"].append( + f"Incorrect type for parameter {repr(param)}. Expected type {expected_type_description}, got {type(value).__name__}. Parameter value: {repr(value)}." + ) + result["error_type"] = "type_error:simple" + return result + + +def standardize_string(input_string: str): + # This function standardizes the string by removing all the spaces, ",./-_*^" punctuation, and converting it to lowercase + # It will also convert all the single quotes to double quotes + # This is used to compare the model output with the possible answers + # We don't want to punish model for answer like April 1, 2024 vs April 1,2024, vs April 1 2024 + regex_string = r"[ \,\.\/\-\_\*\^]" + return re.sub(regex_string, "", input_string).lower().replace("'", '"') + + +def string_checker(param: str, model_output: str, possible_answer: list): + standardize_possible_answer = [] + standardize_model_output = standardize_string(model_output) + for i in range(len(possible_answer)): + if type(possible_answer[i]) == str: + standardize_possible_answer.append(standardize_string(possible_answer[i])) + + if standardize_model_output not in standardize_possible_answer: + return { + "valid": False, + "error": [ + f"Invalid value for parameter {repr(param)}: {repr(model_output)}. Expected one of {possible_answer}. Case insensitive." + ], + "error_type": "value_error:string", + } + + return {"valid": True, "error": []} + + +def list_checker(param: str, model_output: list, possible_answer: list): + # Convert the tuple to a list + + standardize_model_output = list(model_output) + + # If the element in the list is a string, we need to standardize it + for i in range(len(standardize_model_output)): + if type(standardize_model_output[i]) == str: + standardize_model_output[i] = standardize_string(model_output[i]) + + standardize_possible_answer = [] + # We also need to standardize the possible answers + for i in range(len(possible_answer)): + standardize_possible_answer.append([]) + for j in range(len(possible_answer[i])): + if type(possible_answer[i][j]) == str: + standardize_possible_answer[i].append( + standardize_string(possible_answer[i][j]) + ) + else: + standardize_possible_answer[i].append(possible_answer[i][j]) + + if standardize_model_output not in standardize_possible_answer: + return { + "valid": False, + "error": [ + f"Invalid value for parameter {repr(param)}: {repr(model_output)}. Expected one of {possible_answer}." + ], + "error_type": "value_error:list/tuple", + } + + return {"valid": True, "error": []} + + +def dict_checker(param: str, model_output: dict, possible_answers: list): + # This function works for simple dictionaries, but not dictionaries with nested dictionaries. + # The current dataset only contains simple dictionaries, so this is sufficient. + + result = {"valid": False, "error": [], "error_type": "dict_checker:unclear"} + for i in range(len(possible_answers)): + + if possible_answers[i] == "": + continue + + result = {"valid": False, "error": [], "error_type": "dict_checker:unclear"} + + flag = True + + possible_answer = possible_answers[i] + # possible_anwer is a single dictionary + + for key, value in model_output.items(): + if key not in possible_answer: + result["valid"] = False + result["error"].append(f"Unexpected dict key parameter: '{key}'.") + result["error_type"] = "value_error:dict_key" + flag = False + break + + standardize_value = value + # If the value is a string, we need to standardize it + if type(value) == str: + standardize_value = standardize_string(value) + + # We also need to standardize the possible answers if they are string + standardize_possible_answer = [] + for i in range(len(possible_answer[key])): + if type(possible_answer[key][i]) == str: + standardize_possible_answer.append( + standardize_string(possible_answer[key][i]) + ) + else: + standardize_possible_answer.append(possible_answer[key][i]) + + if standardize_value not in standardize_possible_answer: + result["valid"] = False + result["error"].append( + f"Invalid value for parameter {repr(key)}: {repr(value)}. Expected one of {standardize_possible_answer}." + ) + result["error_type"] = "value_error:dict_value" + flag = False + break + + for key, value in possible_answer.items(): + if key not in model_output and "" not in value: + result["valid"] = False + result["error"].append(f"Missing dict key parameter: '{key}'.") + result["error_type"] = "value_error:dict_key" + flag = False + break + + if flag: + return {"valid": True, "error": []} + + return result + + +def list_dict_checker(param: str, model_output: list, possible_answers: list): + # This function takes in a list of dictionaries and checks if each dictionary is valid + # The order of the dictionaries in the list must match the order of the possible answers + + result = {"valid": False, "error": [], "error_type": "list_dict_checker:unclear"} + + for answer_index in range(len(possible_answers)): + flag = True # True means so far, all dictionaries are valid + + # Only proceed if the number of dictionaries in the list matches the number of dictionaries in the possible answers + if len(model_output) != len(possible_answers[answer_index]): + result["valid"] = False + result["error"] = ["Wrong number of dictionaries in the list."] + result["error_type"] = "value_error:list_dict_count" + flag = False + continue + + for dict_index in range(len(model_output)): + result = dict_checker( + param, + model_output[dict_index], + [possible_answers[answer_index][dict_index]], + ) + if not result["valid"]: + flag = False + break + if flag: + return {"valid": True, "error": []} + + return result + + +def simple_function_checker( + func_description: dict, + model_output: dict, + possible_answer: dict, + language: str, + model_name: str, +): + possible_answer = list(possible_answer.values())[0] + # Extract function name and parameters details + func_name = func_description["name"] + param_details = func_description["parameters"]["properties"] + required_params = func_description["parameters"]["required"] + + # Initialize a result dictionary + result = { + "valid": True, + "error": [], + "error_type": "simple_function_checker:unclear", + } + + func_name = convert_func_name(func_name, model_name) + + # Check if function name matches + if func_name not in model_output: + result["valid"] = False + result["error"].append( + f"Function name {repr(func_name)} not found in model output." + ) + result["error_type"] = "simple_function_checker:wrong_func_name" + return result + + model_params = model_output[func_name] + + # Check for required parameters in model output + for param in required_params: + if param not in model_params: + result["valid"] = False + result["error"].append(f"Missing required parameter: {repr(param)}.") + result["error_type"] = "simple_function_checker:missing_required" + return result + + # Validate types and values for each parameter in model output + for param, value in model_params.items(): + if param not in param_details or param not in possible_answer: + result["valid"] = False + result["error"].append(f"Unexpected parameter: {repr(param)}.") + result["error_type"] = "simple_function_checker:unexpected_param" + return result + + full_param_details = param_details[param] + expected_type_description = full_param_details["type"] # This is a string + is_variable = False + nested_type_converted = None + + if language == "Java": + expected_type_converted = JAVA_TYPE_CONVERSION[expected_type_description] + + if expected_type_description in JAVA_TYPE_CONVERSION: + if type(value) != str: + result["valid"] = False + result["error"].append( + f"Incorrect type for parameter {repr(param)}. Expected type String, got {type(value).__name__}. Parameter value: {repr(value)}." + ) + result["error_type"] = "type_error:java" + return result + + if expected_type_description in NESTED_CONVERSION_TYPE_LIST: + nested_type = param_details[param]["items"]["type"] + nested_type_converted = JAVA_TYPE_CONVERSION[nested_type] + value = java_type_converter( + value, expected_type_description, nested_type + ) + else: + value = java_type_converter(value, expected_type_description) + + elif language == "JavaScript": + expected_type_converted = JS_TYPE_CONVERSION[expected_type_description] + + if expected_type_description in JS_TYPE_CONVERSION: + if type(value) != str: + result["valid"] = False + result["error"].append( + f"Incorrect type for parameter {repr(param)}. Expected type String, got {type(value).__name__}. Parameter value: {repr(value)}." + ) + result["error_type"] = "type_error:js" + return result + + if expected_type_description in NESTED_CONVERSION_TYPE_LIST: + nested_type = param_details[param]["items"]["type"] + nested_type_converted = JS_TYPE_CONVERSION[nested_type] + value = js_type_converter( + value, expected_type_description, nested_type + ) + else: + value = js_type_converter(value, expected_type_description) + + elif language == "Python": + expected_type_converted = PYTHON_TYPE_MAPPING[expected_type_description] + if expected_type_description in PYTHON_NESTED_TYPE_CHECK_LIST: + nested_type = param_details[param]["items"]["type"] + nested_type_converted = PYTHON_TYPE_MAPPING[nested_type] + + # We convert all tuple value to list when the expected type is tuple. + # The conversion is necessary because any tuple in the possible answer would become a list after being processed through json.dump() and json.load(). + # This does introduce some false positive (eg, when the model provides a list value instead of tuple). We hope to find a better solution in the future. + if expected_type_description == "tuple" and type(value) == tuple: + value = list(value) + + # Allow python auto conversion from int to float + if ( + language == "Python" + and expected_type_description == "float" + and type(value) == int + ): + value = float(value) + + # Type checking + # In fact, we only check for Python here. + # Type check for other languages are handled by the type converter, and so their value (after conversion) is always correct. + type_check_result = type_checker( + param, + value, + possible_answer[param], + expected_type_description, + expected_type_converted, + nested_type_converted, + ) + is_variable = type_check_result["is_variable"] + if not type_check_result["valid"]: + return type_check_result + + # It doesn't make sense to special handle dictionaries and list of dictionaries if the value is a variable. + # We can just treat the variable as a string and use the normal flow. + if not is_variable: + # Special handle for dictionaries + if expected_type_converted == dict: + result = dict_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + # Special handle for list of dictionaries + elif expected_type_converted == list and nested_type_converted == dict: + result = list_dict_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + # Special handle for strings + elif expected_type_converted == str: + # We don't check for case sensitivity for string, as long as it's not a variable + result = string_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + elif expected_type_converted == list: + result = list_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + # Check if the value is within the possible answers + if value not in possible_answer[param]: + result["valid"] = False + result["error"].append( + f"Invalid value for parameter {repr(param)}: {repr(value)}. Expected one of {possible_answer[param]}." + ) + result["error_type"] = "value_error:others" + return result + + # Check for optional parameters not provided but allowed + for param in possible_answer: + if param not in model_params and "" not in possible_answer[param]: + result["valid"] = False + result["error"].append( + f"Optional parameter {repr(param)} not provided and not marked as optional." + ) + result["error_type"] = "simple_function_checker:missing_optional" + return result + + return result + + +def parallel_function_checker_enforce_order( + func_descriptions: list, + model_output: list, + possible_answers: dict, + language: str, + model_name: str, +): + if len(model_output) != len(possible_answers): + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "parallel_function_checker_enforce_order:wrong_count", + } + + func_name_list = list(possible_answers.keys()) + possible_answers_list = [] + + for key, value in possible_answers.items(): + possible_answers_list.append({key: value}) + + for i in range(len(possible_answers_list)): + func_description = find_description(func_descriptions, func_name_list[i]) + + result = simple_function_checker( + func_description, + model_output[i], + possible_answers_list[i], + language, + model_name, + ) + if not result["valid"]: + return result + + return {"valid": True, "error": []} + + +def parallel_function_checker_no_order( + func_descriptions: list, + model_output: list, + possible_answers: list, + language: str, + model_name: str, +): + if len(model_output) != len(possible_answers): + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "parallel_function_checker_no_order:wrong_count", + } + + matched_indices = [] + + # We go throught the possible answers one by one, and eliminate the model output that matches the possible answer + # It must be this way because we need ground truth to fetch the correct function description + for i in range(len(possible_answers)): + # possible_answers[i] is a dictionary with only one key + func_name_expected = list(possible_answers[i].keys())[0] + func_description = find_description(func_descriptions, func_name_expected) + + + all_errors = [] + + for index in range(len(model_output)): + if index in matched_indices: + continue + + result = simple_function_checker( + func_description, + model_output[index], + possible_answers[i], + language, + model_name, + ) + + if result["valid"]: + matched_indices.append(index) + break + else: + all_errors.append( + { + f"Model Result Index {index}": { + "sub_error": result["error"], + "sub_error_type": result["error_type"], + "model_output_item": model_output[index], + "possible_answer_item": possible_answers[i], + } + } + ) + + if not result["valid"]: + considered_indices = [ + i for i in range(len(model_output)) if i not in matched_indices + ] + all_errors.insert( + 0, + f"Could not find a matching function among index {considered_indices} of model output for index {i} of possible answers.", + ) + return { + "valid": False, + "error": all_errors, + "error_type": "parallel_function_checker_no_order:cannot_find_match", + } + + return {"valid": True, "error": []} + + +def multiple_function_checker( + func_descriptions: list, + model_output: list, + possible_answers: list, + language: str, + model_name: str, +): + if len(model_output) != len(possible_answers): + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "multiple_function_checker:wrong_count", + } + + # possible_answers is a list of only one dictionary with only one key + func_name_expected = list(possible_answers[0].keys())[0] + func_description = find_description(func_descriptions, func_name_expected) + return simple_function_checker( + func_description, + model_output[0], + possible_answers[0], + language, + model_name, + ) diff --git a/sieval/community/bfcl_v3/output_checks.py b/sieval/community/bfcl_v3/output_checks.py new file mode 100644 index 00000000..92bf012e --- /dev/null +++ b/sieval/community/bfcl_v3/output_checks.py @@ -0,0 +1,49 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/utils.py +Upstream blob: c28472cd8b24e250d3412c5edf3b7b49c748f915 +License: Apache-2.0 + +Symbols taken (bodies unchanged): `is_function_calling_format_output`, +`is_empty_output`. This is a partial extraction, not a whole-file copy -- the +rest of `bfcl_eval/utils.py` (test-category helpers, file I/O, CLI argument +parsing) is out of scope for this port. + +Only deviation: neither function needs any import, so there was nothing to +rewrite. +""" + + +def is_function_calling_format_output(decoded_output): + """ + Ensure the output is a list of dictionaries of the form: + `[{func1: {param1: val1, param2: val2, ...}}, {func2: {param1: val1, param2: val2, ...}}, ...]` + Sometimes the model handler's `decode_ast` method will return successfully, but the output is not in the correct format, and that will mess up the downstream evaluation that expects this format. + This is especially the case when the model doesn't predict any function calls, and the output is an human-readable string. + Note: Empty list `[]` is considered the correct format in this check. + """ + if type(decoded_output) != list: + return False + for item in decoded_output: + if type(item) != dict: + return False + # Check for `{func1: {param1: val1, param2: val2, ...}}`, should only have one key-value pair + if len(item) != 1: + return False + # Check for `{param1: val1, param2: val2, ...}`; the parameter-value pairs should be a dictionary + if type(list(item.values())[0]) != dict: + return False + return True + + +def is_empty_output(decoded_output): + # This function is a patch to the ast decoder for relevance detection + # Sometimes the ast decoder will parse successfully, but the input doens't really have a function call + # [], [{}], and anything that is not in function calling format is considered empty (and thus should be marked as correct) + if not is_function_calling_format_output(decoded_output): + return True + if len(decoded_output) == 0: + return True + if len(decoded_output) == 1 and len(decoded_output[0]) == 0: + return True + return False diff --git a/sieval/community/bfcl_v3/parser.py b/sieval/community/bfcl_v3/parser.py new file mode 100644 index 00000000..6d392b80 --- /dev/null +++ b/sieval/community/bfcl_v3/parser.py @@ -0,0 +1,114 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/model_handler/utils.py +Upstream blob: f78acc240bee3c43b18de3d4195fbf6ee1a207f1 +License: Apache-2.0 + +Symbols taken (bodies unchanged): `ast_parse`, `resolve_ast_call`, +`resolve_ast_by_type`. This is a partial extraction, not a whole-file copy -- +the rest of `model_handler/utils.py` (prompt construction, decoders for other +model styles, retry helpers) is out of scope for this port. + +Only deviation: the module needs no `bfcl_eval.*` imports at all for these +three functions (`import ast` is stdlib), so there was nothing to rewrite. + +Note on the Java/JavaScript branches of `ast_parse`: upstream dispatches to +`bfcl_eval.model_handler.parser.java_parser.parse_java_function_call` and the +`js_parser` twin, neither of which is vendored here. Their bodies are kept +verbatim (unreachable rather than deleted, per the "removing a helper is a +deviation too" rule), because sieval never calls `ast_parse` with +`language="Java"` or `"JavaScript"` -- the category's language only ever +selects the type converter inside `ast_checker`, not the parser's language +(see `_tables.py`'s `LANGUAGE_BY_CATEGORY` docstring). Calling `ast_parse` +with either of those two language values would raise `NameError` on the +undefined parser call, exactly as it would if the names were simply typos; +that failure mode is intentional rather than silent, since it is unreachable +from the intended call path. +""" + +import ast + + +def ast_parse(input_str: str, language: str="Python") -> list[dict]: + if language == "Python": + cleaned_input = input_str.strip("[]'") + parsed = ast.parse(cleaned_input, mode="eval") + extracted = [] + if isinstance(parsed.body, ast.Call): + extracted.append(resolve_ast_call(parsed.body)) + else: + for elem in parsed.body.elts: + assert isinstance(elem, ast.Call) + extracted.append(resolve_ast_call(elem)) + return extracted + elif language == "Java": + return parse_java_function_call( + input_str[1:-1] + ) # Remove the [ and ] from the string + elif language == "JavaScript": + return parse_javascript_function_call(input_str[1:-1]) + else: + raise NotImplementedError(f"Unsupported language: {language}") + + +def resolve_ast_call(elem): + # Handle nested attributes for deeply nested module paths + func_parts = [] + func_part = elem.func + while isinstance(func_part, ast.Attribute): + func_parts.append(func_part.attr) + func_part = func_part.value + if isinstance(func_part, ast.Name): + func_parts.append(func_part.id) + func_name = ".".join(reversed(func_parts)) + args_dict = {} + for arg in elem.keywords: + output = resolve_ast_by_type(arg.value) + args_dict[arg.arg] = output + return {func_name: args_dict} + + +def resolve_ast_by_type(value): + if isinstance(value, ast.Constant): + if value.value is Ellipsis: + output = "..." + else: + output = value.value + elif isinstance(value, ast.UnaryOp): + output = -value.operand.value + elif isinstance(value, ast.List): + output = [resolve_ast_by_type(v) for v in value.elts] + elif isinstance(value, ast.Dict): + output = { + resolve_ast_by_type(k): resolve_ast_by_type(v) + for k, v in zip(value.keys, value.values) + } + elif isinstance( + value, ast.NameConstant + ): # Added this condition to handle boolean values + output = value.value + elif isinstance( + value, ast.BinOp + ): # Added this condition to handle function calls as arguments + output = eval(ast.unparse(value)) + elif isinstance(value, ast.Name): + output = value.id + elif isinstance(value, ast.Call): + if len(value.keywords) == 0: + output = ast.unparse(value) + else: + output = resolve_ast_call(value) + elif isinstance(value, ast.Tuple): + output = tuple(resolve_ast_by_type(v) for v in value.elts) + elif isinstance(value, ast.Lambda): + output = eval(ast.unparse(value.body[0].value)) + elif isinstance(value, ast.Ellipsis): + output = "..." + elif isinstance(value, ast.Subscript): + try: + output = ast.unparse(value.body[0].value) + except: + output = ast.unparse(value.value) + "[" + ast.unparse(value.slice) + "]" + else: + raise Exception(f"Unsupported AST type: {type(value)}") + return output diff --git a/sieval/community/bfcl_v3/prompts.py b/sieval/community/bfcl_v3/prompts.py new file mode 100644 index 00000000..6053cb23 --- /dev/null +++ b/sieval/community/bfcl_v3/prompts.py @@ -0,0 +1,32 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/constants/default_prompts.py +Upstream blob: 37eb080480daff689748601c2c77179578c187e3 +License: Apache-2.0 + +Whole file, copied entire. This file has no imports, so no rewrite was +needed here. +""" + +MAXIMUM_STEP_LIMIT = 20 + +DEFAULT_SYSTEM_PROMPT_WITHOUT_FUNC_DOC = """You are an expert in composing functions. You are given a question and a set of possible functions. Based on the question, you will need to make one or more function/tool calls to achieve the purpose. +If none of the functions can be used, point it out. If the given question lacks the parameters required by the function, also point it out. +You should only return the function calls in your response. + +If you decide to invoke any of the function(s), you MUST put it in the format of [func_name1(params_name1=params_value1, params_name2=params_value2...), func_name2(params)] +You SHOULD NOT include any other text in the response. + +At each turn, you should try your best to complete the tasks requested by the user within the current turn. Continue to output functions to call until you have fulfilled the user's request to the best of your ability. Once you have no more functions to call, the system will consider the current turn complete and proceed to the next turn or task. +""" + +DEFAULT_SYSTEM_PROMPT = ( + DEFAULT_SYSTEM_PROMPT_WITHOUT_FUNC_DOC + + """ +Here is a list of functions in JSON format that you can invoke.\n{functions}\n +""" +) + +DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC = "I have updated some more functions you can choose from. What about now?" + +DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_PROMPTING = "{functions}\n" + DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC diff --git a/sieval/community/bfcl_v3/tool_convert.py b/sieval/community/bfcl_v3/tool_convert.py new file mode 100644 index 00000000..b8470cd7 --- /dev/null +++ b/sieval/community/bfcl_v3/tool_convert.py @@ -0,0 +1,223 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/model_handler/utils.py +Upstream blob: f78acc240bee3c43b18de3d4195fbf6ee1a207f1 +License: Apache-2.0 + +Symbols taken (bodies unchanged): `convert_to_tool`, `_cast_to_openai_type`. +This is a partial extraction, not a whole-file copy -- the rest of +`model_handler/utils.py` is out of scope for this port. + +Deviations, beyond the usual import rewrite: + +- `_cast_to_openai_type` compares its `mapping` argument against the module + global `GORILLA_TO_OPENAPI`, so that name is imported from the already-vendored + `.type_mappings` -- the same relative-import rewrite Step 3's whole-file + modules get, just landing in a partial module instead. +- `convert_to_tool` branches on upstream's `ModelStyle` enum + (`bfcl_eval.model_handler.model_style.ModelStyle`). Rather than vendor that + whole file -- most of whose members name providers sieval does not model -- + only the members these two functions actually reference are copied below as + a plain enum: `OpenAI_Completions`, `OpenAI_Responses`, `Anthropic`, + `Mistral`, `GOOGLE`, `AMAZON`, `FIREWORK_AI`, `OSSMODEL`, `COHERE`, `WRITER`, + `NOVITA_AI`. Omitted: `Gorilla`, `NEXUS` (present upstream, never read by + either copied function). +""" + +import copy +import json +import re +from enum import Enum + +from .type_mappings import GORILLA_TO_OPENAPI + + +class ModelStyle(Enum): + OpenAI_Completions = "openai-completions" + OpenAI_Responses = "openai-responses" + Anthropic = "claude" + Mistral = "mistral" + GOOGLE = "google" + AMAZON = "amazon" + FIREWORK_AI = "firework_ai" + OSSMODEL = "ossmodel" + COHERE = "cohere" + WRITER = "writer" + NOVITA_AI = "novita_ai" + + +def _cast_to_openai_type(properties, mapping): + for key, value in properties.items(): + if "type" not in value: + properties[key]["type"] = "string" + else: + var_type = value["type"] + if mapping == GORILLA_TO_OPENAPI and var_type == "float": + properties[key]["format"] = "float" + properties[key]["description"] += " This is a float type value." + if var_type in mapping: + properties[key]["type"] = mapping[var_type] + else: + properties[key]["type"] = "string" + + # Currently support: + # - list of any + # - list of list of any + # - list of dict + # - list of list of dict + # - dict of any + + if properties[key]["type"] == "array" or properties[key]["type"] == "object": + if "properties" in properties[key]: + properties[key]["properties"] = _cast_to_openai_type( + properties[key]["properties"], mapping + ) + elif "items" in properties[key]: + properties[key]["items"]["type"] = mapping[properties[key]["items"]["type"]] + if ( + properties[key]["items"]["type"] == "array" + and "items" in properties[key]["items"] + ): + properties[key]["items"]["items"]["type"] = mapping[ + properties[key]["items"]["items"]["type"] + ] + elif ( + properties[key]["items"]["type"] == "object" + and "properties" in properties[key]["items"] + ): + properties[key]["items"]["properties"] = _cast_to_openai_type( + properties[key]["items"]["properties"], mapping + ) + return properties + + +def convert_to_tool(functions, mapping, model_style): + functions = copy.deepcopy(functions) + oai_tool = [] + for item in functions: + if "." in item["name"] and model_style in [ + ModelStyle.OpenAI_Completions, + ModelStyle.OpenAI_Responses, + ModelStyle.Mistral, + ModelStyle.GOOGLE, + ModelStyle.OSSMODEL, + ModelStyle.Anthropic, + ModelStyle.COHERE, + ModelStyle.AMAZON, + ModelStyle.NOVITA_AI, + ]: + # OAI does not support "." in the function name so we replace it with "_". ^[a-zA-Z0-9_-]{1,64}$ is the regex for the name. + item["name"] = re.sub(r"\.", "_", item["name"]) + + item["parameters"]["type"] = "object" + item["parameters"]["properties"] = _cast_to_openai_type( + item["parameters"]["properties"], mapping + ) + + if model_style == ModelStyle.Anthropic: + item["input_schema"] = item["parameters"] + del item["parameters"] + + if model_style == ModelStyle.AMAZON: + item["inputSchema"] = {"json": item["parameters"]} + del item["parameters"] + + if model_style in [ + ModelStyle.GOOGLE, + ModelStyle.WRITER, + ]: + # Remove fields that are not supported by Gemini or Palmyra. + # No `optional` field in function schema. + if "optional" in item["parameters"]: + del item["parameters"]["optional"] + for params in item["parameters"]["properties"].values(): + if "description" not in params: + params["description"] = "" + # No `default` field in GOOGLE or Palmyra's schema. + if "default" in params: + params["description"] += f" Default is: {str(params['default'])}." + del params["default"] + # No `optional` field in parameter schema as well. + if "optional" in params: + params["description"] += f" Optional: {str(params['optional'])}." + del params["optional"] + # No `required` field in parameter schema as well. + if "required" in params: + params["description"] += f" Required: {str(params['required'])}." + del params["required"] + # No `maximum` field. + if "maximum" in params: + params["description"] += f" Maximum value: {str(params['maximum'])}." + del params["maximum"] + # No `minItems` field. + if "minItems" in params: + params[ + "description" + ] += f" Minimum number of items: {str(params['minItems'])}." + del params["minItems"] + # No `maxItems` field. + if "maxItems" in params: + params[ + "description" + ] += f" Maximum number of items: {str(params['maxItems'])}." + del params["maxItems"] + # No `additionalProperties` field. + if "additionalProperties" in params: + params[ + "description" + ] += f" Additional properties: {str(params['additionalProperties'])}." + del params["additionalProperties"] + # For Gemini, only `enum` field when the type is `string`. + # For Palmyra, `enum` field is not supported. + if "enum" in params and ( + model_style == ModelStyle.WRITER + or (model_style == ModelStyle.GOOGLE and params["type"] != "string") + ): + params["description"] += f" Enum values: {str(params['enum'])}." + del params["enum"] + # No `format` when type is `string` + if "format" in params and params["type"] == "string": + params["description"] += f" Format: {str(params['format'])}." + del params["format"] + + # Process the return field + if "response" in item: + if model_style in [ + ModelStyle.Anthropic, + ModelStyle.GOOGLE, + ModelStyle.FIREWORK_AI, + ModelStyle.WRITER, + ModelStyle.AMAZON, + ModelStyle.NOVITA_AI, + ]: + item[ + "description" + ] += f" The response field has the following schema: {json.dumps(item['response'])}" + del item["response"] + + if model_style in [ + ModelStyle.Anthropic, + ModelStyle.GOOGLE, + ModelStyle.OSSMODEL, + ]: + oai_tool.append(item) + elif model_style in [ + ModelStyle.OpenAI_Responses + ]: + oai_tool.append({"type": "function", + "name": item["name"], + "description": item["description"], + "parameters": item["parameters"]}) + elif model_style in [ + ModelStyle.COHERE, + ModelStyle.OpenAI_Completions, + ModelStyle.Mistral, + ModelStyle.FIREWORK_AI, + ModelStyle.WRITER, + ModelStyle.NOVITA_AI, + ]: + oai_tool.append({"type": "function", "function": item}) + elif model_style == ModelStyle.AMAZON: + oai_tool.append({"toolSpec": item}) + + return oai_tool diff --git a/sieval/community/bfcl_v3/type_convertor/__init__.py b/sieval/community/bfcl_v3/type_convertor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sieval/community/bfcl_v3/type_convertor/java_type_converter.py b/sieval/community/bfcl_v3/type_convertor/java_type_converter.py new file mode 100644 index 00000000..c4a2194d --- /dev/null +++ b/sieval/community/bfcl_v3/type_convertor/java_type_converter.py @@ -0,0 +1,417 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/eval_checker/ast_eval/type_convertor/java_type_converter.py +Upstream blob: 7d2b141dfcb8e527df6424893e60a833eb69f3e1 +License: Apache-2.0 + +Only deviation: absolute `bfcl_eval.*` imports rewritten to relative ones +(`.type_mappings`, `.type_convertor.*`, `._model_config`). No other edit. +""" + +import re +from typing import List, Dict, Union +from ..type_mappings import JAVA_TYPE_CONVERSION + + +def java_type_converter(value, expected_type, nested_type=None): + if expected_type not in JAVA_TYPE_CONVERSION: + raise ValueError(f"Unsupported type: {expected_type}") + if ( + expected_type == "byte" + or expected_type == "short" + or expected_type == "integer" + ): + if not re.match(r"^-?\d+$", value): + return str(value) # default to string + return int(value) + elif expected_type == "float": + if not re.match(r"^-?\d+(\.\d+)?([eE][+-]?\d+)?[fF]$", value): + return str(value) # default to string + return float(re.sub(r"[fF]$", "", value)) + elif expected_type == "double": + if not re.match(r"^-?\d+(\.\d+)?([eE][+-]?\d+)?$", value): + return str(value) # default to string + return float(value) + elif expected_type == "long": + if not re.match(r"^-?\d+[lL]$", value): + return str(value) # default to string + return int(re.sub(r"[lL]$", "", value)) + elif expected_type == "boolean": + if value not in ["true", "false"]: + return str(value) # default to string + return parse_java_boolean(value) + elif expected_type == "char": + if not re.match(r"^\'.$\'", value): + return str(value) # default to string + return value # Remove the single quotes + elif expected_type == "Array" or expected_type == "ArrayList": + return parse_java_collection(value, expected_type, nested_type) + elif expected_type == "Set": + raise NotImplementedError("Set conversion is not implemented") + elif expected_type == "HashMap": + return parse_java_collection(value, expected_type, nested_type) + elif expected_type == "Hashtable": + raise NotImplementedError("Set conversion is not implemented") + elif expected_type == "Queue" or expected_type == "Stack": + raise NotImplementedError(f"{expected_type} conversion is not implemented") + elif expected_type == "String" or expected_type == "any": + return str(value) # we output as string for `any` type + else: + raise ValueError(f"Unsupported type: {expected_type}") + + +def parse_java_boolean(value): + return value == "true" + + +def parse_java_collection( + input_str: str, type_str: str, nested_type=None +) -> Union[List, Dict]: + if type_str == "ArrayList": + return parse_arraylist(input_str, nested_type) + elif type_str == "Array": + return parse_array(input_str, nested_type) + elif type_str == "HashMap": + return parse_hashmap(input_str) + else: + raise ValueError(f"Unsupported type: {type_str}") + + +def parse_arraylist(input_str: str, nested_type=None) -> List: + match_asList = re.search( + r"new\s+ArrayList<\w*>\(Arrays\.asList\((.+?)\)\)", input_str + ) + if match_asList: + elements_str = match_asList.group(1) + elements = [] + for element_str in elements_str.split(","): + element_str = element_str.strip() + if nested_type == "char": + element = element_str[1:-1] # Remove the single quotes + elif nested_type == "String": + element = element_str[1:-1] # Remove the double quotes + else: + element = ( + java_type_converter(element_str, nested_type) + if nested_type + else parse_java_value(element_str) + ) + elements.append(element) + return elements + + match_add = re.search( + r"new\s+ArrayList<\w*>\(\)\s*\{\{\s*(.+?)\s*\}\}", input_str, re.DOTALL + ) + if match_add: + adds_str = match_add.group(1) + elements = [] + matches = re.findall(r"add\((.+?)\)", adds_str) + for match in matches: + value_str = match.strip() + if nested_type == "char": + value = value_str[1:-1] # Remove the single quotes + elif nested_type == "String": + value = value_str[1:-1] # Remove the double quotes + else: + value = ( + java_type_converter(value_str, nested_type) + if nested_type + else parse_java_value(value_str) + ) + elements.append(value) + return elements + + match_empty = re.search(r"new\s+ArrayList<\w*>\(\)", input_str) + if match_empty: + return [] # Return an empty list for an empty ArrayList + + return input_str # default to string + + +def parse_array(input_str: str, nested_type=None) -> List: + match = re.search(r"new\s+\w+\[\]\s*\{(.*?)\}", input_str) + if match: + elements_str = match.group(1) + if nested_type: + elements = [ + java_type_converter(x.strip(), nested_type) + for x in elements_str.split(",") + if x.strip() + ] + else: + elements = [ + parse_java_value(x.strip()) + for x in elements_str.split(",") + if x.strip() + ] + + return elements + else: + return input_str # default to string + + +def parse_hashmap(input_str: str) -> Dict: + elements = {} + match = re.search( + r"new\s+HashMap<.*?>\s*\(\)\s*\{\s*\{?\s*(.*?)\s*\}?\s*\}", input_str, re.DOTALL + ) + if match: + puts_str = match.group(1) + if puts_str.strip(): + matches = re.findall(r"put\(\"(.*?)\",\s*(.*?)\)", puts_str) + for match in matches: + key = match[0] + value = parse_java_value(match[1].strip()) + elements[key] = value + return elements + + match_empty = re.search(r"new\s+HashMap<.*?>\s*\(\)", input_str) + if match_empty: + return {} # Return an empty dictionary for an empty HashMap + + return input_str # default to string + + +# This method parses without the information of what each element type is, contrary of the previous +def parse_java_value(value_str: str): + # check if it's boolean + if value_str == "true": + return True + elif value_str == "false": + return False + # check if it's a string + elif value_str.startswith('"') and value_str.endswith('"'): + return value_str[1:-1] + # check if it's a long + elif re.match(r"^-?\d+[lL]$", value_str): + return int(value_str[:-1]) + # check if it's a float + elif re.match(r"^-?\d+(\.\d+)?([eE][+-]?\d+)?[fF]$", value_str): + return float(re.sub(r"[fF]$", "", value_str)) + # check if it's a integer-like and float-like types (including byte, short, integer, double, etc) + else: + try: + return int(value_str) + except ValueError: + try: + return float(value_str) + except ValueError: + # this assuming all other types are converted to string + return value_str + + +# Write tests for the `java_type_converter` function +def test_java_type_converter(): + # Test valid conversions + assert java_type_converter("true", "boolean") == True + assert java_type_converter("false", "boolean") == False + assert java_type_converter("123", "integer") == 123 + assert java_type_converter("-123", "integer") == -123 + assert java_type_converter("3.14f", "float") == 3.14 + assert java_type_converter("-3.14f", "float") == -3.14 + assert java_type_converter("3.14", "double") == 3.14 + assert java_type_converter("-3.14", "double") == -3.14 + assert java_type_converter("123L", "long") == 123 + assert java_type_converter("-123L", "long") == -123 + assert java_type_converter("a", "char") == "a" + assert java_type_converter("abc", "String") == "abc" + assert java_type_converter("new int[]{1, 2, 3}", "Array") == [1, 2, 3] + assert java_type_converter( + 'new ArrayList<>(Arrays.asList("a", "b"))', "ArrayList" + ) == ["a", "b"] + assert java_type_converter( + 'new HashMap() {{ put("key", "value"); }}', "HashMap" + ) == {"key": "value"} + assert java_type_converter("3f", "float") == 3.0 + assert java_type_converter("3e3F", "float") == 3e3 + assert java_type_converter("3e-3F", "float") == 3e-3 + assert java_type_converter("3.14e2", "double") == 3.14e2 + assert java_type_converter("3.14e-2", "double") == 3.14e-2 + assert java_type_converter("127", "byte") == 127 + assert java_type_converter("-128", "byte") == -128 + assert java_type_converter("32767", "short") == 32767 + assert java_type_converter("-32768", "short") == -32768 + assert java_type_converter("9223372036854775807L", "long") == 9223372036854775807 + assert java_type_converter("-9223372036854775808L", "long") == -9223372036854775808 + assert java_type_converter("123", "any") == "123" + assert java_type_converter("abc", "any") == "abc" + + # Test empty collections + assert java_type_converter("new int[]{}", "Array") == [] + assert java_type_converter("new ArrayList<>()", "ArrayList") == [] + assert java_type_converter("new HashMap<>()", "HashMap") == {} + + # Test collections with mixed types + assert java_type_converter('new Object[]{1, "abc", true}', "Array") == [ + 1, + "abc", + True, + ] + assert java_type_converter( + 'new ArrayList<>(Arrays.asList(1, "abc", true))', "ArrayList" + ) == [1, "abc", True] + assert java_type_converter( + 'new HashMap() {{ put("key1", 1); put("key2", "value"); put("key3", true); }}', + "HashMap", + ) == {"key1": 1, "key2": "value", "key3": True} + + # Test invalid values + try: + java_type_converter("true", "integer") + except ValueError as e: + assert str(e) == "Invalid integer value: true" + + try: + java_type_converter("abc", "integer") + except ValueError as e: + assert str(e) == "Invalid integer value: abc" + + try: + java_type_converter("abc", "long") + except ValueError as e: + assert str(e) == "Invalid long value: abc" + + try: + java_type_converter("3.14", "float") + except ValueError as e: + assert str(e) == "Invalid float value: 3.14" + + try: + java_type_converter("3.14f", "double") + except ValueError as e: + assert str(e) == "Invalid double value: 3.14f" + + try: + java_type_converter("128", "byte") + except ValueError as e: + assert str(e) == "Invalid byte value: 128" + + try: + java_type_converter("32768", "short") + except ValueError as e: + assert str(e) == "Invalid short value: 32768" + + try: + java_type_converter("invalid", "boolean") + except ValueError as e: + assert str(e) == "Invalid boolean value: invalid" + + try: + java_type_converter("abc", "char") + except ValueError as e: + assert str(e) == "Invalid char value: abc" + + # Test unsupported types + try: + java_type_converter("abc", "Set") + except NotImplementedError as e: + assert str(e) == "Set conversion is not implemented" + + try: + java_type_converter("abc", "Hashtable") + except NotImplementedError as e: + assert str(e) == "Set conversion is not implemented" + + try: + java_type_converter("abc", "Queue") + except NotImplementedError as e: + assert str(e) == "Queue conversion is not implemented" + + try: + java_type_converter("abc", "Stack") + except NotImplementedError as e: + assert str(e) == "Stack conversion is not implemented" + + # extra array testing + assert java_type_converter("new int[]{}", "Array") == [] + assert java_type_converter("new int[] {}", "Array") == [] + assert java_type_converter("new int[] { }", "Array") == [] + assert java_type_converter("new int[]{1,2,3}", "Array") == [1, 2, 3] + assert java_type_converter("new int[]{1, 2, 3}", "Array") == [1, 2, 3] + assert java_type_converter("new int[] {1, 2, 3}", "Array") == [1, 2, 3] + assert java_type_converter("new int[] { 1, 2, 3 }", "Array") == [1, 2, 3] + + # extra hashmap testing + assert java_type_converter("new HashMap<>()", "HashMap") == {} + assert java_type_converter("new HashMap<>() {}", "HashMap") == {} + assert java_type_converter("new HashMap<>() {{}}", "HashMap") == {} + assert java_type_converter("new HashMap<>() {{ }}", "HashMap") == {} + assert java_type_converter( + 'new HashMap() {{ put("key", "value"); }}', "HashMap" + ) == {"key": "value"} + assert java_type_converter( + 'new HashMap() {{put("key", "value");}}', "HashMap" + ) == {"key": "value"} + assert java_type_converter( + 'new HashMap() { { put("key", "value"); } }', "HashMap" + ) == {"key": "value"} + assert java_type_converter( + 'new HashMap() {{ put("key1", 123); put("key2", true); }}', + "HashMap", + ) == {"key1": 123, "key2": True} + assert java_type_converter( + 'new HashMap() {{ put("key1", "value 1"); put("key2", "value 2"); }}', + "HashMap", + ) == {"key1": "value 1", "key2": "value 2"} + + def test_parse_array_long(): + input_str = "new long[]{1L, 2L, 3L}" + expected_output = [1, 2, 3] + assert parse_array(input_str, nested_type="long") == expected_output + + def test_parse_array_mixed_long(): + input_str = "new long[]{1L, 2, 3L}" + expected_output = [1, "2", 3] + assert parse_array(input_str, nested_type="long") == expected_output + + def test_parse_array_invalid_long(): + input_str = "new long[]{1L, 2.0, 3L}" + expected_output = [1, "2.0", 3] + assert parse_array(input_str, nested_type="long") == expected_output + + def test_parse_arraylist_int(): + input_str = "new ArrayList(Arrays.asList(1, 2, 3))" + expected_output = [1, 2, 3] + assert parse_arraylist(input_str, nested_type="integer") == expected_output + + def test_parse_arraylist_float(): + input_str = "new ArrayList() {{ add(1.0f); add(2.0f); add(3.0f); }}" + expected_output = [1.0, 2.0, 3.0] + assert parse_arraylist(input_str, nested_type="float") == expected_output + + def test_parse_arraylist_double(): + input_str = "new ArrayList() {{ add(1.0); add(2.0); add(3.0); }}" + expected_output = [1.0, 2.0, 3.0] + assert parse_arraylist(input_str, nested_type="double") == expected_output + + def test_parse_arraylist_boolean(): + input_str = "new ArrayList(Arrays.asList(true, false, true))" + expected_output = [True, False, True] + assert parse_arraylist(input_str, nested_type="boolean") == expected_output + + def test_parse_arraylist_char(): + input_str = "new ArrayList() {{ add('a'); add('b'); add('c'); }}" + expected_output = ["a", "b", "c"] + print(parse_arraylist(input_str, nested_type="char")) + assert parse_arraylist(input_str, nested_type="char") == expected_output + + def test_parse_arraylist_string(): + input_str = 'new ArrayList() {{ add("aasdasd"); add("basdasd"); add("casdasd"); }}' + expected_output = ["aasdasd", "basdasd", "casdasd"] + print(parse_arraylist(input_str)) + assert parse_arraylist(input_str) == expected_output + + test_parse_array_long() + test_parse_array_mixed_long() + test_parse_array_invalid_long() + test_parse_arraylist_int() + test_parse_arraylist_float() + test_parse_arraylist_double() + test_parse_arraylist_boolean() + test_parse_arraylist_char() + test_parse_arraylist_string() + print("All tests passed successfully!") + + +if __name__ == "__main__": + test_java_type_converter() diff --git a/sieval/community/bfcl_v3/type_convertor/js_type_converter.py b/sieval/community/bfcl_v3/type_convertor/js_type_converter.py new file mode 100644 index 00000000..5522967f --- /dev/null +++ b/sieval/community/bfcl_v3/type_convertor/js_type_converter.py @@ -0,0 +1,321 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/eval_checker/ast_eval/type_convertor/js_type_converter.py +Upstream blob: 4205b92e05af83d2e61a7bf4bd2530718b57acee +License: Apache-2.0 + +Only deviation: absolute `bfcl_eval.*` imports rewritten to relative ones +(`.type_mappings`, `.type_convertor.*`, `._model_config`). No other edit. +""" + +import re +from ..type_mappings import JS_TYPE_CONVERSION + + +def js_type_converter(value, expected_type, nested_type=None): + if expected_type not in JS_TYPE_CONVERSION: + raise ValueError(f"Unsupported type: {expected_type}") + + if expected_type == "String": + if not (value.startswith('"') and value.endswith('"')) and not ( + value.startswith("'") and value.endswith("'") + ): + return str(value) + return value[1:-1] + + elif expected_type == "integer": + if not re.match(r"^-?\d+$", value): + return str(value) # default to string + return int(value) + elif expected_type == "float": + if not re.match(r"^-?\d+(\.\d+)?$", value): + return str(value) # default to string + return float(value) + elif expected_type == "Bigint": + if not re.match(r"^-?\d+n$", value): + return str(value) # default to string + return int(value[:-1]) + elif expected_type == "Boolean": + if value not in ["true", "false"]: + return str(value) # default to string + return value == "true" + elif expected_type == "dict": + return parse_js_collection(value, "dict", nested_type) + elif expected_type == "array": + return parse_js_collection(value, "array", nested_type) + elif expected_type == "any": + return str(value) + else: + raise ValueError(f"Unsupported type: {expected_type}") + + +def parse_js_collection(code, type_str, nested_type=None): + code = code.strip() + if type_str == "array": + # Regular expression patterns + array_2d_pattern = r"\[\s*\[.*?\]\s*(,\s*\[.*?\]\s*)*\]|\bnew\s+Array\(\s*\[.*?\]\s*(,\s*\[.*?\]\s*)*\)" + array_pattern = r"\[(.*?)\]|\bnew\s+Array\((.*?)\)" + + # Check if the code is a 2D array + array_2d_match = re.match(array_2d_pattern, code) + try: + if array_2d_match: + elements_str = array_2d_match.group(0) + inner_arrays = re.findall(r"\[(.*?)\]", elements_str) + elements = [] + for idx, inner_array_str in enumerate(inner_arrays): + inner_array_str = inner_array_str.strip() + if idx == 0 and inner_array_str.startswith("["): + inner_array_str = inner_array_str[1:] + inner_array_elements = [ + e.strip() for e in inner_array_str.split(",") + ] + if nested_type: + inner_array = [parse_js_value(e) for e in inner_array_elements] + else: + inner_array = [parse_js_value(e) for e in inner_array_elements] + elements.append(inner_array) + return elements + + # Check if the code is a 1D array + array_match = re.match(array_pattern, code) + if array_match: + if array_match.group(1) is not None: + elements_str = array_match.group(1).strip() + if elements_str: + elements = elements_str.split(",") + else: + elements = [] + elif array_match.group(2) is not None: + elements_str = array_match.group(2).strip() + if elements_str: + elements = elements_str.split(",") + else: + elements = [] + else: + elements = [] + if nested_type: + elements = [ + ( + js_type_converter(e.strip(), nested_type, "String") + if (e.strip().startswith("'") or e.strip().startswith('"')) + else js_type_converter(e.strip(), nested_type) + ) + for e in elements + ] + else: + elements = [parse_js_value(e.strip()) for e in elements] + return elements + else: + return code + except: + return code + + elif type_str == "dict": + if code == "{}": + return {} # Return an empty dictionary for an empty object + dict_pattern = r"\{(.*?)\}" + # Check if the code is a dictionary + dict_match = re.match(dict_pattern, code) + if dict_match: + try: + content = dict_match.group(1) + pairs = re.findall(r"([^:]+):\s*(.*?)(?:,\s*(?=[^,]+:)|$)", content) + dictionary = {} + for key, value in pairs: + key = key.strip().strip("'\"") + value = value.strip() + if value.startswith("[") and value.endswith("]"): + # Handle array values + dictionary[key] = parse_js_collection(value, "array") + elif value.startswith("{") and value.endswith("}"): + # Handle nested dictionary values + dictionary[key] = parse_js_collection(value, "dict") + else: + dictionary[key] = parse_js_value(value.strip("'\"")) + return dictionary + except Exception as e: + print(f"Error parsing dictionary: {e}") + return code + else: + return code # default to string + else: + raise ValueError(f"Unsupported type: {type_str}") + + +def parse_js_value(value_str: str): + value_str = value_str.strip() + if value_str == "true": + return True + elif value_str == "false": + return False + elif (value_str.startswith('"') and value_str.endswith('"')) or ( + value_str.startswith("'") and value_str.endswith("'") + ): + return value_str[1:-1] + else: + try: + return int(value_str) + except ValueError: + try: + return float(value_str) + except ValueError: + return value_str + + +# Write tests for the `js_type_converter` function +def test_js_type_converter(): + assert js_type_converter("true", "Boolean") == True + assert js_type_converter("false", "Boolean") == False + assert js_type_converter("123", "integer") == 123 + assert js_type_converter("3.14", "float") == 3.14 + assert js_type_converter("123n", "Bigint") == 123 + assert js_type_converter("abc", "String") == "abc" + assert js_type_converter("[1, 2, 3]", "array") == [1, 2, 3] + assert js_type_converter("new Array(1, 2, 3)", "array") == [1, 2, 3] + assert js_type_converter("{'key': 'value'}", "dict") == {"key": "value"} + assert js_type_converter("{'key': 123}", "dict") == {"key": 123} + assert js_type_converter("{'key': true}", "dict") == {"key": True} + + # Additional test cases + # Test empty array and dictionary + assert js_type_converter("[]", "array") == [] + assert js_type_converter("{}", "dict") == {} + + # Test array with mixed types + assert js_type_converter("[1, 'two', true]", "array") == [1, "two", True] + + # Test dictionary with mixed types + assert js_type_converter( + "{'key1': 123, 'key2': 'value', 'key3': false}", "dict" + ) == {"key1": 123, "key2": "value", "key3": False} + + # Test string with special characters + + # Test negative integer and float values + assert js_type_converter("-123", "integer") == -123 + assert js_type_converter("-3.14", "float") == -3.14 + + # Test invalid type + try: + js_type_converter("123", "InvalidType") + except ValueError as e: + assert str(e) == "Unsupported type: InvalidType" + + # Test invalid integer value + try: + js_type_converter("123.45", "integer") + except ValueError as e: + assert str(e) == "Invalid integer value: 123.45" + + # Test invalid float value + try: + js_type_converter("3.14abc", "float") + except ValueError as e: + assert str(e) == "Invalid float value: 3.14abc" + + # Test invalid Bigint value + try: + js_type_converter("123", "Bigint") + except ValueError as e: + assert str(e) == "Invalid Bigint value: 123" + + # Test invalid boolean value + try: + js_type_converter("not_a_boolean", "Boolean") + except ValueError as e: + assert str(e) == "Invalid boolean value: not_a_boolean" + + print("All tests passed successfully!") + + +def test_js_type_converter_nested_array(): + # Test array with nested integers + assert js_type_converter("[1, 2, 3]", "array", "integer") == [1, 2, 3] + assert js_type_converter("new Array(4, 5, 6)", "array", "integer") == [4, 5, 6] + + # Test array with nested floats + assert js_type_converter("[1.1, 2.2, 3.3]", "array", "float") == [1.1, 2.2, 3.3] + assert js_type_converter("new Array(4.4, 5.5, 6.6)", "array", "float") == [ + 4.4, + 5.5, + 6.6, + ] + + # Test array with nested Bigints + assert js_type_converter("[1n, 2n, 3n]", "array", "Bigint") == [1, 2, 3] + assert js_type_converter("new Array(4n, 5n, 6n)", "array", "Bigint") == [4, 5, 6] + + # Test array with nested booleans + assert js_type_converter("[true, false, true]", "array", "Boolean") == [ + True, + False, + True, + ] + assert js_type_converter("new Array(false, true, false)", "array", "Boolean") == [ + False, + True, + False, + ] + + # Test array with nested strings + print(js_type_converter('["hello", "world", "!"]', "array", "String")) + assert js_type_converter('["hello", "world", "!"]', "array", "String") == [ + "hello", + "world", + "!", + ] + assert js_type_converter('new Array("foo", "bar", "baz")', "array", "String") == [ + "foo", + "bar", + "baz", + ] + + # Test array with mixed nested types + assert js_type_converter('[1, "two", true]', "array") == [1, "two", True] + assert js_type_converter('new Array(3.14, "pi", false)', "array") == [ + 3.14, + "pi", + False, + ] + + # Test array with nested arrays + print(js_type_converter(" [ [1, 2], [3, 4], [5, 6]]", "array", "array")) + assert js_type_converter(" [ [ 1, 2 ], [ 3, 4], [5, 6]]", "array", "array") == [ + [1, 2], + [3, 4], + [5, 6], + ] # this example has many weird spacings + assert js_type_converter("new Array([1, 2], [3, 4], [5, 6])", "array", "array") == [ + [1, 2], + [3, 4], + [5, 6], + ] + + # Test array with nested dictionaries + assert js_type_converter( + '[{"key1": 1}, {"key2": 2}, {"key3": 3}]', "array", "dict" + ) == [{"key1": 1}, {"key2": 2}, {"key3": 3}] + assert js_type_converter( + 'new Array({"key1": 1}, {"key2": 2}, {"key3": 3})', "array", "dict" + ) == [{"key1": 1}, {"key2": 2}, {"key3": 3}] + + print("All nested array tests passed successfully!") + + +def test_js_type_converter_dictionary_with_arrays(): + complex_dict = js_type_converter( + '{"initialState": initialStateObject, "reducers": reducersMap, "middlewares": ["loggerMiddleware"], "enhancers": ["applyMiddleware", "myMiddleWare"]}', + "dict", + ) + assert isinstance(complex_dict, dict) + assert complex_dict["initialState"] == "initialStateObject" + assert complex_dict["reducers"] == "reducersMap" + assert complex_dict["middlewares"] == ["loggerMiddleware"] + assert complex_dict["enhancers"] == ["applyMiddleware", "myMiddleWare"] + print("Complex dictionary test passed successfully!") + +if __name__ == "__main__": + test_js_type_converter() + test_js_type_converter_nested_array() + test_js_type_converter_dictionary_with_arrays() diff --git a/sieval/community/bfcl_v3/type_mappings.py b/sieval/community/bfcl_v3/type_mappings.py new file mode 100644 index 00000000..ae29ccd8 --- /dev/null +++ b/sieval/community/bfcl_v3/type_mappings.py @@ -0,0 +1,100 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/constants/type_mappings.py +Upstream blob: fdbdde48402f8f2812b146f9b7341b6aae9e55b8 +License: Apache-2.0 + +Only deviation: absolute `bfcl_eval.*` imports rewritten to relative ones +(`.type_mappings`, `.type_convertor.*`, `._model_config`). No other edit. +This file has no imports, so no rewrite was needed here. +""" + +GORILLA_TO_OPENAPI = { + "integer": "integer", + "number": "number", + "float": "number", + "string": "string", + "boolean": "boolean", + "bool": "boolean", + "array": "array", + "list": "array", + "dict": "object", + "object": "object", + "tuple": "array", + "any": "string", + "byte": "integer", + "short": "integer", + "long": "integer", + "double": "number", + "char": "string", + "ArrayList": "array", + "Array": "array", + "HashMap": "object", + "Hashtable": "object", + "Queue": "array", + "Stack": "array", + "Any": "string", + "String": "string", + "Bigint": "integer", +} + +GORILLA_TO_PYTHON = { + "integer": "int", + "number": "float", + "float": "float", + "string": "str", + "boolean": "bool", + "bool": "bool", + "array": "list", + "list": "list", + "dict": "dict", + "object": "dict", + "tuple": "tuple", + "any": "str", + "byte": "int", + "short": "int", + "long": "int", + "double": "float", + "char": "str", + "ArrayList": "list", + "Array": "list", + "HashMap": "dict", + "Hashtable": "dict", + "Queue": "list", + "Stack": "list", + "Any": "str", + "String": "str", + "Bigint": "int", +} + + +JAVA_TYPE_CONVERSION = { + "byte": int, + "short": int, + "integer": int, + "float": float, + "double": float, + "long": int, + "boolean": bool, + "char": str, + "Array": list, + "ArrayList": list, + "Set": set, + "HashMap": dict, + "Hashtable": dict, + "Queue": list, # this can be `queue.Queue` as well, for simplicity we check with list + "Stack": list, + "String": str, + "any": str, +} + +JS_TYPE_CONVERSION = { + "String": str, + "integer": int, + "float": float, + "Bigint": int, + "Boolean": bool, + "dict": dict, + "array": list, + "any": str, +} diff --git a/tests/unit/community/bfcl_v3/test_identity.py b/tests/unit/community/bfcl_v3/test_identity.py new file mode 100644 index 00000000..4ff570af --- /dev/null +++ b/tests/unit/community/bfcl_v3/test_identity.py @@ -0,0 +1,54 @@ +import hashlib +from pathlib import Path + +import pytest + +import sieval.community.bfcl_v3 as pkg + +#: sha256 of each vendored file AS COMMITTED (upstream's bytes plus the +#: documented import rewrite and provenance docstring). Fill these in from the +#: failing run's output, once, after reviewing the diff against upstream. +VENDORED_SHA256 = { + "ast_checker.py": ( + "66ba103f5dd17d9e2b4ee597ebfdb013486add739412366e65c4cb6691acb936" + ), + "type_mappings.py": ( + "90e843fe10cc0a12868b11ef9634e1feb7361ec1f1526b83a0b3a0104ae6eec5" + ), + "type_convertor/java_type_converter.py": ( + "ead108a53b7ee0d4e4e165d735ab586a0befa3e34df934287d5a6db950431f9a" + ), + "type_convertor/js_type_converter.py": ( + "943afe13025dc48ef3902777fc48e45d5ab10e2a439f4c4d0abc3cf311fab595" + ), + "parser.py": ("4bba883c1f537a3ae0ad0c85260974861d056ffe25ab50a90bd07d7bb8dec2a3"), + "tool_convert.py": ( + "11b758572a09fca738479243e4cc3005387def672786b3231c868e6d73ad386e" + ), + "output_checks.py": ( + "683a43d71e3e55d52b5edc7afd94def3ce449261642c3dbde543ae82ceab5268" + ), + "aggregate.py": ( + "b7abb648a15492db0101d7a0628b8cc1fdfa2d1542992e0819a44911729c9472" + ), + "prompts.py": ("fae6ed78b354c3e1a355795569a78ba18f2b0826f52bfa1dca35a2454655b78f"), +} + + +@pytest.mark.parametrize("relative_path", sorted(VENDORED_SHA256)) +def test_vendored_file_has_not_drifted(relative_path: str): + root = Path(pkg.__file__).parent + digest = hashlib.sha256((root / relative_path).read_bytes()).hexdigest() + assert digest == VENDORED_SHA256[relative_path], ( + f"{relative_path} changed since vendoring. A vendored file tracks " + "upstream; if the change is deliberate, say what it is in the module " + "docstring's deviation list and update this hash in the same commit." + ) + + +def test_category_tables_match_the_pinned_snapshot(): + from sieval.community.bfcl_v3 import LIVE_COUNTS, NON_LIVE_COUNTS + + assert sum(NON_LIVE_COUNTS.values()) == 1390 + assert sum(LIVE_COUNTS.values()) == 2251 + assert set(NON_LIVE_COUNTS) & set(LIVE_COUNTS) == set() From db73ebb47f5a4307c811c5346ed763251563ecf5 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 17:24:10 +0800 Subject: [PATCH 03/28] fix(community): correct three misstated BFCL v3 provenance docstrings Review found that ast_checker.py's import-rewrite boilerplate had been copy-pasted into type_mappings.py, java_type_converter.py, and js_type_converter.py, where it doesn't hold: type_mappings.py has no imports at all (and now says so without self-contradiction), and each type_convertor file rewrites exactly one import at one level of relative nesting (`..type_mappings`), not the three-import set ast_checker.py actually has. The audit design in this task depends on these docstrings being followed literally to re-derive the copy, so a wrong one is a real risk, not a wording nit. Also drops _tables.py's dangling reference to a nonexistent `_base.py`, stating directly that LANGUAGE_BY_CATEGORY feeds ast_checker's type conversion rather than the AST parser's language. No vendored code changed -- only the three docstrings and one comment. sha256 hashes for type_mappings.py, java_type_converter.py, and js_type_converter.py are updated in the same commit since the docstring is part of the pinned file. Co-Authored-By: Claude Opus 5 --- sieval/community/bfcl_v3/_tables.py | 5 +++-- .../community/bfcl_v3/type_convertor/java_type_converter.py | 6 ++++-- .../community/bfcl_v3/type_convertor/js_type_converter.py | 6 ++++-- sieval/community/bfcl_v3/type_mappings.py | 5 ++--- tests/unit/community/bfcl_v3/test_identity.py | 6 +++--- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/sieval/community/bfcl_v3/_tables.py b/sieval/community/bfcl_v3/_tables.py index db7f7be8..f67d43b3 100644 --- a/sieval/community/bfcl_v3/_tables.py +++ b/sieval/community/bfcl_v3/_tables.py @@ -38,6 +38,7 @@ #: Of the goldless three, the one where producing a call is CORRECT. CALL_EXPECTED = frozenset({"live_relevance"}) -#: Only these two are not Python. `language` selects the type converter inside -#: `ast_checker` -- it is NOT the parser's language (see `_base.py`). +#: Only these two are not Python. This feeds `ast_checker`'s type-conversion +#: argument, not the AST parser's language -- `ast_parse` always parses Python +#: syntax, since these categories' model outputs are still Python call syntax. LANGUAGE_BY_CATEGORY = {"java": "Java", "javascript": "JavaScript"} diff --git a/sieval/community/bfcl_v3/type_convertor/java_type_converter.py b/sieval/community/bfcl_v3/type_convertor/java_type_converter.py index c4a2194d..98a09564 100644 --- a/sieval/community/bfcl_v3/type_convertor/java_type_converter.py +++ b/sieval/community/bfcl_v3/type_convertor/java_type_converter.py @@ -4,8 +4,10 @@ Upstream blob: 7d2b141dfcb8e527df6424893e60a833eb69f3e1 License: Apache-2.0 -Only deviation: absolute `bfcl_eval.*` imports rewritten to relative ones -(`.type_mappings`, `.type_convertor.*`, `._model_config`). No other edit. +Only deviation: `from bfcl_eval.constants.type_mappings import JAVA_TYPE_CONVERSION` +rewritten to `from ..type_mappings import JAVA_TYPE_CONVERSION` (this module lives +one package level deeper than `type_mappings.py`, hence the double dot). No +other edit. """ import re diff --git a/sieval/community/bfcl_v3/type_convertor/js_type_converter.py b/sieval/community/bfcl_v3/type_convertor/js_type_converter.py index 5522967f..7189664a 100644 --- a/sieval/community/bfcl_v3/type_convertor/js_type_converter.py +++ b/sieval/community/bfcl_v3/type_convertor/js_type_converter.py @@ -4,8 +4,10 @@ Upstream blob: 4205b92e05af83d2e61a7bf4bd2530718b57acee License: Apache-2.0 -Only deviation: absolute `bfcl_eval.*` imports rewritten to relative ones -(`.type_mappings`, `.type_convertor.*`, `._model_config`). No other edit. +Only deviation: `from bfcl_eval.constants.type_mappings import JS_TYPE_CONVERSION` +rewritten to `from ..type_mappings import JS_TYPE_CONVERSION` (this module lives +one package level deeper than `type_mappings.py`, hence the double dot). No +other edit. """ import re diff --git a/sieval/community/bfcl_v3/type_mappings.py b/sieval/community/bfcl_v3/type_mappings.py index ae29ccd8..51936d2b 100644 --- a/sieval/community/bfcl_v3/type_mappings.py +++ b/sieval/community/bfcl_v3/type_mappings.py @@ -4,9 +4,8 @@ Upstream blob: fdbdde48402f8f2812b146f9b7341b6aae9e55b8 License: Apache-2.0 -Only deviation: absolute `bfcl_eval.*` imports rewritten to relative ones -(`.type_mappings`, `.type_convertor.*`, `._model_config`). No other edit. -This file has no imports, so no rewrite was needed here. +This file has no imports of its own, so there was nothing to rewrite. It is +byte-identical to upstream. """ GORILLA_TO_OPENAPI = { diff --git a/tests/unit/community/bfcl_v3/test_identity.py b/tests/unit/community/bfcl_v3/test_identity.py index 4ff570af..e86e4b11 100644 --- a/tests/unit/community/bfcl_v3/test_identity.py +++ b/tests/unit/community/bfcl_v3/test_identity.py @@ -13,13 +13,13 @@ "66ba103f5dd17d9e2b4ee597ebfdb013486add739412366e65c4cb6691acb936" ), "type_mappings.py": ( - "90e843fe10cc0a12868b11ef9634e1feb7361ec1f1526b83a0b3a0104ae6eec5" + "cebc6f9a1b28e277483dd13b2a3fd3ceb95ad8a4631daf50933ad1af3015c698" ), "type_convertor/java_type_converter.py": ( - "ead108a53b7ee0d4e4e165d735ab586a0befa3e34df934287d5a6db950431f9a" + "d33be3f7e65f0302ef729b0ae328e4d7395758b9b92d8d4562365f97dbc84942" ), "type_convertor/js_type_converter.py": ( - "943afe13025dc48ef3902777fc48e45d5ab10e2a439f4c4d0abc3cf311fab595" + "dfb0373b58b51c45e63346e1a893c055168dd6ae39e27307436fefa45981e6cc" ), "parser.py": ("4bba883c1f537a3ae0ad0c85260974861d056ffe25ab50a90bd07d7bb8dec2a3"), "tool_convert.py": ( From c037c479ebd258638c2e0336f85e16db3c5206b5 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 17:50:34 +0800 Subject: [PATCH 04/28] feat(community): vendor BFCL's Java and JavaScript source parsers `ast_parse` dispatches the java and javascript categories to tree-sitter parsers rather than Python's `ast`, and the first vendoring pass extracted `ast_parse` without the two imports that reach them -- leaving 150 of the 1390 non-live samples raising NameError at decode time on the prompt protocol. Both files are copied verbatim; neither needed an import rewrite. The three tree-sitter pins are upstream's own. 0.21.3 predates the 0.22 API break, so it is pinned rather than floored, and CI installs the group so the path cannot rot unimported. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 8 +- pdm.lock | 54 ++++++- pyproject.toml | 14 ++ sieval/community/bfcl_v3/__init__.py | 3 +- sieval/community/bfcl_v3/_tables.py | 6 +- sieval/community/bfcl_v3/parser.py | 30 ++-- .../bfcl_v3/source_parser/__init__.py | 0 .../bfcl_v3/source_parser/java_parser.py | 151 ++++++++++++++++++ .../bfcl_v3/source_parser/js_parser.py | 79 +++++++++ tests/unit/community/bfcl_v3/test_identity.py | 8 +- .../community/bfcl_v3/test_source_parsers.py | 40 +++++ 11 files changed, 370 insertions(+), 23 deletions(-) create mode 100644 sieval/community/bfcl_v3/source_parser/__init__.py create mode 100644 sieval/community/bfcl_v3/source_parser/java_parser.py create mode 100644 sieval/community/bfcl_v3/source_parser/js_parser.py create mode 100644 tests/unit/community/bfcl_v3/test_source_parsers.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 965e2a69..43e93c64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,9 +27,13 @@ env: # is tested at all. nltk alone is not enough -- the pre-2020 grader reaches # only nltk, so without sqlparse the test-suite metric and the hardened # execution under it are silently uncollected here while passing locally. + # `bfcl-v3` is installed rather than left out like `hle`/`multi-if`: it gates + # the java/javascript decode path, the one thing the vendored source-text + # parsers exist to make work, and an untested path silently rots. The cost is + # one small C extension (tree-sitter) built from sdist per run. INSTALL_GROUPS: >- - -G drop -G ifbench -G ifeval -G iheval -G math -G scicode -G spider -G test - -G dev + -G bfcl-v3 -G drop -G ifbench -G ifeval -G iheval -G math -G scicode -G spider + -G test -G dev jobs: changes: diff --git a/pdm.lock b/pdm.lock index dd695add..14cf586b 100644 --- a/pdm.lock +++ b/pdm.lock @@ -2,10 +2,10 @@ # It is not intended for manual editing. [metadata] -groups = ["default", "dev", "drop", "hle", "ifbench", "ifeval", "iheval", "math", "multi-if", "nl2sh-alfa", "ruler", "ruler-gen", "scicode", "spider", "spider2", "t-eval", "test", "wikisql"] +groups = ["default", "bfcl-v3", "dev", "drop", "hle", "ifbench", "ifeval", "iheval", "math", "multi-if", "nl2sh-alfa", "ruler", "ruler-gen", "scicode", "spider", "spider2", "t-eval", "test", "wikisql"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:2733395ff19bb311187e466cf75746088e4f331191d869ff9dedcbe5fc05bc33" +content_hash = "sha256:f23e18a1776f70c47e1953d93775d51c9aa69cec822c0398116c3752eb787bfc" [[metadata.targets]] requires_python = ">=3.12,<3.15" @@ -3748,6 +3748,56 @@ files = [ {file = "transformers-4.57.3.tar.gz", hash = "sha256:df4945029aaddd7c09eec5cad851f30662f8bd1746721b34cc031d70c65afebc"}, ] +[[package]] +name = "tree-sitter" +version = "0.21.3" +requires_python = ">=3.8" +summary = "Python bindings for the Tree-Sitter parsing library" +groups = ["bfcl-v3"] +files = [ + {file = "tree-sitter-0.21.3.tar.gz", hash = "sha256:b5de3028921522365aa864d95b3c41926e0ba6a85ee5bd000e10dc49b0766988"}, + {file = "tree_sitter-0.21.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:669b3e5a52cb1e37d60c7b16cc2221c76520445bb4f12dd17fd7220217f5abf3"}, + {file = "tree_sitter-0.21.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2aa2a5099a9f667730ff26d57533cc893d766667f4d8a9877e76a9e74f48f0d3"}, + {file = "tree_sitter-0.21.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a3e06ae2a517cf6f1abb682974f76fa760298e6d5a3ecf2cf140c70f898adf0"}, + {file = "tree_sitter-0.21.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af992dfe08b4fefcfcdb40548d0d26d5d2e0a0f2d833487372f3728cd0772b48"}, + {file = "tree_sitter-0.21.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c7cbab1dd9765138505c4a55e2aa857575bac4f1f8a8b0457744a4fefa1288e6"}, + {file = "tree_sitter-0.21.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e1e66aeb457d1529370fcb0997ae5584c6879e0e662f1b11b2f295ea57e22f54"}, + {file = "tree_sitter-0.21.3-cp312-cp312-win_amd64.whl", hash = "sha256:013c750252dc3bd0e069d82e9658de35ed50eecf31c6586d0de7f942546824c5"}, +] + +[[package]] +name = "tree-sitter-java" +version = "0.21.0" +requires_python = ">=3.8" +summary = "Java grammar for tree-sitter" +groups = ["bfcl-v3"] +files = [ + {file = "tree_sitter_java-0.21.0-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:d30fb39af6590f6be877ff8d39c7e193f37594375b8fcdda55babd264411616b"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:696391968cdc467f97e525fbe43790c2d5ba46bf446376eb30d5c20b5c9d795a"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b82318982531a473c598601d73956b0b1f9e608d39e427ba8c1a59f2d6da262"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9c01f48ec3a3d4b55fb2c56b7cad6ebf94dba568d690b45d966731859485dd4"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:d83ca51b26826550051c53468ec74f5b570b4afb4d4520df0aa2a35b52de2289"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:8afb56c90048272b079235b82fc2a4a69b058b4ace206d9475cbad2eb143f40a"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-win_amd64.whl", hash = "sha256:6534fac27a93160a2b27a0c77f22a7c91f0aee2dba9d4cdbf835bf509ddfbf4f"}, +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.21.4" +requires_python = ">=3.8" +summary = "Javascript grammar for tree-sitter" +groups = ["bfcl-v3"] +files = [ + {file = "tree-sitter-javascript-0.21.4.tar.gz", hash = "sha256:5ae97db218c22f16f1fd1108d77c4017b453addc36041136779c99800f23ff20"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:e0f1d26ec85cc66d56fb25fd41ce499dc5c4a60b478a91754684e2289433daae"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e9172dc9e59649d7598c6509103db1648e7187c2bcb39c34222fc052b9db0ac4"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d45bd39ce931b164f84cfb6a186df504e38c6f91f541e2e60d5a40eb9d574005"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00ff426e9bb552abb49ce3234b083edb1a6aa73bfeea7d9bf0f6bb3d0256c4cd"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c2945bc49985ab396773e2738ae9b1e7d06950d6ad70c535c010fb9b2816a855"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5fc87d07f4c6a7067d4e0f004956a549c05e21eeced3287f3ea2bdd04a71a97a"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-win_amd64.whl", hash = "sha256:2843b0e81564d8176922ef2b40db128a4de026606d6b1d22a06efb8e8a5c01b8"}, +] + [[package]] name = "triton" version = "3.5.1" diff --git a/pyproject.toml b/pyproject.toml index 2a2995eb..d4170fc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,20 @@ Issues = "https://github.com/scitix/sieval/issues" Changelog = "https://github.com/scitix/sieval/blob/main/CHANGELOG.md" [project.optional-dependencies] +bfcl-v3 = [ + # Reached only by the Prompt protocol on the `java` and `javascript` + # categories: upstream's `ast_parse` dispatches those two to tree-sitter + # parsers rather than Python's `ast`. The FC protocol never parses source + # text, and every Live category is Python. + # + # Pinned to upstream's own versions. tree_sitter 0.21.3 predates the 0.22 API + # break -- the vendored parsers call `Language(ptr, name)` and + # `Parser.set_language()`, both removed later -- so this is a hard pin, not a + # floor. It has no wheel for Python 3.13 and builds from sdist. + "tree_sitter==0.21.3", + "tree-sitter-java==0.21.0", + "tree-sitter-javascript==0.21.4", +] drop = ["numpy<=2.2", "scipy>=1.16.3"] # numpy backs the vendored HLE `calib_err` (calibration error). No grader/judge # extras: the judge is reached through the base `openai` client like any model. diff --git a/sieval/community/bfcl_v3/__init__.py b/sieval/community/bfcl_v3/__init__.py index 87e55803..fdb39609 100644 --- a/sieval/community/bfcl_v3/__init__.py +++ b/sieval/community/bfcl_v3/__init__.py @@ -13,7 +13,8 @@ applied, so a reviewer can re-derive the copy from a fresh checkout. Whole-file vendored modules: `ast_checker.py`, `type_mappings.py`, -`type_convertor/java_type_converter.py`, `type_convertor/js_type_converter.py`. +`type_convertor/java_type_converter.py`, `type_convertor/js_type_converter.py`, +`source_parser/java_parser.py`, `source_parser/js_parser.py`. Partial vendored modules (named subsets of a larger upstream file -- see each module's own docstring for exactly which symbols were taken and why): diff --git a/sieval/community/bfcl_v3/_tables.py b/sieval/community/bfcl_v3/_tables.py index f67d43b3..31f63934 100644 --- a/sieval/community/bfcl_v3/_tables.py +++ b/sieval/community/bfcl_v3/_tables.py @@ -38,7 +38,7 @@ #: Of the goldless three, the one where producing a call is CORRECT. CALL_EXPECTED = frozenset({"live_relevance"}) -#: Only these two are not Python. This feeds `ast_checker`'s type-conversion -#: argument, not the AST parser's language -- `ast_parse` always parses Python -#: syntax, since these categories' model outputs are still Python call syntax. +#: Only these two are not Python. The same string selects both the +#: type converter inside `ast_checker` and the source-text parser inside +#: `ast_parse` -- these categories' model outputs are Java/JavaScript source. LANGUAGE_BY_CATEGORY = {"java": "Java", "javascript": "JavaScript"} diff --git a/sieval/community/bfcl_v3/parser.py b/sieval/community/bfcl_v3/parser.py index 6d392b80..4cfcfde3 100644 --- a/sieval/community/bfcl_v3/parser.py +++ b/sieval/community/bfcl_v3/parser.py @@ -9,25 +9,27 @@ the rest of `model_handler/utils.py` (prompt construction, decoders for other model styles, retry helpers) is out of scope for this port. -Only deviation: the module needs no `bfcl_eval.*` imports at all for these -three functions (`import ast` is stdlib), so there was nothing to rewrite. +Deviations: +1. The module needs no `bfcl_eval.*` imports at all for these three + functions (`import ast` is stdlib), so their bodies needed no rewrite. +2. Upstream's `utils.py:13-14` imports `parse_java_function_call` and + `parse_javascript_function_call` from `bfcl_eval.model_handler.parser.*`. + Rewritten to `from .source_parser.java_parser import + parse_java_function_call` and the `js_parser` twin -- same rename as + `source_parser/`'s own docstrings explain (this package's `parser.py` + already occupies upstream's directory name). -Note on the Java/JavaScript branches of `ast_parse`: upstream dispatches to -`bfcl_eval.model_handler.parser.java_parser.parse_java_function_call` and the -`js_parser` twin, neither of which is vendored here. Their bodies are kept -verbatim (unreachable rather than deleted, per the "removing a helper is a -deviation too" rule), because sieval never calls `ast_parse` with -`language="Java"` or `"JavaScript"` -- the category's language only ever -selects the type converter inside `ast_checker`, not the parser's language -(see `_tables.py`'s `LANGUAGE_BY_CATEGORY` docstring). Calling `ast_parse` -with either of those two language values would raise `NameError` on the -undefined parser call, exactly as it would if the names were simply typos; -that failure mode is intentional rather than silent, since it is unreachable -from the intended call path. +The Java/JavaScript branches of `ast_parse` dispatch to those two functions: +a `java`/`javascript`-category sample's model output is Java/JavaScript +source text, not Python call syntax, so `ast_parse` parses it with the +matching tree-sitter grammar rather than `ast.parse`. """ import ast +from .source_parser.java_parser import parse_java_function_call +from .source_parser.js_parser import parse_javascript_function_call + def ast_parse(input_str: str, language: str="Python") -> list[dict]: if language == "Python": diff --git a/sieval/community/bfcl_v3/source_parser/__init__.py b/sieval/community/bfcl_v3/source_parser/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sieval/community/bfcl_v3/source_parser/java_parser.py b/sieval/community/bfcl_v3/source_parser/java_parser.py new file mode 100644 index 00000000..37148bab --- /dev/null +++ b/sieval/community/bfcl_v3/source_parser/java_parser.py @@ -0,0 +1,151 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/model_handler/parser/java_parser.py +Upstream blob: f9081a958f1d173427064c3505d25ecbebf8aa70 +License: Apache-2.0 + +Only deviation: none beyond this docstring. The module imports only +`tree_sitter` and `tree_sitter_java` -- neither is `bfcl_eval.*` -- so there +was no import to rewrite. + +Held in `source_parser/`, not upstream's `parser/`: this package's +`parser.py` (the vendored `ast_parse`) already occupies that name, so +upstream's containing directory was renamed to avoid the collision. Nothing +inside the file changed for the rename. +""" + +from tree_sitter import Language, Parser +import tree_sitter_java + +JAVA_LANGUAGE = Language(tree_sitter_java.language(), "java") + +parser = Parser() +parser.set_language(JAVA_LANGUAGE) + + +def parse_java_function_call(source_code): + tree = parser.parse(bytes(source_code, "utf8")) + root_node = tree.root_node + sexp_result = root_node.sexp() + + if "ERROR" in sexp_result: + raise SyntaxError("Error parsing java the source code.") + + def get_text(node): + """Returns the text represented by the node.""" + return source_code[node.start_byte : node.end_byte] + + def traverse_node(node, nested=False): + if node.type == "string_literal": + if nested: + return get_text(node) + # Strip surrounding quotes from string literals + return get_text(node)[1:-1] + elif node.type == "character_literal": + if nested: + return get_text(node) + # Strip surrounding single quotes from character literals + return get_text(node)[1:-1] + """Traverse the node to collect texts for complex structures.""" + if node.type in [ + "identifier", + "class_literal", + "type_identifier", + "method_invocation", + ]: + return get_text(node) + elif node.type == "array_creation_expression": + # Handle array creation expression specifically + type_node = node.child_by_field_name("type") + value_node = node.child_by_field_name("value") + type_text = traverse_node(type_node, True) + value_text = traverse_node(value_node, True) + return f"new {type_text}[]{value_text}" + elif node.type == "object_creation_expression": + # Handle object creation expression specifically + type_node = node.child_by_field_name("type") + arguments_node = node.child_by_field_name("arguments") + type_text = traverse_node(type_node, True) + if arguments_node: + # Process each argument carefully, avoiding unnecessary punctuation + argument_texts = [] + for child in arguments_node.children: + if child.type not in [ + ",", + "(", + ")", + ]: # Exclude commas and parentheses + argument_text = traverse_node(child, True) + argument_texts.append(argument_text) + arguments_text = ", ".join(argument_texts) + return f"new {type_text}({arguments_text})" + else: + return f"new {type_text}()" + elif node.type == "set": + # Handling sets specifically + items = [ + traverse_node(n, True) + for n in node.children + if n.type not in [",", "set"] + ] + return "{" + ", ".join(items) + "}" + + elif node.child_count > 0: + return "".join(traverse_node(child, True) for child in node.children) + else: + return get_text(node) + + def extract_arguments(args_node): + arguments = {} + for child in args_node.children: + if child.type == "assignment_expression": + # For named parameters + name_node, value_node = child.children[0], child.children[2] + name = get_text(name_node) + value = traverse_node(value_node) + if name in arguments: + if not isinstance(arguments[name], list): + arguments[name] = [arguments[name]] + arguments[name].append(value) + else: + arguments[name] = value + # arguments.append({'name': name, 'value': value}) + elif child.type in ["identifier", "class_literal", "set"]: + # For unnamed parameters and handling sets + value = traverse_node(child) + if None in arguments: + if not isinstance(arguments[None], list): + arguments[None] = [arguments[None]] + arguments[None].append(value) + else: + arguments[None] = value + return arguments + + def traverse(node): + if node.type == "method_invocation": + # Extract the function name and its arguments + method_name = get_text(node.child_by_field_name("name")) + class_name_node = node.child_by_field_name("object") + if class_name_node: + class_name = get_text(class_name_node) + function_name = f"{class_name}.{method_name}" + else: + function_name = method_name + arguments_node = node.child_by_field_name("arguments") + if arguments_node: + arguments = extract_arguments(arguments_node) + for key, value in arguments.items(): + if isinstance(value, list): + raise Exception( + "Error: Multiple arguments with the same name are not supported." + ) + return [{function_name: arguments}] + + else: + for child in node.children: + result = traverse(child) + if result: + return result + + result = traverse(root_node) + return result if result else {} diff --git a/sieval/community/bfcl_v3/source_parser/js_parser.py b/sieval/community/bfcl_v3/source_parser/js_parser.py new file mode 100644 index 00000000..0962ef2b --- /dev/null +++ b/sieval/community/bfcl_v3/source_parser/js_parser.py @@ -0,0 +1,79 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/model_handler/parser/js_parser.py +Upstream blob: 162510cf681283c95458904177776a41271d014a +License: Apache-2.0 + +Only deviation: none beyond this docstring. The module imports only +`tree_sitter` and `tree_sitter_javascript` -- neither is `bfcl_eval.*` -- so +there was no import to rewrite. + +Held in `source_parser/`, not upstream's `parser/`: this package's +`parser.py` (the vendored `ast_parse`) already occupies that name, so +upstream's containing directory was renamed to avoid the collision. Nothing +inside the file changed for the rename. +""" + +from tree_sitter import Language, Parser +import tree_sitter_javascript + +JS_LANGUAGE = Language(tree_sitter_javascript.language(), "javascript") + +parser = Parser() +parser.set_language(JS_LANGUAGE) + + +def parse_javascript_function_call(source_code): + # Parse the source code + tree = parser.parse(bytes(source_code, "utf8")) + root_node = tree.root_node + sexp_result = root_node.sexp() + if "ERROR" in sexp_result: + raise SyntaxError("Error js parsing the source code.") + + # Function to recursively extract argument details + def extract_arguments(node): + args = {} + for child in node.children: + if child.type == "assignment_expression": + # Extract left (name) and right (value) parts of the assignment + name = child.children[0].text.decode("utf-8") + value = child.children[2].text.decode("utf-8") + if (value.startswith('"') and value.endswith('"')) or ( + value.startswith("'") and value.endswith("'") + ): + value = value[1:-1] # Trim the quotation marks + if name in args: + if not isinstance(args[name], list): + args[name] = [args[name]] + args[name].append(value) + else: + args[name] = value + + elif child.type == "identifier" or child.type == "true": + # Handle non-named arguments and boolean values + value = child.text.decode("utf-8") + if None in args: + if not isinstance(args[None], list): + args[None] = [args[None]] + args[None].append(value) + else: + args[None] = value + return args + + # Find the function call and extract its name and arguments + if root_node.type == "program": + for child in root_node.children: + if child.type == "expression_statement": + for sub_child in child.children: + if sub_child.type == "call_expression": + function_name = sub_child.children[0].text.decode("utf8") + arguments_node = sub_child.children[1] + parameters = extract_arguments(arguments_node) + for key, value in parameters.items(): + if isinstance(value, list): + raise Exception( + "Error: Multiple arguments with the same name are not supported." + ) + result = [{function_name: parameters}] + return result diff --git a/tests/unit/community/bfcl_v3/test_identity.py b/tests/unit/community/bfcl_v3/test_identity.py index e86e4b11..3886bf3f 100644 --- a/tests/unit/community/bfcl_v3/test_identity.py +++ b/tests/unit/community/bfcl_v3/test_identity.py @@ -21,7 +21,13 @@ "type_convertor/js_type_converter.py": ( "dfb0373b58b51c45e63346e1a893c055168dd6ae39e27307436fefa45981e6cc" ), - "parser.py": ("4bba883c1f537a3ae0ad0c85260974861d056ffe25ab50a90bd07d7bb8dec2a3"), + "source_parser/java_parser.py": ( + "024dfc247dbb7890c4ed969ab48a9fab3212b85bdb74402528a5ce7b8abab5fe" + ), + "source_parser/js_parser.py": ( + "3a096d914cc54514009e11915e8d4cc4c35af26b5a1c9f21ced640cd896dc737" + ), + "parser.py": ("ae3863501c133eea661120d455cce3c27fe8709e01532cba639e45693f8b8c1b"), "tool_convert.py": ( "11b758572a09fca738479243e4cc3005387def672786b3231c868e6d73ad386e" ), diff --git a/tests/unit/community/bfcl_v3/test_source_parsers.py b/tests/unit/community/bfcl_v3/test_source_parsers.py new file mode 100644 index 00000000..677dcc6a --- /dev/null +++ b/tests/unit/community/bfcl_v3/test_source_parsers.py @@ -0,0 +1,40 @@ +"""`ast_parse` must actually dispatch to the vendored source-text parsers. + +The identity test proves the vendored bytes have not drifted; it says nothing +about whether the Java/JavaScript branches run. These assert real behaviour +through `ast_parse`'s public surface, covering all three branches (Python, +Java, JavaScript) plus the unsupported-language error path. + +Expected values were derived by running the parsers, not assumed: both +tree-sitter parsers return every argument value as a string (`"true"`, not +`True`), which is upstream's behaviour, not a porting artifact. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import pytest + +from sieval.community.bfcl_v3 import ast_parse + + +def test_java_source_is_parsed_into_a_call_dict(): + assert ast_parse( + '[TaskManager.removeTask(task="Meeting", urgent=true)]', "Java" + ) == [{"TaskManager.removeTask": {"task": "Meeting", "urgent": "true"}}] + + +def test_javascript_source_is_parsed_into_a_call_dict(): + assert ast_parse('[doTask(name="Meeting", urgent=true)]', "JavaScript") == [ + {"doTask": {"name": "Meeting", "urgent": "true"}} + ] + + +def test_python_source_is_still_parsed_by_ast(): + assert ast_parse('[doTask(name="Meeting", urgent=True)]', "Python") == [ + {"doTask": {"name": "Meeting", "urgent": True}} + ] + + +def test_unknown_language_still_raises_not_implemented(): + with pytest.raises(NotImplementedError): + ast_parse('[doTask(name="Meeting")]', "Ruby") From 49019121fc3de85c8d8112f75485f96a61f00b17 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 18:13:10 +0800 Subject: [PATCH 05/28] feat(datasets): add BFCL v3 single-turn non-live and live datasets One pinned HF snapshot, two registered datasets split on the group boundary upstream scores separately. Three categories ship no gold, because the correct outcome there is about whether a call is produced, not which one. Gold joins on the universal index -- the `` of `_[--]` -- because that is upstream's key: its runner sorts both files on it, discarding the sub-indices, then pairs them positionally. One row depends on it. The prompt file's `live_multiple_1052-79-0` is graded against the gold file's `live_multiple_1052-279-0`; joining on the whole `id` would drop it and score 1052 of live_multiple's 1053 rows, publishing a column upstream never did. The key's uniqueness is asserted on every file that is joined, which is what makes an index join and upstream's positional join the same pairing -- and is stricter than upstream, which only checks the two files are the same length. It is not asserted on the goldless three, which are never joined: `live_relevance` ships one row twice, 18 rows over 17 distinct ids, and upstream scores all 18. Per-category row counts are asserted on load. The revision pin already stops a silent re-upload -- the assertion is what turns bumping the pin into a visible failure instead of a quietly rescored column. Co-Authored-By: Claude Opus 5 --- sieval/datasets/__init__.pyi | 12 ++ sieval/datasets/_bfcl_v3.py | 138 +++++++++++++++++++++ sieval/datasets/bfcl_v3_live.py | 65 ++++++++++ sieval/datasets/bfcl_v3_non_live.py | 64 ++++++++++ sieval/meta/index.json | 42 +++++++ tests/unit/datasets/test_bfcl_v3.py | 183 ++++++++++++++++++++++++++++ 6 files changed, 504 insertions(+) create mode 100644 sieval/datasets/_bfcl_v3.py create mode 100644 sieval/datasets/bfcl_v3_live.py create mode 100644 sieval/datasets/bfcl_v3_non_live.py create mode 100644 tests/unit/datasets/test_bfcl_v3.py diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index b5c5fb8c..a55ab7b0 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -45,6 +45,14 @@ from .arc_easy import ( ARCEasyDataset, ARCEasyDatasetSample, ) +from .bfcl_v3_live import ( + BfclV3LiveDataset, + BfclV3LiveDatasetSample, +) +from .bfcl_v3_non_live import ( + BfclV3NonLiveDataset, + BfclV3NonLiveDatasetSample, +) from .browsecomp import ( BrowseCompDataset, BrowseCompDatasetSample, @@ -275,6 +283,10 @@ __all__ = [ "ApexShortlist2025DatasetSample", "BRUMO2025Dataset", "BRUMO2025DatasetSample", + "BfclV3LiveDataset", + "BfclV3LiveDatasetSample", + "BfclV3NonLiveDataset", + "BfclV3NonLiveDatasetSample", "BrowseCompDataset", "BrowseCompDatasetSample", "CEvalDataset", diff --git a/sieval/datasets/_bfcl_v3.py b/sieval/datasets/_bfcl_v3.py new file mode 100644 index 00000000..616dba83 --- /dev/null +++ b/sieval/datasets/_bfcl_v3.py @@ -0,0 +1,138 @@ +"""Shared loader for the BFCL v3 single-turn category files. + +One HF repo holds every category as its own JSONL file, with ground truth in a +parallel `possible_answer/` file. Three categories ship no gold at all -- +`irrelevance`, `live_irrelevance` and `live_relevance` are scored on whether a +call was produced, not on which call it was. + +The join key is the **universal index** -- the `` of +`_[--]` -- not the whole `id` string. That is +upstream's key: its runner sorts both files on `(category, universal_index)`, +discarding the sub-indices, then pairs them positionally. The distinction is +load-bearing on exactly one row at the pinned revision: the prompt file's +`live_multiple_1052-79-0` is graded against the gold file's +`live_multiple_1052-279-0`. One of the two sub-indices is an upstream typo, but +because it never enters upstream's key, upstream grades the row and cannot see +the disagreement. Joining on the whole `id` would instead score 1052 of +`live_multiple`'s 1053 rows and report a column upstream never published. + +The key is unique within every file that is joined, which is what makes an index +join and upstream's positional join the same pairing; the loader asserts that +rather than assuming it, and so is strictly stricter than upstream, which only +checks that the two files are the same length. It is deliberately not asserted +for the goldless three, which are never joined -- `live_relevance` ships the +same row twice and is 18 rows over 17 distinct ids. + +Row counts are asserted per category. The revision pin already prevents a silent +re-upload; what it cannot prevent is someone bumping the pin, so a count that +moves fails here rather than quietly rescoring a leaderboard column. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json +from collections.abc import Mapping +from pathlib import Path + +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.community.bfcl_v3 import GOLDLESS_CATEGORIES, LANGUAGE_BY_CATEGORY + +#: Pinned snapshot. Verified row-for-row against the harness's own +#: `bfcl_eval/data/` at gorilla v1.3. +BFCL_V3_REVISION = "61fc0608cfd831fcfbbaa676ebdfef0ed963eeda" + + +def _read_jsonl(path: Path) -> list[dict]: + with path.open(encoding="utf-8") as handle: + return [json.loads(line) for line in handle if line.strip()] + + +def _universal_index(row_id: str) -> int: + """The join key of one row: the index in `_[--]`. + + Mirrors the index that upstream's own sort key extracts -- it splits on the + last underscore and keeps only what precedes the first dash, so the two + sub-indices the live categories carry are discarded here exactly as they + are there. + """ + index = row_id.rsplit("_", 1)[-1] + return int(index.split("-")[0]) + + +def _index_rows(rows: list[dict], category: str, what: str) -> dict[int, dict]: + by_index = {_universal_index(row["id"]): row for row in rows} + if len(by_index) != len(rows): + raise ValueError( + f"BFCL v3 category {category!r} has {len(rows) - len(by_index)} " + f"{what} rows sharing a universal index. The index is the join key " + "and upstream's runner pairs the two files positionally under it, " + "so a repeat makes the pairing ambiguous rather than merely odd." + ) + return by_index + + +def load_categories(name_or_path: str, counts: Mapping[str, int]) -> HFDatasetDict: + root = Path(name_or_path) + rows: list[dict] = [] + for category, expected in counts.items(): + prompts = _read_jsonl(root / f"BFCL_v3_{category}.json") + if len(prompts) != expected: + raise ValueError( + f"BFCL v3 category {category!r} has {len(prompts)} rows at the " + f"pinned revision, expected {expected}. A count that moved means " + "the pin was bumped; re-verify the category against upstream and " + "update the table deliberately, do not relax this check." + ) + + gold_by_index: dict[int, str] = {} + if category not in GOLDLESS_CATEGORIES: + # Only a category that is actually joined needs a unique key. The + # goldless three are not joined, and one of them must not be held to + # this: `live_relevance` ships `live_relevance_3-3-0` twice, so it + # has 18 rows and 17 distinct ids. Upstream scores all 18 and so do + # we -- deduplicating here would publish a denominator upstream + # never used. + prompt_by_index = _index_rows(prompts, category, "prompt") + answers = _index_rows( + _read_jsonl(root / "possible_answer" / f"BFCL_v3_{category}.json"), + category, + "possible_answer", + ) + gold_by_index = { + index: json.dumps(entry["ground_truth"], ensure_ascii=False) + for index, entry in answers.items() + } + missing = set(prompt_by_index) - set(gold_by_index) + if missing: + examples = [prompt_by_index[i]["id"] for i in sorted(missing)[:3]] + raise ValueError( + f"BFCL v3 category {category!r} has {len(missing)} prompt rows " + f"with no possible_answer entry (e.g. {examples}). The join key " + "is the universal index; a miss means the two files disagree." + ) + + for row in prompts: + turns = row["question"] + # Every phase-1 category is single-turn. Asserting beats taking [0]: + # a multi-turn file loaded here would silently score only its first + # turn and look like a very poor model. + if len(turns) != 1: + raise ValueError( + f"BFCL v3 row {row['id']!r} in {category!r} has {len(turns)} " + "turns; the single-turn datasets accept exactly one. " + "Multi-turn categories are phase 2 and must not be loaded here." + ) + rows.append( + { + "id": row["id"], + "category": category, + "language": LANGUAGE_BY_CATEGORY.get(category, "Python"), + "question": turns[0], + "function": json.dumps(row["function"], ensure_ascii=False), + "ground_truth": gold_by_index.get(_universal_index(row["id"])), + } + ) + + return HFDatasetDict({"test": HFDataset.from_list(rows)}) diff --git a/sieval/datasets/bfcl_v3_live.py b/sieval/datasets/bfcl_v3_live.py new file mode 100644 index 00000000..da867947 --- /dev/null +++ b/sieval/datasets/bfcl_v3_live.py @@ -0,0 +1,65 @@ +"""BFCL v3 live categories -- the user-contributed half of the benchmark. + +Six categories, 2251 rows: four AST categories built from real user queries, +plus irrelevance and relevance detection sets with no gold. + +`function` and `ground_truth` are json strings because pyarrow cannot type +their heterogeneous nesting -- the same reason `t_eval.py` stores its +`ground_truth` that way. + +References: + +* Blog: +* Harness: +* Dataset: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict + +from sieval.community.bfcl_v3 import LIVE_COUNTS +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) + +from ._bfcl_v3 import BFCL_V3_REVISION, load_categories + + +class BfclV3LiveDatasetSample(TypedDict): + """One BFCL v3 live row. + + `ground_truth` is None for `live_irrelevance` and `live_relevance`, which + have no gold: the correct outcome there is about whether a call is + produced, not about which call it is. + """ + + id: str + category: str + language: str + question: list[dict] + function: str + ground_truth: str | None + + +@sieval_dataset( + name="bfcl_v3_live", + display_name="BFCL v3 (live)", + description=( + "Berkeley Function Calling Leaderboard v3 live: " + "user-contributed AST, irrelevance, relevance." + ), + source=f"hf:gorilla-llm/Berkeley-Function-Calling-Leaderboard@{BFCL_V3_REVISION}", + categories=(Category(Level1Category.AGENT, "ToolUseSimple"),), + tags=("english", "function-calling"), + license="Apache-2.0", +) +class BfclV3LiveDataset(Dataset[BfclV3LiveDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + return load_categories(name_or_path, LIVE_COUNTS) diff --git a/sieval/datasets/bfcl_v3_non_live.py b/sieval/datasets/bfcl_v3_non_live.py new file mode 100644 index 00000000..bab6db31 --- /dev/null +++ b/sieval/datasets/bfcl_v3_non_live.py @@ -0,0 +1,64 @@ +"""BFCL v3 non-live categories -- the hand-written half of the benchmark. + +Seven categories, 1390 rows: four Python AST categories, Java and JavaScript +simple-call categories, and an irrelevance-detection set with no gold. + +`function` and `ground_truth` are json strings because pyarrow cannot type +their heterogeneous nesting -- the same reason `t_eval.py` stores its +`ground_truth` that way. + +References: + +* Blog: +* Harness: +* Dataset: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict + +from sieval.community.bfcl_v3 import NON_LIVE_COUNTS +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) + +from ._bfcl_v3 import BFCL_V3_REVISION, load_categories + + +class BfclV3NonLiveDatasetSample(TypedDict): + """One BFCL v3 non-live row. + + `ground_truth` is None for `irrelevance`, which has no gold: the correct + outcome there is that no call is produced at all. + """ + + id: str + category: str + language: str + question: list[dict] + function: str + ground_truth: str | None + + +@sieval_dataset( + name="bfcl_v3_non_live", + display_name="BFCL v3 (non-live)", + description=( + "Berkeley Function Calling Leaderboard v3 non-live: " + "Python/Java/JS AST plus irrelevance." + ), + source=f"hf:gorilla-llm/Berkeley-Function-Calling-Leaderboard@{BFCL_V3_REVISION}", + categories=(Category(Level1Category.AGENT, "ToolUseSimple"),), + tags=("english", "function-calling"), + license="Apache-2.0", +) +class BfclV3NonLiveDataset(Dataset[BfclV3NonLiveDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + return load_categories(name_or_path, NON_LIVE_COUNTS) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 63ea41e8..ef88e888 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -284,6 +284,48 @@ "license": "cc-by-sa-4.0", "checksums": {} }, + { + "name": "bfcl_v3_live", + "display_name": "BFCL v3 (live)", + "description": "Berkeley Function Calling Leaderboard v3 live: user-contributed AST, irrelevance, relevance.", + "source": [ + "hf:gorilla-llm/Berkeley-Function-Calling-Leaderboard@61fc0608cfd831fcfbbaa676ebdfef0ed963eeda" + ], + "categories": [ + { + "level1": "Agent", + "level2": "ToolUseSimple" + } + ], + "tags": [ + "english", + "function-calling" + ], + "deps_group": null, + "license": "Apache-2.0", + "checksums": {} + }, + { + "name": "bfcl_v3_non_live", + "display_name": "BFCL v3 (non-live)", + "description": "Berkeley Function Calling Leaderboard v3 non-live: Python/Java/JS AST plus irrelevance.", + "source": [ + "hf:gorilla-llm/Berkeley-Function-Calling-Leaderboard@61fc0608cfd831fcfbbaa676ebdfef0ed963eeda" + ], + "categories": [ + { + "level1": "Agent", + "level2": "ToolUseSimple" + } + ], + "tags": [ + "english", + "function-calling" + ], + "deps_group": null, + "license": "Apache-2.0", + "checksums": {} + }, { "name": "browsecomp", "display_name": "BrowseComp", diff --git a/tests/unit/datasets/test_bfcl_v3.py b/tests/unit/datasets/test_bfcl_v3.py new file mode 100644 index 00000000..ffbcaffd --- /dev/null +++ b/tests/unit/datasets/test_bfcl_v3.py @@ -0,0 +1,183 @@ +import json + +import pytest + +from sieval.community.bfcl_v3 import GOLDLESS_CATEGORIES +from sieval.datasets._bfcl_v3 import load_categories + + +def _write(root, category, rows, gold=None): + (root / f"BFCL_v3_{category}.json").write_text( + "\n".join(json.dumps(r) for r in rows), encoding="utf-8" + ) + if gold is not None: + answers = root / "possible_answer" + answers.mkdir(exist_ok=True) + (answers / f"BFCL_v3_{category}.json").write_text( + "\n".join(json.dumps(g) for g in gold), encoding="utf-8" + ) + + +def test_gold_is_joined_onto_its_prompt_row(tmp_path): + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [[{"role": "user", "content": "hi"}]], + "function": [{"name": "f"}], + } + ], + gold=[{"id": "simple_0", "ground_truth": [{"f": {"a": [1]}}]}], + ) + row = load_categories(str(tmp_path), {"simple": 1})["test"][0] + assert row["id"] == "simple_0" + assert row["language"] == "Python" + assert json.loads(row["ground_truth"]) == [{"f": {"a": [1]}}] + assert row["question"] == [{"role": "user", "content": "hi"}] + + +def test_a_goldless_category_needs_no_answer_file(tmp_path): + assert "irrelevance" in GOLDLESS_CATEGORIES + _write( + tmp_path, + "irrelevance", + [ + { + "id": "irrelevance_0", + "question": [[{"role": "user", "content": "hi"}]], + "function": [{"name": "f"}], + } + ], + ) + row = load_categories(str(tmp_path), {"irrelevance": 1})["test"][0] + assert row["ground_truth"] is None + + +def test_java_and_javascript_carry_their_language(tmp_path): + for category, language in (("java", "Java"), ("javascript", "JavaScript")): + _write( + tmp_path, + category, + [ + { + "id": f"{category}_0", + "question": [[{"role": "user", "content": "x"}]], + "function": [{"name": "f"}], + } + ], + gold=[{"id": f"{category}_0", "ground_truth": [{"f": {}}]}], + ) + row = load_categories(str(tmp_path), {category: 1})["test"][0] + assert row["language"] == language + + +def test_a_moved_row_count_raises(tmp_path): + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [[{"role": "user", "content": "hi"}]], + "function": [], + } + ], + gold=[{"id": "simple_0", "ground_truth": []}], + ) + with pytest.raises(ValueError, match="expected 400"): + load_categories(str(tmp_path), {"simple": 400}) + + +def test_a_multi_turn_row_is_refused(tmp_path): + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [ + [{"role": "user", "content": "a"}], + [{"role": "user", "content": "b"}], + ], + "function": [], + } + ], + gold=[{"id": "simple_0", "ground_truth": []}], + ) + with pytest.raises(ValueError, match="turns"): + load_categories(str(tmp_path), {"simple": 1}) + + +def test_gold_joins_on_the_universal_index_not_the_whole_id(tmp_path): + """Upstream sorts both files on the universal index and pairs them + positionally, so a row whose sub-indices disagree between the two files is + still graded. `live_multiple_1052-79-0` / `live_multiple_1052-279-0` is that + row at the pinned revision; joining on the whole id would drop it. + """ + _write( + tmp_path, + "live_multiple", + [ + { + "id": "live_multiple_1052-79-0", + "question": [[{"role": "user", "content": "volume 50"}]], + "function": [{"name": "set_volume"}], + } + ], + gold=[ + { + "id": "live_multiple_1052-279-0", + "ground_truth": [{"set_volume": {"volume": [50]}}], + } + ], + ) + row = load_categories(str(tmp_path), {"live_multiple": 1})["test"][0] + assert row["id"] == "live_multiple_1052-79-0" + assert json.loads(row["ground_truth"]) == [{"set_volume": {"volume": [50]}}] + + +def test_a_repeated_universal_index_raises(tmp_path): + """Two rows sharing the join key make the pairing ambiguous. Upstream cannot + see this -- it only checks that the two files are the same length. + """ + _write( + tmp_path, + "live_multiple", + [ + { + "id": "live_multiple_0-1-0", + "question": [[{"role": "user", "content": "a"}]], + "function": [], + }, + { + "id": "live_multiple_0-2-0", + "question": [[{"role": "user", "content": "b"}]], + "function": [], + }, + ], + gold=[ + {"id": "live_multiple_0-1-0", "ground_truth": []}, + {"id": "live_multiple_0-2-0", "ground_truth": []}, + ], + ) + with pytest.raises(ValueError, match="sharing a universal index"): + load_categories(str(tmp_path), {"live_multiple": 2}) + + +def test_a_gold_miss_raises(tmp_path): + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [[{"role": "user", "content": "hi"}]], + "function": [], + } + ], + gold=[{"id": "simple_99", "ground_truth": []}], + ) + with pytest.raises(ValueError, match="no possible_answer"): + load_categories(str(tmp_path), {"simple": 1}) From c4f5189e7bd13eeb16a25581d1cd23b6fda1dd44 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 18:25:28 +0800 Subject: [PATCH 06/28] feat(core): carry native tool calls through to the legacy model output `TaskRequirements.function_tools` lets a task declare it needs the tools API, but the reply's structured calls never reached the task: the IR `Response` carries `tool_calls` and `response_to_model_output` rebuilt `ModelOutput` field-by-field without them, so the declaration was unsatisfiable by construction. No task read them, so nothing caught it. Forward the tuple and default the field to None. Additive: every existing construction site omits it and keeps its current behaviour. Co-Authored-By: Claude Opus 5 --- sieval/core/models/_legacy_bridge.py | 6 +++++ tests/unit/core/models/test_model.py | 35 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/sieval/core/models/_legacy_bridge.py b/sieval/core/models/_legacy_bridge.py index d09aba9e..e0d1dd3f 100644 --- a/sieval/core/models/_legacy_bridge.py +++ b/sieval/core/models/_legacy_bridge.py @@ -24,6 +24,7 @@ from .ir import ( CompletionInput, DialectOptions, + FunctionToolCall, ModelInput, ModelProvenance, OpaqueContinuation, @@ -89,6 +90,10 @@ class ModelOutput: texts: list[str] finish_reasons: list[str] | None = None reasoning_texts: list[str] | None = None + #: Structured calls from the native tools API. ``None`` when the reply + #: carried none -- which is not the same as an empty tuple, a reply that + #: used the tools API and deliberately called nothing. + tool_calls: tuple[FunctionToolCall, ...] | None = None logprobs_tokens: list[str] | None = None logprobs: list[float | None] | None = None top_logprobs: list[dict[str, float]] | None = None @@ -471,6 +476,7 @@ def response_to_model_output(model_meta: ModelMeta, response: Response) -> Model else None ), reasoning_texts=reasoning_texts, + tool_calls=response.tool_calls, logprobs_tokens=logprobs_tokens, logprobs=logprobs, top_logprobs=top_logprobs, diff --git a/tests/unit/core/models/test_model.py b/tests/unit/core/models/test_model.py index 0c70a961..9c6404e7 100644 --- a/tests/unit/core/models/test_model.py +++ b/tests/unit/core/models/test_model.py @@ -23,6 +23,7 @@ ConnectionIdentity, ConnectionPool, DialectOptions, + FunctionToolCall, GenModel, InputScoringResult, Model, @@ -1587,6 +1588,40 @@ def test_empty_reasoning_stays_none(self): ) assert out.reasoning_texts is None + def test_tool_calls_reach_the_task_in_order(self): + """A task that declares ``function_tools`` reads the calls back here. + + The bridge rebuilds the record field by field, so a channel it does not + name is dropped without a failure anywhere. + """ + out = self._bridge( + Response( + texts=("",), + tool_calls=( + FunctionToolCall("call_1", "get_weather", '{"city":"Paris"}'), + FunctionToolCall("call_2", "get_time", '{"tz":"UTC"}'), + ), + ) + ) + assert out.tool_calls == ( + FunctionToolCall("call_1", "get_weather", '{"city":"Paris"}'), + FunctionToolCall("call_2", "get_time", '{"tz":"UTC"}'), + ) + + def test_tool_calls_absent_stays_none(self): + out = self._bridge(Response(texts=("t",), tool_calls=None)) + assert out.tool_calls is None + + def test_tool_calls_present_but_empty_survives(self): + """Calling nothing is a reply about the tools; absence is not.""" + out = self._bridge(Response(texts=("t",), tool_calls=())) + assert out.tool_calls == () + + def test_tool_calls_default_to_none_when_omitted(self): + """Every existing construction site omits the field.""" + model = GenModel(model="m", api_key="k") + assert ModelOutput(model=model.meta(), texts=["t"]).tool_calls is None + def test_caller_mapping_is_not_mutated_and_not_shared(self): """Provenance is attached to a copy. From 7c2c640d9dd78451d0c59da03544d812121a0879 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 18:49:26 +0800 Subject: [PATCH 07/28] fix(datasets): stop the bfcl-v3 extra from taking out the dataset registry `sieval/community/bfcl_v3/__init__.py` eagerly imported `.parser`, the one module in that package reaching tree-sitter, which lives in the optional `bfcl-v3` extra. Both BFCL v3 dataset modules import the package for two pure-data tables, and `import_all_datasets()` imports every dataset submodule in a bare loop with no per-module `except` -- so on any install without the extra, `sieval dataset list` and `sieval dataset download ` died with `ModuleNotFoundError: tree_sitter`, for every dataset in the repo. `ast_parse` now resolves through a module `__getattr__`; of the eight modules in the package only `parser.py` reaches tree-sitter, so deferring it unblocks the rest. Nothing at the call sites changes. CI cannot see this: `INSTALL_GROUPS` carries `-G bfcl-v3` on purpose, to gate the Java/JavaScript decode path. The regression test therefore runs a fresh interpreter with `tree_sitter*` blocked at `sys.meta_path` -- an in-process check would pass with the eager import restored. Co-Authored-By: Claude Opus 5 --- sieval/community/bfcl_v3/__init__.py | 16 ++++++- tests/unit/datasets/test_bfcl_v3.py | 63 ++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/sieval/community/bfcl_v3/__init__.py b/sieval/community/bfcl_v3/__init__.py index fdb39609..d5851658 100644 --- a/sieval/community/bfcl_v3/__init__.py +++ b/sieval/community/bfcl_v3/__init__.py @@ -42,10 +42,10 @@ from .aggregate import calculate_unweighted_accuracy, calculate_weighted_accuracy from .ast_checker import ast_checker from .output_checks import is_empty_output, is_function_calling_format_output -from .parser import ast_parse from .prompts import DEFAULT_SYSTEM_PROMPT from .tool_convert import convert_to_tool + __all__ = [ "ast_checker", "ast_parse", @@ -62,3 +62,17 @@ "CALL_EXPECTED", "LANGUAGE_BY_CATEGORY", ] + + +def __getattr__(name: str) -> object: + """`ast_parse` is deferred because `parser.py` is the only module here that + reaches tree-sitter, which lives in the optional `bfcl-v3` extra. Importing + it eagerly makes that extra a hard requirement for importing this package at + all -- and `sieval/datasets/` imports it for two pure-data tables, so the + whole dataset registry died on a base install. + """ + if name == "ast_parse": + from .parser import ast_parse + + return ast_parse + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/tests/unit/datasets/test_bfcl_v3.py b/tests/unit/datasets/test_bfcl_v3.py index ffbcaffd..34ab413c 100644 --- a/tests/unit/datasets/test_bfcl_v3.py +++ b/tests/unit/datasets/test_bfcl_v3.py @@ -1,10 +1,16 @@ import json +import os +import subprocess +import sys +from pathlib import Path import pytest from sieval.community.bfcl_v3 import GOLDLESS_CATEGORIES from sieval.datasets._bfcl_v3 import load_categories +_ROOT = Path(__file__).parents[3] + def _write(root, category, rows, gold=None): (root / f"BFCL_v3_{category}.json").write_text( @@ -166,6 +172,63 @@ def test_a_repeated_universal_index_raises(tmp_path): load_categories(str(tmp_path), {"live_multiple": 2}) +def test_the_dataset_registry_loads_without_the_bfcl_v3_extra(): + """Both loaders import `sieval.community.bfcl_v3` for two pure-data tables, + and `import_all_datasets()` imports every dataset module in a bare loop with + no per-module `except`. So anything that package pulls in at import time is a + hard requirement for *every* dataset in the repo -- `sieval dataset list` + included -- and tree-sitter, which only the vendored `parser.py` needs, is in + the optional `bfcl-v3` extra. + + Out of process with the extra blocked at `sys.meta_path`: this venv has + tree-sitter installed (CI's `INSTALL_GROUPS` carries `-G bfcl-v3` so the + Java/JavaScript decode path is exercised), so an in-process check would pass + with the eager import restored. + """ + child = f""" +import sys + +class _NoTreeSitter: + def find_spec(self, name, path=None, target=None): + if name == "tree_sitter" or name.startswith("tree_sitter."): + raise ModuleNotFoundError("No module named " + name) + return None + +sys.meta_path.insert(0, _NoTreeSitter()) + +import sieval + +# The editable install may name a different checkout than the tree this test +# was loaded from; without this the child would probe that one instead. +assert sieval.__file__.startswith({str(_ROOT)!r}), sieval.__file__ + +from sieval.core.datasets.meta import import_all_datasets + +import_all_datasets() +assert "sieval.datasets.bfcl_v3_non_live" in sys.modules +assert "sieval.datasets.bfcl_v3_live" in sys.modules +""" + inherited = os.environ.get("PYTHONPATH") + env = { + **os.environ, + "PYTHONPATH": os.pathsep.join( + [str(_ROOT), *([inherited] if inherited else [])] + ), + } + completed = subprocess.run( + [sys.executable, "-c", child], + capture_output=True, + text=True, + timeout=300, + check=False, + env=env, + ) + assert completed.returncode == 0, ( + "the dataset registry must import without the optional `bfcl-v3` " + f"extra:\n{completed.stderr}" + ) + + def test_a_gold_miss_raises(tmp_path): _write( tmp_path, From 0b324c2d00386b17860c1645d093df50835d928f Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 18:53:45 +0800 Subject: [PATCH 08/28] test(datasets): cover the cross-category and per-file BFCL v3 guards Every test loaded exactly one category, so nothing in the file could see a bug that only shows up across categories. Three reverse mutations of the loader left all of them green: hoisting the per-category gold table above the loop (which hands `irrelevance_0` the gold of `simple_0`, since both key on universal index 0), resetting the row accumulator inside the loop (1390 rows become 240), and storing `function` raw instead of as the json string the sample TypedDict declares. One test loading two categories at once, one of them goldless, now kills all three. The uniqueness guards were covered only jointly: the fixture repeated the index in both files, so either `_index_rows` call satisfied the `pytest.raises` and deleting either guard alone stayed green. Split into two tests, each repeating the index in one file, each matching on the message's own file label. The file also had no module docstring, so it could not carry the AI attribution line the repo requires. Co-Authored-By: Claude Opus 5 --- tests/unit/datasets/test_bfcl_v3.py | 107 ++++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 13 deletions(-) diff --git a/tests/unit/datasets/test_bfcl_v3.py b/tests/unit/datasets/test_bfcl_v3.py index 34ab413c..391e6d1f 100644 --- a/tests/unit/datasets/test_bfcl_v3.py +++ b/tests/unit/datasets/test_bfcl_v3.py @@ -1,3 +1,16 @@ +"""The shared BFCL v3 single-turn loader: the index join and its guards. + +What is covered here is the join key (the universal index, not the whole id), +the per-category row counts, the guards that refuse an ambiguous or incomplete +join, and the one thing neither can check -- that a goldless category stays +goldless when it is loaded next to a category that has gold. Also pinned: the +package holding the loader's category tables must import without the optional +`bfcl-v3` extra, since every dataset in the repo is registered through the same +loop. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + import json import os import subprocess @@ -61,6 +74,53 @@ def test_a_goldless_category_needs_no_answer_file(tmp_path): assert row["ground_truth"] is None +def test_a_goldless_category_stays_goldless_beside_one_that_has_gold(tmp_path): + """The gold table is per-category, and the universal indices of two + categories overlap -- `irrelevance_0` and `simple_0` both key on 0. A gold + table built once for the whole call would hand every goldless row its + neighbour's answer, which reads downstream as a real gold rather than as an + error: the sample TypedDict documents `None` as "this row has no gold". + """ + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [[{"role": "user", "content": "set it to 50"}]], + "function": [{"name": "set_volume", "parameters": {}}], + } + ], + gold=[{"id": "simple_0", "ground_truth": [{"set_volume": {"volume": [50]}}]}], + ) + _write( + tmp_path, + "irrelevance", + [ + { + "id": "irrelevance_0", + "question": [[{"role": "user", "content": "how are you"}]], + "function": [{"name": "set_volume", "parameters": {}}], + } + ], + ) + + rows = list(load_categories(str(tmp_path), {"simple": 1, "irrelevance": 1})["test"]) + + # Every category's rows, not just the last one's. + assert [row["id"] for row in rows] == ["simple_0", "irrelevance_0"] + by_id = {row["id"]: row for row in rows} + assert json.loads(by_id["simple_0"]["ground_truth"]) == [ + {"set_volume": {"volume": [50]}} + ] + assert by_id["irrelevance_0"]["ground_truth"] is None + # `function` is the tool schema the whole grading path reads, and is declared + # `str` because pyarrow cannot type its nesting. + for row in rows: + assert isinstance(row["function"], str) + assert json.loads(row["function"]) == [{"name": "set_volume", "parameters": {}}] + + def test_java_and_javascript_carry_their_language(tmp_path): for category, language in (("java", "Java"), ("javascript", "JavaScript")): _write( @@ -144,31 +204,52 @@ def test_gold_joins_on_the_universal_index_not_the_whole_id(tmp_path): assert json.loads(row["ground_truth"]) == [{"set_volume": {"volume": [50]}}] -def test_a_repeated_universal_index_raises(tmp_path): +def _two_rows(ids): + return [ + { + "id": row_id, + "question": [[{"role": "user", "content": row_id}]], + "function": [], + } + for row_id in ids + ] + + +def test_a_repeated_universal_index_in_the_prompt_file_raises(tmp_path): """Two rows sharing the join key make the pairing ambiguous. Upstream cannot see this -- it only checks that the two files are the same length. + + The gold file here is well-formed, so only the prompt-side guard can fire. """ _write( tmp_path, "live_multiple", - [ - { - "id": "live_multiple_0-1-0", - "question": [[{"role": "user", "content": "a"}]], - "function": [], - }, - { - "id": "live_multiple_0-2-0", - "question": [[{"role": "user", "content": "b"}]], - "function": [], - }, + _two_rows(["live_multiple_0-1-0", "live_multiple_0-2-0"]), + gold=[ + {"id": "live_multiple_0-1-0", "ground_truth": []}, + {"id": "live_multiple_1-1-0", "ground_truth": []}, ], + ) + with pytest.raises(ValueError, match="prompt rows sharing a universal index"): + load_categories(str(tmp_path), {"live_multiple": 2}) + + +def test_a_repeated_universal_index_in_the_gold_file_raises(tmp_path): + """The mirror of the test above: the prompt file is well-formed and the + repeat is on the gold side, so only the possible_answer guard can fire. + """ + _write( + tmp_path, + "live_multiple", + _two_rows(["live_multiple_0-1-0", "live_multiple_1-1-0"]), gold=[ {"id": "live_multiple_0-1-0", "ground_truth": []}, {"id": "live_multiple_0-2-0", "ground_truth": []}, ], ) - with pytest.raises(ValueError, match="sharing a universal index"): + with pytest.raises( + ValueError, match="possible_answer rows sharing a universal index" + ): load_categories(str(tmp_path), {"live_multiple": 2}) From 95e9add588d7a72e5e01ce404fefaa3d9e0e0f7f Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 18:56:45 +0800 Subject: [PATCH 09/28] fix(community): keep `ast_parse` statically visible behind the lazy import A module `__getattr__` types *every* attribute of the package as its own return type, so after ba18ad51 `ast_parse` resolved to `object` and its four call sites in `tests/unit/community/bfcl_v3/test_source_parsers.py` each read as `call-non-callable`. An explicitly declared symbol takes precedence over the fallback, so a `TYPE_CHECKING` import restores the real signature. `TYPE_CHECKING` is False at runtime, so the deferral is untouched: the out-of-process regression test, which imports the whole dataset registry with `tree_sitter*` blocked at `sys.meta_path`, still passes. This is `typing.TYPE_CHECKING`, not `from __future__ import annotations`. Co-Authored-By: Claude Opus 5 --- sieval/community/bfcl_v3/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sieval/community/bfcl_v3/__init__.py b/sieval/community/bfcl_v3/__init__.py index d5851658..638c61fa 100644 --- a/sieval/community/bfcl_v3/__init__.py +++ b/sieval/community/bfcl_v3/__init__.py @@ -31,6 +31,8 @@ AI-Generated Code - Claude Opus 5 (Anthropic) """ +from typing import TYPE_CHECKING + from ._model_config import set_underscore_to_dot from ._tables import ( CALL_EXPECTED, @@ -45,6 +47,14 @@ from .prompts import DEFAULT_SYSTEM_PROMPT from .tool_convert import convert_to_tool +if TYPE_CHECKING: + # A module `__getattr__` types *every* attribute of the package as its own + # return type, so `ast_parse` would resolve to `object` and every call site + # would read as calling a non-callable. An explicitly declared symbol takes + # precedence over the fallback; `TYPE_CHECKING` is False at runtime, so the + # import below stays deferred. + from .parser import ast_parse + __all__ = [ "ast_checker", From 445fad1cdbb218eb18fb706fc66d1c1417070fd8 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 19:01:34 +0800 Subject: [PATCH 10/28] fix(datasets): assert the BFCL v3 gold count and name the rows that collide The loader claimed to be strictly stricter than upstream, which asserts `len(prompt) == len(possible_answer)`. It was not: the prompt count was checked against the category table and the gold count was not checked at all, so a re-upload that grew only the `possible_answer/` file passed every guard -- the join is by index, so the extra rows are simply never looked up. The gold count is now asserted against the same table, for the joined categories only (the goldless three have no gold file). `_index_rows` reported `len(rows) - len(by_index)` -- excess rows, not colliding keys, so three rows on one index read as "2 rows sharing an index" -- and named neither the index nor a single id, while the gold-miss guard 35 lines below prints three example ids. It now reports the colliding indices with the ids that collide on each. `_universal_index` raised a bare `ValueError: invalid literal for int()` on an id like `simple_0a`, with no category, row or file in it, on exactly the scenario the rest of this module's messages are written for: a pin bump that moves the id shape while the counts hold. It raises the same named shape as its neighbours now. Also marks the ignored `**kwargs` on both `load` overrides deliberate, as `spider.py` does. Co-Authored-By: Claude Opus 5 --- sieval/datasets/_bfcl_v3.py | 66 +++++++++++++++++++++-------- sieval/datasets/bfcl_v3_live.py | 1 + sieval/datasets/bfcl_v3_non_live.py | 1 + tests/unit/datasets/test_bfcl_v3.py | 57 +++++++++++++++++++++++-- 4 files changed, 103 insertions(+), 22 deletions(-) diff --git a/sieval/datasets/_bfcl_v3.py b/sieval/datasets/_bfcl_v3.py index 616dba83..4e97a67f 100644 --- a/sieval/datasets/_bfcl_v3.py +++ b/sieval/datasets/_bfcl_v3.py @@ -23,9 +23,11 @@ for the goldless three, which are never joined -- `live_relevance` ships the same row twice and is 18 rows over 17 distinct ids. -Row counts are asserted per category. The revision pin already prevents a silent -re-upload; what it cannot prevent is someone bumping the pin, so a count that -moves fails here rather than quietly rescoring a leaderboard column. +Row counts are asserted per category, against the gold file as well as the +prompt file -- the pair of them is what covers upstream's own +`len(prompt) == len(possible_answer)`. The revision pin already prevents a +silent re-upload; what it cannot prevent is someone bumping the pin, so a count +that moves fails here rather than quietly rescoring a leaderboard column. AI-Generated Code - Claude Opus 5 (Anthropic) """ @@ -49,7 +51,7 @@ def _read_jsonl(path: Path) -> list[dict]: return [json.loads(line) for line in handle if line.strip()] -def _universal_index(row_id: str) -> int: +def _universal_index(row_id: str, category: str, what: str) -> int: """The join key of one row: the index in `_[--]`. Mirrors the index that upstream's own sort key extracts -- it splits on the @@ -57,18 +59,39 @@ def _universal_index(row_id: str) -> int: sub-indices the live categories carry are discarded here exactly as they are there. """ - index = row_id.rsplit("_", 1)[-1] - return int(index.split("-")[0]) + index = row_id.rsplit("_", 1)[-1].split("-")[0] + try: + return int(index) + except ValueError: + raise ValueError( + f"BFCL v3 {what} row {row_id!r} in category {category!r} has " + f"{index!r} where its universal index should be. The index is the " + "join key upstream sorts both files on, so an id whose shape moved " + "cannot be joined at all; re-verify the category against upstream." + ) from None def _index_rows(rows: list[dict], category: str, what: str) -> dict[int, dict]: - by_index = {_universal_index(row["id"]): row for row in rows} - if len(by_index) != len(rows): + by_index: dict[int, dict] = {} + collisions: dict[int, list[str]] = {} + for row in rows: + index = _universal_index(row["id"], category, what) + if index in by_index: + collisions.setdefault(index, [by_index[index]["id"]]).append(row["id"]) + else: + by_index[index] = row + + if collisions: + # The ids, not just a count: on a pin bump this is the difference + # between a one-minute diagnosis and a manual scan of 1053 rows. + examples = {index: collisions[index] for index in sorted(collisions)[:3]} + plural = "index" if len(collisions) == 1 else "indices" raise ValueError( - f"BFCL v3 category {category!r} has {len(rows) - len(by_index)} " - f"{what} rows sharing a universal index. The index is the join key " - "and upstream's runner pairs the two files positionally under it, " - "so a repeat makes the pairing ambiguous rather than merely odd." + f"BFCL v3 category {category!r} has {len(collisions)} universal " + f"{plural} shared by more than one {what} row (e.g. {examples}). " + "The index is the join key and upstream's runner pairs the two " + "files positionally under it, so a repeat makes the pairing " + "ambiguous rather than merely odd." ) return by_index @@ -95,11 +118,16 @@ def load_categories(name_or_path: str, counts: Mapping[str, int]) -> HFDatasetDi # we -- deduplicating here would publish a denominator upstream # never used. prompt_by_index = _index_rows(prompts, category, "prompt") - answers = _index_rows( - _read_jsonl(root / "possible_answer" / f"BFCL_v3_{category}.json"), - category, - "possible_answer", - ) + gold = _read_jsonl(root / "possible_answer" / f"BFCL_v3_{category}.json") + if len(gold) != expected: + raise ValueError( + f"BFCL v3 category {category!r} has {len(gold)} " + f"possible_answer rows at the pinned revision, expected " + f"{expected}. Upstream asserts the prompt and gold files are " + "the same length, so a gold file that moved on its own would " + "rescore the column without a single prompt changing." + ) + answers = _index_rows(gold, category, "possible_answer") gold_by_index = { index: json.dumps(entry["ground_truth"], ensure_ascii=False) for index, entry in answers.items() @@ -131,7 +159,9 @@ def load_categories(name_or_path: str, counts: Mapping[str, int]) -> HFDatasetDi "language": LANGUAGE_BY_CATEGORY.get(category, "Python"), "question": turns[0], "function": json.dumps(row["function"], ensure_ascii=False), - "ground_truth": gold_by_index.get(_universal_index(row["id"])), + "ground_truth": gold_by_index.get( + _universal_index(row["id"], category, "prompt") + ), } ) diff --git a/sieval/datasets/bfcl_v3_live.py b/sieval/datasets/bfcl_v3_live.py index da867947..2256fd7b 100644 --- a/sieval/datasets/bfcl_v3_live.py +++ b/sieval/datasets/bfcl_v3_live.py @@ -62,4 +62,5 @@ class BfclV3LiveDatasetSample(TypedDict): class BfclV3LiveDataset(Dataset[BfclV3LiveDatasetSample]): @override def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + _ = kwargs return load_categories(name_or_path, LIVE_COUNTS) diff --git a/sieval/datasets/bfcl_v3_non_live.py b/sieval/datasets/bfcl_v3_non_live.py index bab6db31..d47d917c 100644 --- a/sieval/datasets/bfcl_v3_non_live.py +++ b/sieval/datasets/bfcl_v3_non_live.py @@ -61,4 +61,5 @@ class BfclV3NonLiveDatasetSample(TypedDict): class BfclV3NonLiveDataset(Dataset[BfclV3NonLiveDatasetSample]): @override def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + _ = kwargs return load_categories(name_or_path, NON_LIVE_COUNTS) diff --git a/tests/unit/datasets/test_bfcl_v3.py b/tests/unit/datasets/test_bfcl_v3.py index 391e6d1f..8ca070eb 100644 --- a/tests/unit/datasets/test_bfcl_v3.py +++ b/tests/unit/datasets/test_bfcl_v3.py @@ -156,6 +156,52 @@ def test_a_moved_row_count_raises(tmp_path): load_categories(str(tmp_path), {"simple": 400}) +def test_a_moved_gold_row_count_raises(tmp_path): + """Upstream asserts prompt and gold are the same length. Guarding only the + prompt side would let a re-upload that adds gold rows through: the loader + joins on the index, so the extra rows are simply never looked up and the + column is rescored with nothing to show for it. + """ + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [[{"role": "user", "content": "hi"}]], + "function": [], + } + ], + gold=[{"id": f"simple_{i}", "ground_truth": []} for i in range(3)], + ) + with pytest.raises(ValueError, match="3 possible_answer rows"): + load_categories(str(tmp_path), {"simple": 1}) + + +def test_a_non_numeric_universal_index_raises(tmp_path): + """The id shape moving while the counts hold is exactly what a pin bump can + do, and it must not surface as a bare `int()` failure with no category, row + or file in it. + """ + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0a", + "question": [[{"role": "user", "content": "hi"}]], + "function": [], + } + ], + gold=[{"id": "simple_0a", "ground_truth": []}], + ) + with pytest.raises(ValueError, match="where its universal index should be") as e: + load_categories(str(tmp_path), {"simple": 1}) + assert "'simple_0a'" in str(e.value) + assert "'simple'" in str(e.value) + assert "'0a'" in str(e.value) + + def test_a_multi_turn_row_is_refused(tmp_path): _write( tmp_path, @@ -230,8 +276,13 @@ def test_a_repeated_universal_index_in_the_prompt_file_raises(tmp_path): {"id": "live_multiple_1-1-0", "ground_truth": []}, ], ) - with pytest.raises(ValueError, match="prompt rows sharing a universal index"): + with pytest.raises( + ValueError, match="shared by more than one prompt row" + ) as raised: load_categories(str(tmp_path), {"live_multiple": 2}) + # The colliding index and its ids, not a bare count of excess rows. + assert "{0: ['live_multiple_0-1-0', 'live_multiple_0-2-0']}" in str(raised.value) + assert "1 universal index " in str(raised.value) def test_a_repeated_universal_index_in_the_gold_file_raises(tmp_path): @@ -247,9 +298,7 @@ def test_a_repeated_universal_index_in_the_gold_file_raises(tmp_path): {"id": "live_multiple_0-2-0", "ground_truth": []}, ], ) - with pytest.raises( - ValueError, match="possible_answer rows sharing a universal index" - ): + with pytest.raises(ValueError, match="shared by more than one possible_answer row"): load_categories(str(tmp_path), {"live_multiple": 2}) From 3cc0676e60fbcfe3264f6cadd42a52fb63d5f97c Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 19:45:42 +0800 Subject: [PATCH 11/28] feat(tasks): add the BFCL v3 shared stages for both protocols One generic base carries preprocess/infer/postprocess/feedback; two axis classes differ only in how schemas reach the model and how calls are read back, which is exactly what upstream's (FC) and (Prompt) columns differ in. The `underscore_to_dot` flag travels as a call argument rather than a module global, because grading runs in a worker process -- a global set in the parent never arrives, and getting it wrong grades an FC run under the Prompt convention and yields a plausible score instead of an error. Co-Authored-By: Claude Opus 5 --- sieval/tasks/bfcl_v3/__init__.py | 0 sieval/tasks/bfcl_v3/_base.py | 315 +++++++++++++++++++++++++ tests/unit/tasks/bfcl_v3/__init__.py | 0 tests/unit/tasks/bfcl_v3/conftest.py | 59 +++++ tests/unit/tasks/bfcl_v3/test__base.py | 269 +++++++++++++++++++++ 5 files changed, 643 insertions(+) create mode 100644 sieval/tasks/bfcl_v3/__init__.py create mode 100644 sieval/tasks/bfcl_v3/_base.py create mode 100644 tests/unit/tasks/bfcl_v3/__init__.py create mode 100644 tests/unit/tasks/bfcl_v3/conftest.py create mode 100644 tests/unit/tasks/bfcl_v3/test__base.py diff --git a/sieval/tasks/bfcl_v3/__init__.py b/sieval/tasks/bfcl_v3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sieval/tasks/bfcl_v3/_base.py b/sieval/tasks/bfcl_v3/_base.py new file mode 100644 index 00000000..dc56a0d7 --- /dev/null +++ b/sieval/tasks/bfcl_v3/_base.py @@ -0,0 +1,315 @@ +"""Shared stages for the BFCL v3 single-turn task family. + +Four leaves come out of two independent choices, and this module holds what +every one of them shares. + +**Group** (``non_live`` / ``live``) selects the dataset and the aggregation +rule. It is the generic base, because it is what binds the concrete sample type +and what picks the rollup. + +**Protocol** (``Prompt`` / ``FC``) is upstream's own pair of published columns, +and it is the axis the two mixins here carry. They differ in exactly two +places: how the function schemas reach the model -- rendered into a system +turn, or sent as native tools -- and how calls are read back -- parsed out of +the reply text, or read off structured tool calls. Nothing else about the two +runs differs, which is why the protocol is a stateless mixin rather than a +second class hierarchy. + +**One divergence follows from the protocol, and it is upstream's.** A native +tool name cannot carry a dot in the OpenAI dialects, so upstream rewrites +``geometry.triangle_area`` to ``geometry_triangle_area`` on the way out and +reverses the rewrite before comparing against gold. That reversal is the +``underscore_to_dot`` flag: true for FC, false for Prompt. + +**The flag travels as a call argument, never as a module global.** Grading runs +in a worker process, so a global set in the parent never arrives: an FC run +would then be graded under the Prompt convention, and the symptom is a +plausible-looking score rather than an error. That is the whole reason grading +goes through one module-level function instead of a method. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json +from collections.abc import Callable +from typing import ClassVar + +from sieval.community.bfcl_v3 import ( + CALL_EXPECTED, + DEFAULT_SYSTEM_PROMPT, + GOLDLESS_CATEGORIES, + ast_checker, + convert_to_tool, + is_empty_output, + is_function_calling_format_output, + set_underscore_to_dot, +) +from sieval.community.bfcl_v3.tool_convert import ModelStyle +from sieval.community.bfcl_v3.type_mappings import GORILLA_TO_OPENAPI +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + InputKind, + JudgementRecord, + PredictionRecord, + PromptRecord, + Task, + TaskRequirements, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, +) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound + +# `CALL_EXPECTED` is upstream's opposite polarity: `live_relevance` is correct +# when a call IS produced, the two irrelevance sets when one is NOT. + + +def grade_single_turn( + functions_json: str, + decoded: list | None, + gold_json: str | None, + language: str, + category: str, + underscore_to_dot: bool, +) -> bool: + """Grade one decoded reply. Module-level so `run_cpu_bound` can pickle it. + + `underscore_to_dot` is an ARGUMENT, not a module global set by the caller: + this runs in a worker process, so a global set in the parent never arrives. + Getting that wrong grades an FC run under the Prompt convention and produces + a plausible score rather than an error. + + `decoded` is None when the reply could not be decoded into calls at all. + """ + set_underscore_to_dot(underscore_to_dot) + + produced_call = ( + decoded is not None + and is_function_calling_format_output(decoded) + and not is_empty_output(decoded) + ) + + if category in GOLDLESS_CATEGORIES: + return produced_call if category in CALL_EXPECTED else not produced_call + + if not produced_call: + # `ast_decoder:decoder_failed` / `:decoder_wrong_output_format` -- a + # scored outcome, not an error. + return False + + if gold_json is None: + raise ValueError( + f"BFCL v3 category {category!r} is not goldless but row gold is None; " + "a value-reference task must not grade without its reference." + ) + + result = ast_checker( + json.loads(functions_json), + decoded, + json.loads(gold_json), + language, + category, + "sieval", + ) + return bool(result["valid"]) + + +#: Upstream's harness at the pinned revision, cited by every leaf's +#: `reference_impl`. +BFCL_V3_HARNESS_URL = ( + "https://github.com/ShishirPatil/gorilla/tree/" + "ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard" +) + +#: What every `reference_impl.notes` for this benchmark has to say, whichever +#: group and protocol it is. The specific half is prepended by each leaf. +BFCL_V3_SHARED_NOTES = ( + "AST checker, source parsers, type converters, prompt and aggregation " + "vendored verbatim from gorilla v1.3 (Apache-2.0); only `bfcl_eval.*` " + "imports were rewritten. " + "exec_*/rest/sql/chatable are excluded -- upstream has them commented out of " + "its own TEST_FILE_MAPPING and they need live API keys." +) + + +class BfclV3Task[TSample]( + Task[ + TSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + # `str`: the report carries `score_key` and `denominator_policy`, which + # name things rather than measure them. `list[float]` is an interval and + # `dict[str, str]` the map saying which population each is clustered on. + dict[str, float | int | str | list[float] | dict[str, str]], + ] +): + """Every stage but the protocol hooks and the group-level rollup.""" + + #: Set by the protocol mixin. + UNDERSCORE_TO_DOT: ClassVar[bool] + #: Set by the group base. Maps every category of this group to its declared + #: row count, which is also its scoring DENOMINATOR. + CATEGORY_COUNTS: ClassVar[dict[str, int]] + #: Set by the group base. `"non_live"` / `"live"` -- names the headline key + #: (`f"{GROUP_KEY}_overall_acc"`) and so also `score_key`. + GROUP_KEY: ClassVar[str] + + def _build_messages(self, raw) -> tuple[list, list[dict] | None]: + """Return (messages, tools). Implemented by the protocol mixin. + + The messages list is bare: a record's `prompt` is `JSONValue`, and a + `list[dict]` is not assignable to it. + """ + raise NotImplementedError + + def _decode(self, output: ModelOutput, language: str) -> list | None: + """Return decoded calls, or None when the reply yields none. + + `language` is the row's language -- "Python", "Java" or "JavaScript". + The Prompt protocol passes it straight to `ast_parse`, which dispatches + on it. The FC protocol ignores it: it reads structured tool calls and + never parses source text. + """ + raise NotImplementedError + + def _aggregate(self, cell: Callable[[str], dict]) -> dict: + """Group-level rollup. Implemented by the group base.""" + raise NotImplementedError + + async def preprocess(self, raw, ctx) -> PromptRecord: + messages, tools = self._build_messages(raw) + return build_prompt_record( + messages, + reference=raw["ground_truth"], + extra={ + "id": raw["id"], + "category": raw["category"], + "language": raw["language"], + "function": raw["function"], + "tools": tools, + }, + ) + + async def infer(self, pre, ctx) -> ModelOutput: + tools = pre["extra"]["tools"] + kwargs = {"tools": tools} if tools else {} + return await self.model.agenerate(pre["prompt"], **kwargs) + + async def postprocess(self, inf, ctx) -> PredictionRecord: + # The row's language selects the PARSER, so it has to reach `_decode`. + # It is carried on the preprocess record rather than re-derived here. + language = ctx.preprocess_result["extra"]["language"] + try: + decoded = self._decode(inf, language) + except Exception as exc: # noqa: BLE001 -- a decode failure is a SCORE + # Upstream's `ast_decoder:decoder_failed`: the model produced + # something undecodable, which is the model's outcome, not a fault. + return build_prediction_record([None], extras=[{"decode_error": str(exc)}]) + return build_prediction_record([decoded]) + + async def feedback(self, post, ctx) -> tuple[bool, JudgementRecord]: + pre_extra = ctx.preprocess_result["extra"] + category = pre_extra["category"] + # `.get()`, not `[]`: a None prediction is dropped by serialization, so + # on a resume the key is ABSENT for exactly the rows that failed to + # decode -- the ones this task most needs to score. + decoded = post["rollouts"][0].get("prediction") + gold = ctx.preprocess_result.get("reference") + + try: + correct = await run_cpu_bound( + grade_single_turn, + pre_extra["function"], + decoded, + gold, + pre_extra["language"], + category, + self.UNDERSCORE_TO_DOT, + timeout=GRADE_TIMEOUT, + ) + except TimeoutError: + # A grade that could not be computed in time is a wrong answer -- + # the prediction is a shape the grader cannot bound. Every OTHER + # exception propagates: a broken grader must not read as a wrong + # model. + correct = False + + # A goldless category's reference is the expected OUTCOME, not None: + # `reference_kind="value"` is inferred from whether the call sites pass + # a literal None, and these rows do have something to compare against. + reference = ( + gold + if gold is not None + else ("call" if category in CALL_EXPECTED else "no_call") + ) + return True, build_judgement_record( + reference, + [build_rollout_judgement(0, correct)], + extra={"category": category, "id": pre_extra["id"]}, + ) + + +def _call_arguments(call) -> dict: + """Read one tool call's arguments as a dict. + + `FunctionToolCall.arguments` is typed `JSONValue`, not `str`. The OpenAI + dialects put the provider's wire JSON string there and substitute `""` when + the provider omits it entirely, so a bare `json.loads` raises on a + no-argument call -- a real shape for a zero-parameter tool. + """ + arguments = call.arguments + if isinstance(arguments, str): + return json.loads(arguments) if arguments.strip() else {} + return dict(arguments) if arguments else {} + + +class BfclV3PromptMixin: + """Upstream's prompting protocol: schemas in the system turn, calls in text.""" + + UNDERSCORE_TO_DOT: ClassVar[bool] = False + + def _build_messages(self, raw) -> tuple[list, list[dict] | None]: + system = DEFAULT_SYSTEM_PROMPT.format(functions=raw["function"]) + return [{"role": "system", "content": system}, *raw["question"]], None + + def _decode(self, output: ModelOutput, language: str) -> list | None: + # Deferred on purpose: `ast_parse` is the one symbol in the vendored + # package that reaches tree-sitter, which lives in the optional + # `bfcl-v3` extra. Importing a task module registers it, and + # registration is paid by every task listing and by every eval that + # fails before grading -- so at module scope this would make the extra + # a hard requirement and break importing the task registry on a base + # install. It is genuinely needed only here -- but here it is needed + # for EVERY language, not just Java and JavaScript: `ast_parse` imports + # both source parsers at module scope, so the first call pulls + # tree-sitter in whichever grammar it ends up dispatching to. + from sieval.community.bfcl_v3 import ast_parse + + text = output.texts[0] if output.texts else "" + return ast_parse(text, language) + + +class BfclV3FCMixin: + """Upstream's native-function-calling protocol: schemas as tools.""" + + UNDERSCORE_TO_DOT: ClassVar[bool] = True + + requires = TaskRequirements(input=InputKind.CHAT, function_tools=True) + + def _build_messages(self, raw) -> tuple[list, list[dict] | None]: + tools = convert_to_tool( + json.loads(raw["function"]), + GORILLA_TO_OPENAPI, + ModelStyle.OpenAI_Completions, + ) + return list(raw["question"]), tools + + def _decode(self, output: ModelOutput, language: str) -> list | None: + del language # Structured calls arrive typed; no source text is parsed. + calls = output.tool_calls + if not calls: + return [] + return [{call.name: _call_arguments(call)} for call in calls] diff --git a/tests/unit/tasks/bfcl_v3/__init__.py b/tests/unit/tasks/bfcl_v3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/tasks/bfcl_v3/conftest.py b/tests/unit/tasks/bfcl_v3/conftest.py new file mode 100644 index 00000000..15bf3bc5 --- /dev/null +++ b/tests/unit/tasks/bfcl_v3/conftest.py @@ -0,0 +1,59 @@ +"""Shared rows for the BFCL v3 task family. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json + +import pytest + + +@pytest.fixture +def simple_row(): + return { + "id": "simple_0", + "category": "simple", + "language": "Python", + "question": [{"role": "user", "content": "Area of a 10x5 triangle?"}], + "function": json.dumps( + [ + { + "name": "calculate_triangle_area", + "description": "Area of a triangle.", + "parameters": { + "type": "dict", + "properties": { + "base": {"type": "integer", "description": "base"}, + "height": {"type": "integer", "description": "height"}, + }, + "required": ["base", "height"], + }, + } + ] + ), + "ground_truth": json.dumps( + [{"calculate_triangle_area": {"base": [10], "height": [5]}}] + ), + } + + +@pytest.fixture +def dotted_row(simple_row): + """A function name containing a dot -- the FC/Prompt divergence probe.""" + row = dict(simple_row) + functions = json.loads(row["function"]) + functions[0]["name"] = "geometry.triangle_area" + row["function"] = json.dumps(functions) + row["ground_truth"] = json.dumps( + [{"geometry.triangle_area": {"base": [10], "height": [5]}}] + ) + return row + + +@pytest.fixture +def irrelevance_row(simple_row): + row = dict(simple_row) + row["id"] = "irrelevance_0" + row["category"] = "irrelevance" + row["ground_truth"] = None + return row diff --git a/tests/unit/tasks/bfcl_v3/test__base.py b/tests/unit/tasks/bfcl_v3/test__base.py new file mode 100644 index 00000000..881dd3a2 --- /dev/null +++ b/tests/unit/tasks/bfcl_v3/test__base.py @@ -0,0 +1,269 @@ +"""Cover the BFCL v3 shared grading entry point and the two protocol mixins. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json + +import pytest + +from sieval.core.models import FunctionToolCall, ModelMeta, ModelOutput +from sieval.tasks.bfcl_v3._base import ( + BfclV3FCMixin, + BfclV3PromptMixin, + grade_single_turn, +) + +_META: ModelMeta = {"model": "mock-chat", "api_base": None, "default_params": {}} + + +def _output(text: str = "", tool_calls=None) -> ModelOutput: + return ModelOutput(model=_META, texts=[text], tool_calls=tool_calls) + + +def test_a_correct_call_grades_true(simple_row): + decoded = [{"calculate_triangle_area": {"base": 10, "height": 5}}] + assert ( + grade_single_turn( + simple_row["function"], + decoded, + simple_row["ground_truth"], + "Python", + "simple", + False, + ) + is True + ) + + +def test_a_wrong_argument_grades_false(simple_row): + decoded = [{"calculate_triangle_area": {"base": 10, "height": 99}}] + assert ( + grade_single_turn( + simple_row["function"], + decoded, + simple_row["ground_truth"], + "Python", + "simple", + False, + ) + is False + ) + + +def test_a_failed_decode_grades_false_for_an_ast_category(simple_row): + assert ( + grade_single_turn( + simple_row["function"], + None, + simple_row["ground_truth"], + "Python", + "simple", + False, + ) + is False + ) + + +# Both irrelevance sets, because they are separate keys in the polarity table +# and a typo in one of them is invisible while the other is tested. +@pytest.mark.parametrize("category", ["irrelevance", "live_irrelevance"]) +def test_irrelevance_is_correct_when_no_call_is_produced(irrelevance_row, category): + assert ( + grade_single_turn( + irrelevance_row["function"], None, None, "Python", category, False + ) + is True + ) + assert ( + grade_single_turn( + irrelevance_row["function"], [], None, "Python", category, False + ) + is True + ) + + +@pytest.mark.parametrize("category", ["irrelevance", "live_irrelevance"]) +def test_irrelevance_is_wrong_when_a_call_is_produced(irrelevance_row, category): + decoded = [{"calculate_triangle_area": {"base": 10, "height": 5}}] + assert ( + grade_single_turn( + irrelevance_row["function"], decoded, None, "Python", category, False + ) + is False + ) + + +def test_live_relevance_inverts_irrelevance(irrelevance_row): + decoded = [{"calculate_triangle_area": {"base": 10, "height": 5}}] + assert ( + grade_single_turn( + irrelevance_row["function"], + decoded, + None, + "Python", + "live_relevance", + False, + ) + is True + ) + assert ( + grade_single_turn( + irrelevance_row["function"], None, None, "Python", "live_relevance", False + ) + is False + ) + + +def test_underscore_to_dot_reconciles_the_fc_tool_name(dotted_row): + """FC sent `geometry_triangle_area`; gold says `geometry.triangle_area`.""" + fc_decoded = [{"geometry_triangle_area": {"base": 10, "height": 5}}] + prompt_decoded = [{"geometry.triangle_area": {"base": 10, "height": 5}}] + + assert ( + grade_single_turn( + dotted_row["function"], + fc_decoded, + dotted_row["ground_truth"], + "Python", + "simple", + True, + ) + is True + ) + assert ( + grade_single_turn( + dotted_row["function"], + prompt_decoded, + dotted_row["ground_truth"], + "Python", + "simple", + False, + ) + is True + ) + # And the flag is load-bearing in both directions, which is what makes + # passing it through the process boundary necessary rather than tidy. + assert ( + grade_single_turn( + dotted_row["function"], + fc_decoded, + dotted_row["ground_truth"], + "Python", + "simple", + False, + ) + is False + ) + + +# -------------------------------------------------------------------------- +# The protocol axis. Both mixins are stateless, so they are exercised directly +# rather than through a whole Task. +# -------------------------------------------------------------------------- + +#: Valid JavaScript that is ALSO valid Python, and decodes differently under +#: each. Deliberately not a snippet only one parser accepts: this one pins the +#: failure mode a hardcoded language actually has on real rows. +_JS_CALL = "[obj.method(opts={a: 1, b: 2})]" + + +def test_the_prompt_mixin_parses_under_the_row_language(): + """A JavaScript row must reach the JavaScript parser, not Python's `ast`. + + The failure this pins is the quiet one: `_JS_CALL` parses under BOTH + languages, and Python's parser resolves the object literal into a real + dict where JavaScript's keeps it as the source string. Hardcoding + `"Python"` therefore yields a plausible wrong value rather than a raise -- + a wrong score with nothing in the record to show for it. + """ + decoded = BfclV3PromptMixin()._decode(_output(_JS_CALL), "JavaScript") + assert decoded == [{"obj.method": {"opts": "{a: 1, b: 2}"}}] + # The same text under Python -- what a hardcoded language would return. + assert BfclV3PromptMixin()._decode(_output(_JS_CALL), "Python") == [ + {"obj.method": {"opts": {"a": 1, "b": 2}}} + ] + + +def test_the_prompt_mixin_puts_the_schemas_in_a_system_turn_and_sends_no_tools( + dotted_row, +): + messages, tools = BfclV3PromptMixin()._build_messages(dotted_row) + assert tools is None + assert messages[0]["role"] == "system" + # Verbatim, dot included: the Prompt protocol never renames anything, which + # is why its `UNDERSCORE_TO_DOT` is False. + assert "geometry.triangle_area" in messages[0]["content"] + assert messages[1:] == dotted_row["question"] + + +def test_the_fc_mixin_sends_tools_and_renames_the_dotted_function(dotted_row): + """Upstream's own rewrite -- the one `UNDERSCORE_TO_DOT` later reverses.""" + messages, tools = BfclV3FCMixin()._build_messages(dotted_row) + assert messages == dotted_row["question"] + assert tools is not None + assert [tool["function"]["name"] for tool in tools] == ["geometry_triangle_area"] + + +def test_the_fc_mixin_ignores_the_language(): + """No source text is parsed, so the row's language changes nothing.""" + output = _output( + tool_calls=(FunctionToolCall("c0", "calculate_triangle_area", '{"base": 10}'),) + ) + expected = [{"calculate_triangle_area": {"base": 10}}] + assert BfclV3FCMixin()._decode(output, "Java") == expected + assert BfclV3FCMixin()._decode(output, "Python") == expected + + +def test_the_fc_mixin_reads_no_calls_as_an_empty_list(): + """`[]` is "decoded fine, called nothing" -- correct for irrelevance. + + Distinct from a raise, which means undecodable. Both score wrong for an AST + category, but they are different upstream outcomes. + """ + assert BfclV3FCMixin()._decode(_output("no thanks"), "Python") == [] + + +@pytest.mark.parametrize("arguments", ["", " ", {}]) +def test_the_fc_mixin_reads_an_absent_argument_payload_as_no_arguments(arguments): + """A zero-parameter tool is a real shape, and `json.loads("")` raises.""" + output = _output(tool_calls=(FunctionToolCall("c0", "list_all", arguments),)) + assert BfclV3FCMixin()._decode(output, "Python") == [{"list_all": {}}] + + +def test_the_two_protocols_disagree_on_a_dotted_name_and_each_flag_reconciles_it( + dotted_row, +): + """The end-to-end reason the flag exists, across both mixins at once.""" + prompt_decoded = BfclV3PromptMixin()._decode( + _output("[geometry.triangle_area(base=10, height=5)]"), "Python" + ) + fc_decoded = BfclV3FCMixin()._decode( + _output( + tool_calls=( + FunctionToolCall( + "c0", + "geometry_triangle_area", + json.dumps({"base": 10, "height": 5}), + ), + ) + ), + "Python", + ) + assert prompt_decoded != fc_decoded + + for decoded, mixin in ( + (prompt_decoded, BfclV3PromptMixin), + (fc_decoded, BfclV3FCMixin), + ): + assert ( + grade_single_turn( + dotted_row["function"], + decoded, + dotted_row["ground_truth"], + "Python", + "simple", + mixin.UNDERSCORE_TO_DOT, + ) + is True + ) From f999135cf14c8406c73d61cc98cdf37d008d92bb Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 19:49:14 +0800 Subject: [PATCH 12/28] docs(deps): correct when the `bfcl-v3` extra is actually reached The comment said the extra is reached "only by the Prompt protocol on the `java` and `javascript` categories". The first half is right; the second is not. `parser.py` imports both source parsers at module scope and each imports `tree_sitter` at its own module scope, with no guard, so touching `ast_parse` at all needs the extra -- a Python row included. Caught by blocking `tree_sitter` with a `find_spec` meta-path finder: importing the task package still succeeds, but calling `_decode` on a Python row raises `ModuleNotFoundError` exactly as a JavaScript row does. Comment only. No dependency changed. Co-Authored-By: Claude Opus 5 --- pyproject.toml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d4170fc0..06157b49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,10 +37,11 @@ Changelog = "https://github.com/scitix/sieval/blob/main/CHANGELOG.md" [project.optional-dependencies] bfcl-v3 = [ - # Reached only by the Prompt protocol on the `java` and `javascript` - # categories: upstream's `ast_parse` dispatches those two to tree-sitter - # parsers rather than Python's `ast`. The FC protocol never parses source - # text, and every Live category is Python. + # Reached only by the Prompt protocol, and there by every category rather + # than just `java` and `javascript`: upstream's `ast_parse` dispatches those + # two to tree-sitter parsers rather than Python's `ast`, but it imports both + # source parsers at module scope, so touching `ast_parse` at all requires + # them. The FC protocol never parses source text and does not need this. # # Pinned to upstream's own versions. tree_sitter 0.21.3 predates the 0.22 API # break -- the vendored parsers call `Language(ptr, name)` and From f5635fc49171fc8e6b262e893db2a81869f7d642 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 19:59:42 +0800 Subject: [PATCH 13/28] test(tasks): pin that a missing gold fails the row rather than scoring it A reverse mutation replacing the `gold_json is None` raise with `return False` left all 18 tests passing, and no preflight check covers it either -- the two adjacent ones, `check_record_key_access` and `check_reference_kind`, both pass under that mutation. The guarantee is a rule of its own (a value-reference task that finds no gold raises), so it is worth a test of its own: scored `False`, a dataset fault is charged to the model, and under `DENOMINATOR_REQUESTED` the headline cannot tell the two apart. Co-Authored-By: Claude Opus 5 --- tests/unit/tasks/bfcl_v3/test__base.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/unit/tasks/bfcl_v3/test__base.py b/tests/unit/tasks/bfcl_v3/test__base.py index 881dd3a2..13dd7791 100644 --- a/tests/unit/tasks/bfcl_v3/test__base.py +++ b/tests/unit/tasks/bfcl_v3/test__base.py @@ -157,6 +157,25 @@ def test_underscore_to_dot_reconciles_the_fc_tool_name(dotted_row): ) +def test_a_missing_gold_raises_rather_than_scoring_the_row(simple_row): + """A value-reference row with no gold fails; it is never graded as wrong. + + The distinction is the whole point: scoring it `False` would charge the + model for a dataset fault, and under `DENOMINATOR_REQUESTED` the two read + the same in the headline. Reached only past the produced-call guard, so the + call passed here has to be a valid one. + """ + with pytest.raises(ValueError, match="not goldless"): + grade_single_turn( + simple_row["function"], + [{"calculate_triangle_area": {"base": 10, "height": 5}}], + None, + "Python", + "simple", + False, + ) + + # -------------------------------------------------------------------------- # The protocol axis. Both mixins are stateless, so they are exercised directly # rather than through a whole Task. From 7392109f0726b75838ce77976c55110aa04b8412 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 20:20:49 +0800 Subject: [PATCH 14/28] feat(tasks): roll BFCL v3 up the way upstream scores it Non-live is an unweighted mean of five, with `simple_ast` nested inside it; live is a sample-count-weighted mean of six. Both go through the vendored aggregation functions rather than being re-derived here. The split also decides the intervals. A weighted mean is algebraically the pooled rate over its rows, so live's two rollups are genuine per-sample rates and carry one. An unweighted mean of rates is not -- javascript's 50 rows weigh the same as simple's 400 -- so non-live's three publish none rather than bracketing a statistic other than the one printed beside them. Per-sample values are 0/1, not 0/100: the estimators read `sum(values) / denominator` as a probability, so percentage-point values make every rate above one point read as a saturated set and publish an interval bracketing 100. The rate is scaled to percentage points once, in `cell`, so the published number and its bounds share their units. Co-Authored-By: Claude Opus 5 --- sieval/tasks/bfcl_v3/_base.py | 195 +++++++++++++++++++++++- tests/unit/tasks/bfcl_v3/conftest.py | 94 ++++++++++++ tests/unit/tasks/bfcl_v3/test_report.py | 172 +++++++++++++++++++++ 3 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 tests/unit/tasks/bfcl_v3/test_report.py diff --git a/sieval/tasks/bfcl_v3/_base.py b/sieval/tasks/bfcl_v3/_base.py index dc56a0d7..ee80abf3 100644 --- a/sieval/tasks/bfcl_v3/_base.py +++ b/sieval/tasks/bfcl_v3/_base.py @@ -31,14 +31,19 @@ """ import json -from collections.abc import Callable +from collections import defaultdict +from collections.abc import Callable, Mapping from typing import ClassVar from sieval.community.bfcl_v3 import ( CALL_EXPECTED, DEFAULT_SYSTEM_PROMPT, GOLDLESS_CATEGORIES, + LIVE_COUNTS, + NON_LIVE_COUNTS, ast_checker, + calculate_unweighted_accuracy, + calculate_weighted_accuracy, convert_to_tool, is_empty_output, is_function_calling_format_output, @@ -59,6 +64,14 @@ build_prompt_record, build_rollout_judgement, ) +from sieval.core.tasks.metrics import ( + DENOMINATOR_FIELD, + DENOMINATOR_REQUESTED, + SCORE_KEY_FIELD, + interval_metrics, + merge_metrics, + metric_interval, +) from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound # `CALL_EXPECTED` is upstream's opposite polarity: `live_relevance` is correct @@ -251,6 +264,186 @@ async def feedback(self, post, ctx) -> tuple[bool, JudgementRecord]: extra={"category": category, "id": pre_extra["id"]}, ) + async def report(self, finals: list, fails: list) -> dict: + # One 0/1 per sample that came back, NOT one percentage point: this list + # is both what the cells average and what the interval estimators are + # handed, and those read `sum(values) / denominator` as a probability. + # Hand them percent and any rate above a single point reads as a + # saturated set, which still publishes a whole triple -- an interval + # bracketing 100 beside a rate that is nothing of the kind. The rate is + # scaled to percentage points once, in `cell`, so the two agree. + correct_by_category: dict[str, list[float]] = defaultdict(list) + for ctx in finals: + fb = ctx.feedback_result + correct_by_category[fb["extra"]["category"]].append( + 1.0 if fb["rollouts"][0]["correct"] else 0.0 + ) + + def cell(category: str) -> dict: + """Upstream's accuracy-dict shape, over the DECLARED denominator. + + `total_count` is the category's row count, not the number that came + back: `DENOMINATOR_REQUESTED` charges a missing sample as wrong, and + a sample that failed before `preprocess` has no category to be + attributed to anyway. + + All three of upstream's keys, because both aggregation helpers read + `display_accuracy` unconditionally and a two-key cell raises + `KeyError`. It is never "N/A" here: that marks a category upstream + did not evaluate, and this task always scores the full declared set. + Their return value carries the same three keys, which is what lets + `simple_ast` nest straight back in as a cell. + """ + values = correct_by_category.get(category, []) + denominator = self.CATEGORY_COUNTS[category] + # Percentage points -- the units every rate here is published in, + # and the units the estimators return their bounds in. + accuracy = 100.0 * sum(values) / denominator + return { + "accuracy": accuracy, + "total_count": denominator, + "display_accuracy": accuracy, + } + + rollup = self._aggregate(cell) + + result: dict = { + "score": rollup[f"{self.GROUP_KEY}_overall_acc"], + "fails": len(fails), + SCORE_KEY_FIELD: f"{self.GROUP_KEY}_overall_acc", + DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, + } + result |= rollup + for category in self.CATEGORY_COUNTS: + result[f"n_{category}"] = self.CATEGORY_COUNTS[category] + + # Per-category intervals: each IS `sum(values) / denominator`, so each + # is a candidate. No `group_keys`: one row is one problem is one + # rollout, so there are no repeated copies to collapse. + fragments = [ + metric_interval( + category, + correct_by_category.get(category, []), + denominator=self.CATEGORY_COUNTS[category], + unit=f"n_{category}", + ) + for category in self.CATEGORY_COUNTS + ] + fragments.extend(self._group_intervals(correct_by_category)) + return result | merge_metrics(*fragments) + + def _group_intervals( + self, correct_by_category: Mapping[str, list[float]] + ) -> list[dict]: + """Intervals for the group-level rollups, where any are defensible.""" + del correct_by_category + return [] + + +class BfclV3NonLiveTask[TSample](BfclV3Task[TSample]): + """Non-live: seven categories, rolled up by UNWEIGHTED means. + + `simple_ast` nests three categories inside the five-way overall, so the + headline is a mean of means -- not the pooled rate over 1390 rows, and not + convergent to it: `javascript` (50 rows) enters with the same weight as + `simple` (400). That is why it publishes no interval. + """ + + GROUP_KEY: ClassVar[str] = "non_live" + CATEGORY_COUNTS: ClassVar[dict[str, int]] = dict(NON_LIVE_COUNTS) + + def _aggregate(self, cell: Callable[[str], dict]) -> dict: + simple_ast = calculate_unweighted_accuracy( + [cell("simple"), cell("java"), cell("javascript")] + ) + ast_cells = [ + simple_ast, + cell("multiple"), + cell("parallel"), + cell("parallel_multiple"), + ] + ast_summary = calculate_unweighted_accuracy(ast_cells) + overall = calculate_unweighted_accuracy([*ast_cells, cell("irrelevance")]) + return { + "non_live_overall_acc": overall["accuracy"], + "ast_summary": ast_summary["accuracy"], + "simple_ast": simple_ast["accuracy"], + **{ + category: cell(category)["accuracy"] + for category in self.CATEGORY_COUNTS + }, + } + + +class BfclV3LiveTask[TSample](BfclV3Task[TSample]): + """Live: six categories, rolled up by SAMPLE-COUNT-WEIGHTED means. + + A weighted mean over these cells is algebraically the pooled rate over the + union of their rows, so both rollups are genuine per-sample rates and both + carry intervals. + """ + + GROUP_KEY: ClassVar[str] = "live" + CATEGORY_COUNTS: ClassVar[dict[str, int]] = dict(LIVE_COUNTS) + + #: The four AST categories, in upstream's column order. `ast_summary` is + #: over these only -- irrelevance and relevance join at the overall. + AST_CATEGORIES: ClassVar[tuple[str, ...]] = ( + "live_simple", + "live_multiple", + "live_parallel", + "live_parallel_multiple", + ) + + def _aggregate(self, cell: Callable[[str], dict]) -> dict: + ast_summary = calculate_weighted_accuracy( + [cell(category) for category in self.AST_CATEGORIES] + ) + overall = calculate_weighted_accuracy( + [cell(category) for category in self.CATEGORY_COUNTS] + ) + return { + "live_overall_acc": overall["accuracy"], + "ast_summary": ast_summary["accuracy"], + **{ + category: cell(category)["accuracy"] + for category in self.CATEGORY_COUNTS + }, + } + + def _group_intervals( + self, correct_by_category: Mapping[str, list[float]] + ) -> list[dict]: + ast_values = [ + value + for category in self.AST_CATEGORIES + for value in correct_by_category.get(category, []) + ] + all_values = [ + value + for category in self.CATEGORY_COUNTS + for value in correct_by_category.get(category, []) + ] + return [ + # The headline and `live_overall_acc` are ONE number under two key + # names, so the alias rides along on the same call -- never a second + # call with the same arguments. This emits `n_problems` itself. + interval_metrics( + all_values, + denominator=sum(self.CATEGORY_COUNTS.values()), + aliases=("live_overall_acc",), + ), + # Emits `n_ast` itself, alongside the interval it is the unit for. + metric_interval( + "ast_summary", + ast_values, + denominator=sum( + self.CATEGORY_COUNTS[category] for category in self.AST_CATEGORIES + ), + unit="n_ast", + ), + ] + def _call_arguments(call) -> dict: """Read one tool call's arguments as a dict. diff --git a/tests/unit/tasks/bfcl_v3/conftest.py b/tests/unit/tasks/bfcl_v3/conftest.py index 15bf3bc5..c960533c 100644 --- a/tests/unit/tasks/bfcl_v3/conftest.py +++ b/tests/unit/tasks/bfcl_v3/conftest.py @@ -4,9 +4,13 @@ """ import json +import types import pytest +from sieval.tasks.bfcl_v3._base import BfclV3LiveTask, BfclV3NonLiveTask +from tests.conftest import MockChatModel, MockDataset + @pytest.fixture def simple_row(): @@ -57,3 +61,93 @@ def irrelevance_row(simple_row): row["category"] = "irrelevance" row["ground_truth"] = None return row + + +# -------------------------------------------------------------------------- +# The report. `report` reads one thing off a finalized context -- its judgement +# record -- so these stand in for one rather than driving a runner. +# -------------------------------------------------------------------------- + + +def _ctx(category: str, correct: bool, index: int): + return types.SimpleNamespace( + feedback_result={ + "reference": None, + "rollouts": [{"index": 0, "correct": correct}], + "n_rollouts": 1, + "n_correct": int(correct), + "extra": {"category": category, "id": f"{category}_{index}"}, + } + ) + + +def _finals(spec: dict[str, tuple[int, int]]) -> list: + """One context per sample that came back; the first *n_correct* are right. + + *spec* maps a category to `(n_returned, n_correct)`. A category returning + fewer rows than it declares is how a pipeline failure reaches the report: + the failed samples are simply absent, with nothing naming their category. + """ + return [ + _ctx(category, index < correct, index) + for category, (returned, correct) in spec.items() + for index in range(returned) + ] + + +def _group_task(cls): + """A group base with no protocol mixin -- `report` reads neither hook.""" + return cls(MockDataset(), MockChatModel()) + + +#: Every non-live category complete, and the three `simple_ast` members at +#: DIFFERENT rates: a pooled rate over their 550 rows is 72.7, an unweighted +#: mean of the three is 33.3, so the two rollup rules cannot be confused. +_NON_LIVE_FULL = { + "simple": (400, 400), + "multiple": (200, 200), + "parallel": (200, 200), + "parallel_multiple": (200, 200), + "java": (100, 0), + "javascript": (50, 0), + "irrelevance": (240, 240), +} + +#: Every live category complete, at six rates that are deliberately unequal and +#: none of them 0 or 1. Unequal, so a weighted rollup (63.3) and an unweighted +#: one (41.1) are far apart; strictly interior, so every interval comes off the +#: Wilson branch rather than a saturated-set fallback. +_LIVE_SPREAD = { + "live_simple": (258, 129), + "live_multiple": (1053, 843), + "live_parallel": (16, 4), + "live_parallel_multiple": (24, 6), + "live_irrelevance": (882, 441), + "live_relevance": (18, 3), +} + + +@pytest.fixture +async def non_live_report(): + return await _group_task(BfclV3NonLiveTask).report(_finals(_NON_LIVE_FULL), []) + + +@pytest.fixture +async def non_live_report_short(): + """One of `simple`'s 400 rows never came back, and no other category ran.""" + return await _group_task(BfclV3NonLiveTask).report( + _finals({"simple": (399, 399)}), [] + ) + + +@pytest.fixture +async def non_live_report_with_fails(): + """The same run, with the missing row present as a FAILED sample.""" + return await _group_task(BfclV3NonLiveTask).report( + _finals({"simple": (399, 399)}), [object()] + ) + + +@pytest.fixture +async def live_report(): + return await _group_task(BfclV3LiveTask).report(_finals(_LIVE_SPREAD), []) diff --git a/tests/unit/tasks/bfcl_v3/test_report.py b/tests/unit/tasks/bfcl_v3/test_report.py new file mode 100644 index 00000000..4a7c8139 --- /dev/null +++ b/tests/unit/tasks/bfcl_v3/test_report.py @@ -0,0 +1,172 @@ +"""Cover the two BFCL v3 group rollups: what each mean is over, and what carries +an interval. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import pytest + +from sieval.community.bfcl_v3 import ( + calculate_unweighted_accuracy, + calculate_weighted_accuracy, +) +from sieval.core.tasks.metrics import ( + DENOMINATOR_FIELD, + DENOMINATOR_REQUESTED, + SCORE_KEY_FIELD, +) + + +def test_unweighted_is_the_mean_of_rates_not_the_pooled_rate(): + # 1/1 and 0/99: the pooled rate is 0.01, the unweighted mean is 0.5. + # Every cell carries all THREE keys upstream reads, `display_accuracy` + # included -- both helpers dereference it unconditionally, so a two-key + # cell raises `KeyError` rather than returning a wrong number. + cells = [ + {"accuracy": 1.0, "total_count": 1, "display_accuracy": 1.0}, + {"accuracy": 0.0, "total_count": 99, "display_accuracy": 0.0}, + ] + assert calculate_unweighted_accuracy(cells)["accuracy"] == pytest.approx(0.5) + assert calculate_weighted_accuracy(cells)["accuracy"] == pytest.approx(0.01) + + +@pytest.mark.anyio +async def test_non_live_rollup_nests_simple_ast(non_live_report): + # Percentage points throughout: simple 100 (400 rows), java 0 (100), + # javascript 0 (50) -> simple_ast = (100 + 0 + 0) / 3 + assert non_live_report["simple_ast"] == pytest.approx(100 / 3) + # ast_summary = mean(simple_ast, multiple, parallel, parallel_multiple) + assert non_live_report["ast_summary"] == pytest.approx( + (100 / 3 + 100.0 + 100.0 + 100.0) / 4 + ) + # overall additionally folds in irrelevance + assert non_live_report["non_live_overall_acc"] == pytest.approx( + (100 / 3 + 100.0 + 100.0 + 100.0 + 100.0) / 5 + ) + assert non_live_report["score"] == non_live_report["non_live_overall_acc"] + + +@pytest.mark.anyio +async def test_non_live_headline_carries_no_interval(non_live_report): + for key in ( + "score_ci95", + "non_live_overall_acc_ci95", + "simple_ast_ci95", + "ast_summary_ci95", + ): + assert key not in non_live_report, ( + f"{key} is an unweighted mean of category rates; an interval on it " + "would bracket a different statistic than the number beside it." + ) + # And the population key that would have to sit beside such an interval is + # absent too -- the pair is emitted whole or not at all. + assert "n_problems" not in non_live_report + + +@pytest.mark.anyio +async def test_non_live_per_category_rates_all_carry_the_triple(non_live_report): + units = non_live_report["ci95_units"] + for category in ( + "simple", + "multiple", + "parallel", + "parallel_multiple", + "java", + "javascript", + "irrelevance", + ): + assert f"{category}_ci95" in non_live_report + assert f"n_{category}" in non_live_report + assert units[category] == f"n_{category}" + + +@pytest.mark.anyio +async def test_live_headline_carries_an_interval(live_report): + assert "score_ci95" in live_report + assert "n_problems" in live_report + assert live_report["ci95_units"]["score"] == "n_problems" + assert "ast_summary_ci95" in live_report + # The headline and its aliased twin are ONE number, carrying ONE interval. + assert live_report["live_overall_acc"] == live_report["score"] + assert live_report["live_overall_acc_ci95"] == live_report["score_ci95"] + assert live_report["ci95_units"]["live_overall_acc"] == "n_problems" + + +@pytest.mark.anyio +async def test_live_rollups_are_pooled_over_rows_not_means_of_rates(live_report): + """Weighted, so both rollups are the rate over the union of their rows. + + The six categories run from 16.7 to 80.1 at sizes from 16 to 1053, so the + unweighted mean of them (41.1) is nowhere near the pooled rate -- which is + what makes the choice of helper visible in the number rather than only in + the source. + """ + assert live_report["live_overall_acc"] == pytest.approx(100 * 1426 / 2251) + assert live_report["ast_summary"] == pytest.approx(100 * 982 / 1351) + # `ast_summary` is over the four AST categories only: the two relevance sets + # join at the overall, and folding them in here would make it the headline. + assert live_report["n_ast"] == 1351 + assert live_report["n_problems"] == 2251 + assert live_report["ci95_units"]["ast_summary"] == "n_ast" + + +@pytest.mark.anyio +async def test_every_live_interval_brackets_the_rate_printed_beside_it(live_report): + """The published rate and its bounds have to be in the same units. + + Both come off the same per-sample list: the rate is that list's mean scaled + to percentage points, the bounds are estimated from it as a probability and + scaled the same way. Scale the list instead of the rate and the estimator + reads every rate above one point as a saturated set, which still publishes + a triple -- a plausible-looking interval that brackets 100 rather than the + number it is printed beside. + """ + units = live_report["ci95_units"] + assert units, "a live report with no declared interval proves nothing here" + for metric in units: + low, high = live_report[f"{metric}_ci95"] + assert low < live_report[metric] < high, ( + f"{metric} is {live_report[metric]} but its interval is [{low}, {high}]" + ) + assert 0.0 <= low < high <= 100.0 + + +@pytest.mark.anyio +async def test_a_missing_sample_scores_wrong_not_excluded(non_live_report_short): + # 399 of simple's 400 came back, all correct. The denominator is still 400. + assert non_live_report_short["simple"] == pytest.approx(100 * 399 / 400) + assert non_live_report_short["n_simple"] == 400 + + +@pytest.mark.anyio +async def test_a_failed_sample_moves_nothing_but_the_fail_count( + non_live_report_short, non_live_report_with_fails +): + """The denominator is DECLARED, so charging a failure is already done. + + A failed sample has no category to be attributed to -- one that failed in + `preprocess` never read a row -- and under this policy it does not need + one: it is missing from its category's numerator and the denominator never + moved. So `fails` is the only key that reacts to it. + """ + assert non_live_report_short["fails"] == 0 + assert non_live_report_with_fails["fails"] == 1 + assert non_live_report_with_fails["score"] == non_live_report_short["score"] + assert non_live_report_with_fails["simple"] == non_live_report_short["simple"] + + +@pytest.mark.anyio +async def test_both_groups_declare_the_column_and_the_population( + non_live_report, live_report +): + """`score_key` is built from the group key, so no static reader resolves it. + + It names a key the report writes, and the report is the only place that can + be checked -- which is why it is asserted here rather than left to the + preflight, whose rule 4 skips a computed key. + """ + for report, group in ((non_live_report, "non_live"), (live_report, "live")): + assert report[SCORE_KEY_FIELD] == f"{group}_overall_acc" + assert report[SCORE_KEY_FIELD] in report + assert report[report[SCORE_KEY_FIELD]] == report["score"] + assert report[DENOMINATOR_FIELD] == DENOMINATOR_REQUESTED From f3d6a9257e8e53a423214057e4808efd6f74dc5c Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 21:00:29 +0800 Subject: [PATCH 15/28] feat(tasks): register the four BFCL v3 single-turn tasks Two groups upstream scores separately, each under both published protocols. The unqualified name is the prompting protocol: every model on the leaderboard has a Prompt row, while FC exists only where the provider exposes a tools API -- and FC gates on a capability, so giving it the default name would make the default the narrow one. Every stage lives in the already-shared `_base.py`, so each leaf is its metadata plus a base pair: a protocol mixin and a group base. The family test asserts both halves separately, because they fail differently -- a leaf on the wrong group base still binds the right dataset and still demands the right capability, and would publish an unweighted mean of category rates over live rows under a live-sounding score key. The FC leaves' capability refusal is `Task.__init__`'s, reached at prelaunch reconciliation before a request is made; static schema validation does not import tasks and so does not see it. Asserted rather than described, since no implemented chat dialect denies the capability today and the claim would otherwise be unfalsifiable. All four ship experimental: the licence does not force it, but nothing has been aligned against a published BFCL number yet. Co-Authored-By: Claude Opus 5 --- sieval/meta/index.json | 84 +++++++++++ sieval/tasks/__init__.pyi | 16 +++ .../tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py | 70 +++++++++ .../bfcl_v3/bfcl_v3_live_0shot_gen_fc.py | 83 +++++++++++ .../bfcl_v3/bfcl_v3_non_live_0shot_gen.py | 73 ++++++++++ .../bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py | 86 +++++++++++ .../unit/tasks/bfcl_v3/test_bfcl_v3_family.py | 136 ++++++++++++++++++ .../tasks/test_import_discipline_family.py | 11 ++ 8 files changed, 559 insertions(+) create mode 100644 sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py create mode 100644 sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py create mode 100644 sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py create mode 100644 sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py create mode 100644 tests/unit/tasks/bfcl_v3/test_bfcl_v3_family.py diff --git a/sieval/meta/index.json b/sieval/meta/index.json index ef88e888..cdd109f1 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -1778,6 +1778,90 @@ "status": "experimental", "reference_kind": "value" }, + { + "name": "bfcl_v3_live_0shot_gen", + "display_name": "BFCL v3 Live (0-shot, prompting)", + "description": "Berkeley Function Calling Leaderboard v3, live categories, upstream's prompting protocol.", + "dataset": "bfcl_v3_live", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "function-calling" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "gorilla/berkeley-function-call-leaderboard", + "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored verbatim from gorilla v1.3 (Apache-2.0); only `bfcl_eval.*` imports were rewritten. exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + }, + "status": "experimental", + "reference_kind": "value" + }, + { + "name": "bfcl_v3_live_0shot_gen_fc", + "display_name": "BFCL v3 Live (0-shot, FC)", + "description": "Berkeley Function Calling Leaderboard v3, live categories, upstream's FC (tools API) protocol.", + "dataset": "bfcl_v3_live", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "function-calling" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "gorilla/berkeley-function-call-leaderboard", + "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored verbatim from gorilla v1.3 (Apache-2.0); only `bfcl_eval.*` imports were rewritten. exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + }, + "status": "experimental", + "reference_kind": "value" + }, + { + "name": "bfcl_v3_non_live_0shot_gen", + "display_name": "BFCL v3 Non-Live (0-shot, prompting)", + "description": "Berkeley Function Calling Leaderboard v3, non-live categories, upstream's prompting protocol.", + "dataset": "bfcl_v3_non_live", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "function-calling" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "gorilla/berkeley-function-call-leaderboard", + "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored verbatim from gorilla v1.3 (Apache-2.0); only `bfcl_eval.*` imports were rewritten. exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + }, + "status": "experimental", + "reference_kind": "value" + }, + { + "name": "bfcl_v3_non_live_0shot_gen_fc", + "display_name": "BFCL v3 Non-Live (0-shot, FC)", + "description": "Berkeley Function Calling Leaderboard v3, non-live categories, upstream's FC (tools API) protocol.", + "dataset": "bfcl_v3_non_live", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "function-calling" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "gorilla/berkeley-function-call-leaderboard", + "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored verbatim from gorilla v1.3 (Apache-2.0); only `bfcl_eval.*` imports were rewritten. exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + }, + "status": "experimental", + "reference_kind": "value" + }, { "name": "browsecomp_0shot_gen", "display_name": "BrowseComp (0-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index dc8656f4..115f9da7 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -40,6 +40,18 @@ from .arc.arc_easy_kshot_clp import ( from .arc.arc_easy_kshot_ppl import ( ARCEasyFewShotPplTask, ) +from .bfcl_v3.bfcl_v3_live_0shot_gen import ( + BfclV3LiveZeroShotGenTask, +) +from .bfcl_v3.bfcl_v3_live_0shot_gen_fc import ( + BfclV3LiveZeroShotGenFCTask, +) +from .bfcl_v3.bfcl_v3_non_live_0shot_gen import ( + BfclV3NonLiveZeroShotGenTask, +) +from .bfcl_v3.bfcl_v3_non_live_0shot_gen_fc import ( + BfclV3NonLiveZeroShotGenFCTask, +) from .browsecomp_0shot_gen import ( BrowseCompZeroShotGenTask, ) @@ -257,6 +269,10 @@ __all__ = [ "Apex2025ZeroShotGenTask", "ApexShortlist2025ZeroShotGenTask", "BRUMO2025ZeroShotGenTask", + "BfclV3LiveZeroShotGenFCTask", + "BfclV3LiveZeroShotGenTask", + "BfclV3NonLiveZeroShotGenFCTask", + "BfclV3NonLiveZeroShotGenTask", "BrowseCompZeroShotGenTask", "CEvalFewShotCLPTask", "CMIMC2025ZeroShotGenTask", diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py new file mode 100644 index 00000000..dc8aa205 --- /dev/null +++ b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py @@ -0,0 +1,70 @@ +"""BFCL v3 live categories, 0-shot, upstream's prompting protocol. + +Six user-contributed categories, 2251 rows: four Python AST categories plus +irrelevance and relevance detection. The headline is upstream's `Live Overall +Acc` -- the SAMPLE-COUNT-WEIGHTED mean of the six, which is algebraically the +pooled rate over all 2251 rows. That is why this group publishes an interval +where the non-live group cannot: a weighted mean of category rates over their +own row counts is the rate over their union, so a problem-clustered interval +brackets the number printed beside it. `ast_summary`, weighted over the four AST +categories, carries one on `n_ast` for the same reason. + +This is the `(Prompt)` column: the function schemas arrive in the system turn +and the reply is parsed with `ast_parse`. The `_fc` sibling is the `(FC)` +column, where the same schemas arrive as tools. The unqualified name is this one +because every model on the leaderboard has a Prompt row, while an FC row exists +only where the provider exposes a tools API -- and because FC gates on the +`function_tools` capability, which would make the default name the narrow one. + +References: + +* Blog: +* Harness: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import BfclV3LiveDatasetSample + +from ._base import ( + BFCL_V3_HARNESS_URL, + BFCL_V3_SHARED_NOTES, + BfclV3LiveTask, + BfclV3PromptMixin, +) + + +@sieval_task( + name="bfcl_v3_live_0shot_gen", + display_name="BFCL v3 Live (0-shot, prompting)", + description=( + "Berkeley Function Calling Leaderboard v3, live categories, " + "upstream's prompting protocol." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "function-calling"), + model_type="chat", + status="experimental", + reference_kind="value", + reference_impl=ReferenceImpl( + source="gorilla/berkeley-function-call-leaderboard", + url=BFCL_V3_HARNESS_URL, + notes=( + "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. " + "Headline is `Live Overall Acc`: the sample-count-weighted mean of " + "six categories, which is algebraically the pooled rate over all " + "2251 live rows, so it carries a problem-clustered interval; " + "`ast_summary` (weighted over the four AST categories) carries one " + "on `n_ast` for the same reason. underscore_to_dot=False: schemas " + "reach the model verbatim, so dotted function names survive and " + "gold needs no rewrite. experimental until an alignment run " + f"against a published BFCL v3 number lands. {BFCL_V3_SHARED_NOTES}" + ), + ), +) +class BfclV3LiveZeroShotGenTask( + BfclV3PromptMixin, BfclV3LiveTask[BfclV3LiveDatasetSample] +): + pass diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py new file mode 100644 index 00000000..1239ee7a --- /dev/null +++ b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py @@ -0,0 +1,83 @@ +"""BFCL v3 live categories, 0-shot, upstream's function-calling protocol. + +Six user-contributed categories, 2251 rows: four Python AST categories plus +irrelevance and relevance detection. The headline is upstream's `Live Overall +Acc` -- the SAMPLE-COUNT-WEIGHTED mean of the six, which is algebraically the +pooled rate over all 2251 rows. That is why this group publishes an interval +where the non-live group cannot: a weighted mean of category rates over their +own row counts is the rate over their union, so a problem-clustered interval +brackets the number printed beside it. `ast_summary`, weighted over the four AST +categories, carries one on `n_ast` for the same reason. + +This is the `(FC)` column: the same schemas reach the model through the +provider's tools API, and the calls are read back off structured tool calls +rather than parsed out of the reply. The sibling without the suffix is the +`(Prompt)` column, and it holds the unqualified name because every model on the +leaderboard has a Prompt row, while an FC row exists only where the provider +exposes a tools API -- and because this task gates on the `function_tools` +capability, which would make the default name the narrow one. + +One scoring divergence follows from the transport and it is upstream's. A native +tool name cannot carry a dot in the OpenAI dialects, so `convert_to_tool` +rewrites `geometry.triangle_area` to `geometry_triangle_area` on the way out; +gold still spells the dot, so the comparison rewrites gold to match. That is +`underscore_to_dot=True`, and it is the only behavioural difference from the +Prompt sibling beyond how the schemas and the calls travel. + +References: + +* Blog: +* Harness: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import BfclV3LiveDatasetSample + +from ._base import ( + BFCL_V3_HARNESS_URL, + BFCL_V3_SHARED_NOTES, + BfclV3FCMixin, + BfclV3LiveTask, +) + + +@sieval_task( + name="bfcl_v3_live_0shot_gen_fc", + display_name="BFCL v3 Live (0-shot, FC)", + description=( + "Berkeley Function Calling Leaderboard v3, live categories, " + "upstream's FC (tools API) protocol." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "function-calling"), + model_type="chat", + status="experimental", + reference_kind="value", + reference_impl=ReferenceImpl( + source="gorilla/berkeley-function-call-leaderboard", + url=BFCL_V3_HARNESS_URL, + notes=( + "FC protocol -- upstream's `(FC)` leaderboard column. Headline is " + "`Live Overall Acc`: the sample-count-weighted mean of six " + "categories, which is algebraically the pooled rate over all 2251 " + "live rows, so it carries a problem-clustered interval; " + "`ast_summary` (weighted over the four AST categories) carries one " + "on `n_ast` for the same reason. underscore_to_dot=True: " + "convert_to_tool rewrote `.` to `_` in every function name on the " + "way out, because a native tool name cannot carry a dot, so gold is " + "rewritten to match before comparison. Requires the " + "`function_tools` capability; a binding that does not offer it is " + "rejected when the task is constructed, which prelaunch " + "reconciliation reaches before the first request and before a " + "result directory exists. experimental until an alignment run " + f"against a published BFCL v3 number lands. {BFCL_V3_SHARED_NOTES}" + ), + ), +) +class BfclV3LiveZeroShotGenFCTask( + BfclV3FCMixin, BfclV3LiveTask[BfclV3LiveDatasetSample] +): + pass diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py new file mode 100644 index 00000000..8b081a96 --- /dev/null +++ b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py @@ -0,0 +1,73 @@ +"""BFCL v3 non-live categories, 0-shot, upstream's prompting protocol. + +Seven hand-written categories, 1390 rows: four Python AST categories, Java and +JavaScript simple-call categories, and irrelevance detection. The headline is +upstream's `Non_Live Overall Acc` -- the UNWEIGHTED mean of five numbers, one of +which (`simple_ast`) is itself the unweighted mean of three. It publishes no +confidence interval for that reason: it is a mean of rates, not the pooled rate +over 1390 rows, and an interval computed over those rows would bracket a +different statistic than the one printed beside it. Every per-category rate does +carry one. + +This is the `(Prompt)` column: the function schemas arrive in the system turn +and the reply is parsed with `ast_parse`. The `_fc` sibling is the `(FC)` +column, where the same schemas arrive as tools. The unqualified name is this one +because every model on the leaderboard has a Prompt row, while an FC row exists +only where the provider exposes a tools API -- and because FC gates on the +`function_tools` capability, which would make the default name the narrow one. + +References: + +* Blog: +* Harness: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import BfclV3NonLiveDatasetSample + +from ._base import ( + BFCL_V3_HARNESS_URL, + BFCL_V3_SHARED_NOTES, + BfclV3NonLiveTask, + BfclV3PromptMixin, +) + + +@sieval_task( + name="bfcl_v3_non_live_0shot_gen", + display_name="BFCL v3 Non-Live (0-shot, prompting)", + description=( + "Berkeley Function Calling Leaderboard v3, non-live categories, " + "upstream's prompting protocol." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "function-calling"), + model_type="chat", + status="experimental", + reference_kind="value", + reference_impl=ReferenceImpl( + source="gorilla/berkeley-function-call-leaderboard", + url=BFCL_V3_HARNESS_URL, + notes=( + "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. " + "Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast " + "(itself the unweighted mean of simple/java/javascript), multiple, " + "parallel, parallel_multiple and irrelevance. NO interval is " + "published on the headline, nor on simple_ast or ast_summary: an " + "unweighted mean of category rates is not the pooled rate over the " + "rows (javascript's 50 weigh as much as simple's 400), so a " + "sample-clustered CI would bracket a different statistic. Every " + "per-category rate carries its own. underscore_to_dot=False: " + "schemas reach the model verbatim, so dotted function names survive " + "and gold needs no rewrite. experimental until an alignment run " + f"against a published BFCL v3 number lands. {BFCL_V3_SHARED_NOTES}" + ), + ), +) +class BfclV3NonLiveZeroShotGenTask( + BfclV3PromptMixin, BfclV3NonLiveTask[BfclV3NonLiveDatasetSample] +): + pass diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py new file mode 100644 index 00000000..09943fed --- /dev/null +++ b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py @@ -0,0 +1,86 @@ +"""BFCL v3 non-live categories, 0-shot, upstream's function-calling protocol. + +Seven hand-written categories, 1390 rows: four Python AST categories, Java and +JavaScript simple-call categories, and irrelevance detection. The headline is +upstream's `Non_Live Overall Acc` -- the UNWEIGHTED mean of five numbers, one of +which (`simple_ast`) is itself the unweighted mean of three. It publishes no +confidence interval for that reason: it is a mean of rates, not the pooled rate +over 1390 rows, and an interval computed over those rows would bracket a +different statistic than the one printed beside it. Every per-category rate does +carry one. + +This is the `(FC)` column: the same schemas reach the model through the +provider's tools API, and the calls are read back off structured tool calls +rather than parsed out of the reply. The sibling without the suffix is the +`(Prompt)` column, and it holds the unqualified name because every model on the +leaderboard has a Prompt row, while an FC row exists only where the provider +exposes a tools API -- and because this task gates on the `function_tools` +capability, which would make the default name the narrow one. + +One scoring divergence follows from the transport and it is upstream's. A native +tool name cannot carry a dot in the OpenAI dialects, so `convert_to_tool` +rewrites `geometry.triangle_area` to `geometry_triangle_area` on the way out; +gold still spells the dot, so the comparison rewrites gold to match. That is +`underscore_to_dot=True`, and it is the only behavioural difference from the +Prompt sibling beyond how the schemas and the calls travel. + +References: + +* Blog: +* Harness: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import BfclV3NonLiveDatasetSample + +from ._base import ( + BFCL_V3_HARNESS_URL, + BFCL_V3_SHARED_NOTES, + BfclV3FCMixin, + BfclV3NonLiveTask, +) + + +@sieval_task( + name="bfcl_v3_non_live_0shot_gen_fc", + display_name="BFCL v3 Non-Live (0-shot, FC)", + description=( + "Berkeley Function Calling Leaderboard v3, non-live categories, " + "upstream's FC (tools API) protocol." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "function-calling"), + model_type="chat", + status="experimental", + reference_kind="value", + reference_impl=ReferenceImpl( + source="gorilla/berkeley-function-call-leaderboard", + url=BFCL_V3_HARNESS_URL, + notes=( + "FC protocol -- upstream's `(FC)` leaderboard column. Headline is " + "`Non_Live Overall Acc`: unweighted mean of simple_ast (itself the " + "unweighted mean of simple/java/javascript), multiple, parallel, " + "parallel_multiple and irrelevance. NO interval is published on the " + "headline, nor on simple_ast or ast_summary: an unweighted mean of " + "category rates is not the pooled rate over the rows (javascript's " + "50 weigh as much as simple's 400), so a sample-clustered CI would " + "bracket a different statistic. Every per-category rate carries its " + "own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in " + "every function name on the way out, because a native tool name " + "cannot carry a dot, so gold is rewritten to match before " + "comparison. Requires the `function_tools` capability; a binding " + "that does not offer it is rejected when the task is constructed, " + "which prelaunch reconciliation reaches before the first request " + "and before a result directory exists. experimental until an " + f"alignment run against a published BFCL v3 number lands. " + f"{BFCL_V3_SHARED_NOTES}" + ), + ), +) +class BfclV3NonLiveZeroShotGenFCTask( + BfclV3FCMixin, BfclV3NonLiveTask[BfclV3NonLiveDatasetSample] +): + pass diff --git a/tests/unit/tasks/bfcl_v3/test_bfcl_v3_family.py b/tests/unit/tasks/bfcl_v3/test_bfcl_v3_family.py new file mode 100644 index 00000000..28ac8cf7 --- /dev/null +++ b/tests/unit/tasks/bfcl_v3/test_bfcl_v3_family.py @@ -0,0 +1,136 @@ +"""One contract over the four registered BFCL v3 leaves. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from types import SimpleNamespace + +import pytest + +from sieval.core.tasks.meta import get_task_class, get_task_meta +from sieval.tasks.bfcl_v3._base import BfclV3Task +from tests.conftest import MockChatModel, MockDataset + +NAMES = ( + "bfcl_v3_non_live_0shot_gen", + "bfcl_v3_non_live_0shot_gen_fc", + "bfcl_v3_live_0shot_gen", + "bfcl_v3_live_0shot_gen_fc", +) + + +@pytest.mark.parametrize("name", NAMES) +def test_every_leaf_resolves_by_name(name): + """A nested task that imports but does not RESOLVE is the failure mode the + subpackage rule exists for -- a bare KeyError from `sieval task show`.""" + assert get_task_class(name) is not None + + +@pytest.mark.parametrize("name", NAMES) +def test_every_leaf_declares_the_metadata_the_rules_require(name): + meta = get_task_meta(get_task_class(name)) + assert meta.model_type == "chat" + assert meta.reference_kind == "value" + assert meta.n_shot == 0 + # `reference_impl` is optional on `TaskMeta`, so the narrowing is an + # assertion in its own right: a leaf that shipped without one would + # otherwise fail on an attribute error rather than on the thing under test. + assert meta.reference_impl is not None + assert meta.reference_impl.url.endswith( + "ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard" + ) + + +@pytest.mark.parametrize( + ("name", "expected"), + [(n, n.endswith("_fc")) for n in NAMES], +) +def test_only_the_fc_leaves_demand_the_tools_capability(name, expected): + # Direct attribute access, not `getattr(..., False)`: `requires` is a + # `ClassVar[TaskRequirements]` that every task inherits, so a default would + # turn Task 1's field going missing into a silently passing assertion for + # the two Prompt leaves. + cls = get_task_class(name) + assert cls.requires.function_tools is expected + + +class _ToollessChatModel(MockChatModel): + """A chat model whose runtime plan offers no `function_tools`. + + Not reachable through a registered dialect today -- the only one that denies + the capability is `openai_completions`, whose completion input kind trips a + different check first -- so the runtime-plan branch is how the gate is + exercised at all. + """ + + @property + def runtime_plan(self): + return SimpleNamespace( + dialect_id="openai_chat", + available_capabilities=frozenset(), + capability_minimums={}, + ) + + +def test_an_fc_leaf_is_refused_at_construction_when_tools_are_unavailable(): + """What the FC leaves' notes claim, asserted where it happens. + + The refusal is `Task.__init__`'s, so it lands at prelaunch reconciliation -- + before a request is made and before a result directory exists. Static schema + validation does NOT see it: `validate_eval_config` runs without importing or + instantiating a task, so a config naming an FC leaf passes it clean. + """ + with pytest.raises(ValueError, match="function_tools"): + get_task_class("bfcl_v3_live_0shot_gen_fc")(MockDataset(), _ToollessChatModel()) + + +def test_the_prompt_leaves_construct_on_the_same_toolless_model(): + """The other half of the pair: without it, a gate that refused EVERY model + would pass the test above just as happily.""" + get_task_class("bfcl_v3_live_0shot_gen")(MockDataset(), _ToollessChatModel()) + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("bfcl_v3_non_live_0shot_gen", "bfcl_v3_non_live"), + ("bfcl_v3_non_live_0shot_gen_fc", "bfcl_v3_non_live"), + ("bfcl_v3_live_0shot_gen", "bfcl_v3_live"), + ("bfcl_v3_live_0shot_gen_fc", "bfcl_v3_live"), + ], +) +def test_the_two_groups_bind_different_datasets(name, expected): + """The FK is not declared -- `@sieval_task` resolves it by reverse lookup + from the sample `TypedDict` the leaf binds, so this is what proves the two + groups really did bind different sample types rather than both inheriting + one.""" + assert get_task_meta(get_task_class(name)).dataset == expected + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("bfcl_v3_non_live_0shot_gen", "non_live"), + ("bfcl_v3_non_live_0shot_gen_fc", "non_live"), + ("bfcl_v3_live_0shot_gen", "live"), + ("bfcl_v3_live_0shot_gen_fc", "live"), + ], +) +def test_each_leaf_rolls_up_under_its_own_group(name, expected): + """Which group base a leaf inherits is a separate fact from which dataset it + binds, and only this one is about the arithmetic. + + The two bases differ in how the headline is computed -- the non-live one + takes an unweighted mean of five category rates and publishes no interval, + the live one pools over rows and does -- and `GROUP_KEY` is what selects the + rollup and names `score_key`. Nothing else here would notice a leaf wired to + the wrong base while still binding the right sample type: the dataset FK + comes from the generic arg, the capability from the protocol mixin, and the + rollups are exercised against the bases rather than through the leaves. A + live leaf on the non-live base would publish a mean of rates over live data + under a live-sounding score key, and pass everything else. + """ + cls = get_task_class(name) + assert issubclass(cls, BfclV3Task) + group_key = cls.GROUP_KEY + assert group_key == expected diff --git a/tests/unit/tasks/test_import_discipline_family.py b/tests/unit/tasks/test_import_discipline_family.py index e84ef93d..d4dfedbe 100644 --- a/tests/unit/tasks/test_import_discipline_family.py +++ b/tests/unit/tasks/test_import_discipline_family.py @@ -32,6 +32,17 @@ # task module (under `sieval.tasks.`) -> modules its registration must not pull. FORBIDDEN: dict[str, tuple[str, ...]] = { + # `ast_parse` reaches tree-sitter, which is behind the optional `bfcl-v3` + # extra. It is needed only by the Prompt protocol -- but there by every + # category, not just java/javascript: `ast_parse` imports both source + # parsers at module scope, so touching it at all pulls them. All four + # leaves are listed because they share `_base.py`, so a module-scope import + # there would pull tree-sitter for the FC leaves too, which never parse + # source text at all. + "bfcl_v3.bfcl_v3_non_live_0shot_gen": ("tree_sitter",), + "bfcl_v3.bfcl_v3_non_live_0shot_gen_fc": ("tree_sitter",), + "bfcl_v3.bfcl_v3_live_0shot_gen": ("tree_sitter",), + "bfcl_v3.bfcl_v3_live_0shot_gen_fc": ("tree_sitter",), # math_verify is behind the `math` group and is slow to import. "aime_2024_0shot_gen": ("math_verify",), "aime_2025_0shot_gen": ("math_verify",), From aa85279e686f9179a2abd1207b24d6f9d1f7ecce Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 21:15:21 +0800 Subject: [PATCH 16/28] fix(tasks): declare the tree-sitter extra on the two BFCL v3 Prompt leaves The Prompt protocol decodes replies with `ast_parse`, whose module imports both tree-sitter source parsers at module scope. The import therefore runs before the per-language dispatch does, so the extra is needed for every category rather than just `java` and `javascript` -- including the live group, whose categories are all Python. The FC leaves parse no source text and need nothing, so they keep `deps_group=None`. Without this, the readiness probe never inspects the task-deps axis (`extras_unsatisfied` is only consulted when a `deps_group` is declared) and reports `yes` on a base install. Because the import is deferred into `_decode`, the run then bills for inference before every sample dies at postprocess. `deps_group` is a frozen field on `TaskMeta`, so declaring it after these tasks ship would be a schema-contract change rather than an edit. The test asserts the mapping in both directions, since both faults are silent in production: dropping it from a Prompt leaf buys the wasted run above, and adding it to an FC leaf leaves that leaf reporting NOT ready -- unrunnable over a dependency it never loads. Co-Authored-By: Claude Opus 5 --- sieval/meta/index.json | 4 ++-- .../tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py | 7 ++++++ .../bfcl_v3/bfcl_v3_non_live_0shot_gen.py | 7 ++++++ .../unit/tasks/bfcl_v3/test_bfcl_v3_family.py | 22 +++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index cdd109f1..a5e67674 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -1789,7 +1789,7 @@ "english", "function-calling" ], - "deps_group": null, + "deps_group": "bfcl-v3", "model_type": "chat", "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", @@ -1831,7 +1831,7 @@ "english", "function-calling" ], - "deps_group": null, + "deps_group": "bfcl-v3", "model_type": "chat", "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py index dc8aa205..bf1508ae 100644 --- a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py +++ b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py @@ -45,6 +45,13 @@ eval_mode=EvalMode.GEN, n_shot=0, tags=("english", "function-calling"), + # Needed even though every live category is Python, which is the part that + # reads wrong: `_decode` calls `ast_parse`, and its module imports both + # tree-sitter source parsers at module scope, so the import is paid before + # the language dispatch ever runs. Not on the `_fc` sibling -- FC parses no + # source text. Without this, readiness reports `yes`, the run bills for + # inference, and then every sample dies at postprocess. + deps_group="bfcl-v3", model_type="chat", status="experimental", reference_kind="value", diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py index 8b081a96..4dd9c7e3 100644 --- a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py +++ b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py @@ -45,6 +45,13 @@ eval_mode=EvalMode.GEN, n_shot=0, tags=("english", "function-calling"), + # Declared here and NOT on the `_fc` sibling: `_decode` calls `ast_parse`, + # whose module imports both tree-sitter source parsers at module scope, so + # the Prompt protocol needs the extra for every category rather than just + # `java` and `javascript`. FC never parses source text. Without this the + # import is deferred far enough that readiness reports `yes`, the run bills + # for inference, and then every sample dies at postprocess. + deps_group="bfcl-v3", model_type="chat", status="experimental", reference_kind="value", diff --git a/tests/unit/tasks/bfcl_v3/test_bfcl_v3_family.py b/tests/unit/tasks/bfcl_v3/test_bfcl_v3_family.py index 28ac8cf7..45bc435a 100644 --- a/tests/unit/tasks/bfcl_v3/test_bfcl_v3_family.py +++ b/tests/unit/tasks/bfcl_v3/test_bfcl_v3_family.py @@ -54,6 +54,28 @@ def test_only_the_fc_leaves_demand_the_tools_capability(name, expected): assert cls.requires.function_tools is expected +@pytest.mark.parametrize( + ("name", "expected"), + [(n, None if n.endswith("_fc") else "bfcl-v3") for n in NAMES], +) +def test_only_the_prompt_leaves_declare_the_tree_sitter_extra(name, expected): + """The extra follows the protocol, and the split is not the obvious one. + + Both Prompt leaves need it -- including the live one, whose every category + is Python -- because `ast_parse`'s module imports both tree-sitter source + parsers at module scope, so the cost is paid before the language dispatch + runs. Neither FC leaf does: it reads structured calls and parses no source. + + Asserted in both directions, because both are silent in production. Dropping + it from a Prompt leaf makes readiness report `yes` on a base install, and + since the import is deferred into `_decode`, the run bills for inference + before every sample dies at postprocess. Adding it to an FC leaf is the + quieter fault: that leaf then reports NOT ready, and stays unrunnable, over + a dependency it never loads. + """ + assert get_task_meta(get_task_class(name)).deps_group == expected + + class _ToollessChatModel(MockChatModel): """A chat model whose runtime plan offers no `function_tools`. From e2a1c5f427b40ff61134c5b304028839107e04b2 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 21:24:04 +0800 Subject: [PATCH 17/28] test(tasks): replay BFCL v3 gold as the reply to check the grader wiring Replaying each row's own possible_answer must grade correct, which separates "is the port wired up right" from "is the model good" -- a question no model run can answer. 2344 of 2351 Python rows do. The replay reads the function schema, not just gold: a `""` among a parameter's accepted values licenses omitting it, but only where the schema does not mark it required, and gold sometimes offers a parameter the schema never declares. Gold also nests alternatives through both dicts and lists. The 7 residuals are all upstream gold contradicting upstream's own schema and are asserted BY NAME, so a snapshot that fixes one fails here rather than passing quietly: simple_363 gold calls it find_closest, the schema calls it restaurant_search.find_closest live_simple_106-63-0 gold offers "" (omit) for a required parameter live_simple_112-68-0 same live_multiple_507-149-4 same live_multiple_964-207-0 same live_multiple_862-181-3 gold names a parameter the schema never declares live_multiple_1038-265-0 an optional parameter with no default, which the checker then demands Java and JavaScript are excluded: the checker requires their arguments as raw source text for the type converters to parse, while gold holds native values, so a replay would have to serialize Java and JS inside a test. Co-Authored-By: Claude Opus 5 --- tests/acceptance/bfcl_v3/test_gold_replay.py | 151 +++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 tests/acceptance/bfcl_v3/test_gold_replay.py diff --git a/tests/acceptance/bfcl_v3/test_gold_replay.py b/tests/acceptance/bfcl_v3/test_gold_replay.py new file mode 100644 index 00000000..ecf666b9 --- /dev/null +++ b/tests/acceptance/bfcl_v3/test_gold_replay.py @@ -0,0 +1,151 @@ +"""Replay BFCL v3 gold as the reply, and require the grader to accept it. + +Each row's `possible_answer` IS an accepted answer, so grading it must return +correct. That checks the dataset join, the category routing and the vendored +checker together, against real data, with no model spend -- and it is the only +check here that can distinguish "the port is wired up right" from "the model is +good". + +Restricted to `language == "Python"`. Java and JavaScript gold is not in the +decoded form: `ast_checker` requires every parameter there to arrive as a raw +source-text string for the type converters to parse, while gold holds native +values. Replaying it would mean writing a Java/JS serializer inside a test, and +its bugs would be indistinguishable from grader bugs. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json +import pathlib + +import pytest + +from sieval.community.bfcl_v3 import GOLDLESS_CATEGORIES +from sieval.core.utils.hf import maybe_resolve_hf_path +from sieval.datasets import BfclV3LiveDataset, BfclV3NonLiveDataset +from sieval.tasks.bfcl_v3._base import grade_single_turn + +#: The snapshot both datasets are pinned to; `maybe_resolve_hf_path` rewrites it +#: to `$SIEVAL_DATA_DIR/gorilla-llm/...`. +SNAPSHOT = "gorilla-llm/Berkeley-Function-Calling-Leaderboard" + +#: Rows where upstream's gold contradicts upstream's own schema, so no reply can +#: be both the gold answer and schema-valid. Each was read individually; see the +#: table in this module's commit message. Asserted by EQUALITY: a pin bump that +#: fixes one must fail here and be updated in the same commit. +UPSTREAM_GOLD_DEFECTS = frozenset( + { + "simple_363", + "live_simple_106-63-0", + "live_simple_112-68-0", + "live_multiple_507-149-4", + "live_multiple_862-181-3", + "live_multiple_964-207-0", + "live_multiple_1038-265-0", + } +) + + +def _resolve(node): + """Gold nests alternatives inside dicts AND lists; unwrap every level.""" + if isinstance(node, dict): + return { + key: _first(accepted) + for key, accepted in node.items() + if _first(accepted) is not None + } + if isinstance(node, list): + return [_resolve(item) for item in node] + return node + + +def _first(accepted): + """One concrete value out of a gold alternatives list. + + A leaf is *usually* a list of acceptable values, but three leaves in 3641 + rows are a bare scalar, so a non-list is passed through rather than indexed. + `""` means "omitting is acceptable" and is skipped here; whether omission is + actually permitted is the schema's call, made in `_replay`. + """ + if not isinstance(accepted, list): + return accepted + for candidate in accepted: + if candidate != "": + return _resolve(candidate) + return None + + +def _replay(gold_entry: list, functions: list) -> list[dict]: + """Turn one `ground_truth` entry into the decoded form a model would yield. + + Reads the schema because gold alone is ambiguous: `""` among a parameter's + alternatives licenses omitting it, but only where the schema does not mark it + required -- and some parameters gold offers are not in the schema at all, so + naming them is an `unexpected_param`. + """ + schemas = {fn["name"]: fn for fn in functions} + replayed = [] + for call in gold_entry: + for name, params in call.items(): + parameters = schemas.get(name, {}).get("parameters", {}) + required = set(parameters.get("required", ())) + args = {} + for param, accepted in params.items(): + if not accepted: + continue + # `""` licenses omitting the parameter, but only where the + # schema does not require it -- dropping a required one is + # `missing_required`, and emitting one the schema never declares + # is `unexpected_param`, so both sides need the schema. + omittable = isinstance(accepted, list) and "" in accepted + if omittable and param not in required: + continue + value = _first(accepted) + if value is not None: + args[param] = value + replayed.append({name: args}) + return replayed + + +@pytest.mark.skipif( + not pathlib.Path(maybe_resolve_hf_path(SNAPSHOT)).is_dir(), + reason=f"needs the downloaded BFCL v3 snapshot ({SNAPSHOT})", +) +def test_replaying_gold_as_the_reply_grades_correct(): + missed = set() + graded = 0 + for dataset in (BfclV3NonLiveDataset(SNAPSHOT), BfclV3LiveDataset(SNAPSHOT)): + test_set = dataset.test_set + # Bound rather than iterated inline: the property is `HFDataset | None`, + # and a missing split would otherwise walk zero rows on the way to the + # count assertion below, reporting a moved pin for an absent split. + assert test_set is not None, f"{type(dataset).__name__} has no test split" + for row in test_set: + if row["category"] in GOLDLESS_CATEGORIES or row["language"] != "Python": + continue + graded += 1 + decoded = _replay( + json.loads(row["ground_truth"]), json.loads(row["function"]) + ) + if not grade_single_turn( + row["function"], + decoded, + row["ground_truth"], + row["language"], + row["category"], + False, + ): + missed.add(row["id"]) + + assert graded == 2351, ( + f"Expected 2351 gradeable Python rows, walked {graded}. The snapshot pin " + "moved, or the goldless/language filter changed -- either way the defect " + "list below was measured against a different corpus." + ) + assert missed == UPSTREAM_GOLD_DEFECTS, ( + "Replaying gold as the reply must grade correct.\n" + f" newly failing (OUR defect, fix it): " + f"{sorted(missed - UPSTREAM_GOLD_DEFECTS)}\n" + f" newly passing (upstream fixed it, update the set): " + f"{sorted(UPSTREAM_GOLD_DEFECTS - missed)}" + ) From 82c93cba6296d7fcc1f260dbc004c98e90e1ef2f Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 21:40:30 +0800 Subject: [PATCH 18/28] docs(tasks): add the _fc variant row to the naming vocabulary _fc earns a vocabulary row because it is a different axis from _parse: it changes which model capability is exercised, not how output is extracted, so it changes which models can run the task at all. Co-Authored-By: Claude Opus 5 --- sieval/tasks/CLAUDE.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sieval/tasks/CLAUDE.md b/sieval/tasks/CLAUDE.md index b2fc40d5..1ce09813 100644 --- a/sieval/tasks/CLAUDE.md +++ b/sieval/tasks/CLAUDE.md @@ -25,6 +25,7 @@ name, so it is the only place the distinction can live. | *(none)* | Tracks upstream — its protocol, its grader, its defects | | `_fixed` | Ours, diverging to repair a defect in upstream's grader or data | | `_parse` | Upstream's *other* published protocol over the same benchmark and grader: same data, same scoring, a different prompt and an output extractor, each with its own published column | +| `_fc` | Upstream's native-function-calling protocol over the same benchmark and grader: same data, same scoring, the function schemas delivered through the provider's tools API instead of the prompt, and calls read back from `tool_calls` instead of parsed out of the reply | - **The unqualified name always tracks upstream, bugs included**, and is never repurposed by a local change. It stays free even if nothing will occupy it — @@ -43,6 +44,16 @@ name, so it is the only place the distinction can live. 32%, so one number for "NL2SH accuracy" would hide whether a model was scored on translation or on formatting compliance. Not for a prompt *we* prefer, and not for an extractor we added: the second column has to be upstream's. +- **`_fc` is licensed by a capability, not a prompt.** It exists where upstream + publishes a native-function-calling column beside its prompting one, and it is + a different axis from `_parse`: `_parse` layers a second extractor over the + same model interface, while `_fc` changes *which model capability is + exercised* — it gates on `function_tools`, so it changes which models can run + the task at all. The unqualified name goes to the prompting reading, because + every model on a function-calling leaderboard has that row while only some + have an FC row, and a default name that half the fleet cannot run is the + narrow one. `bfcl_v3_non_live_0shot_gen` / `_fc` is the case that earned the + row. - The mode is read positionally, so a variant may not spell one: `foo_0shot_clp_gen.py` has two readings and is rejected. - The table is the current vocabulary, not the limit — a new variant earns a row From 95127e561b69344b54b8d39873de41a155a0419f Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 21:40:45 +0800 Subject: [PATCH 19/28] fix(scripts): exempt vendored relative imports, and count violations not lines The relative-import rule reached into `sieval/community/`, which pre-commit has always excluded. That was inert while every vendored relative import was a bare `from . import x`; a drop using `from ..x import y` made it bite, and the only fix the checker offered was to edit files kept byte-identical to upstream. The exemption is scoped to that one rule, inside `_check_relative_scope` -- not to the file, and not to the wrapper's file list. Both of those also disable the private-access and layer rules over all of community/, which pre-commit does not check at all and which we want to keep. Separately, the preflight wrapper reported `len(stderr lines)` as the violation count. Vendored files emit SyntaxWarning text on every run, so two real violations were reported as twenty-six. Co-Authored-By: Claude Opus 5 --- scripts/check_layer_imports.py | 14 ++++- scripts/check_preflight.py | 52 +++++++++++++--- .../unit/scripts/test_check_layer_imports.py | 28 +++++++++ tests/unit/scripts/test_check_preflight.py | 62 +++++++++++++++++++ 4 files changed, 147 insertions(+), 9 deletions(-) diff --git a/scripts/check_layer_imports.py b/scripts/check_layer_imports.py index 26d2d836..76937cc8 100644 --- a/scripts/check_layer_imports.py +++ b/scripts/check_layer_imports.py @@ -456,11 +456,23 @@ def _check_relative_scope(path: Path, tree: ast.AST) -> list[str]: Scoped to the sieval package only: ``scripts/`` files are standalone modules, not a package, so a relative import there fails at runtime and needs no lint. (The pre-commit hook feeds both trees; this check narrows on purpose.) + + ``sieval/community/`` is exempt from **this rule only**. Vendored files + mirror upstream byte-for-byte and a hash test pins them, so the only fix + this rule can offer there — rewrite the import — is the one edit that tree + forbids. The exemption is deliberately narrower than pre-commit's global + ``exclude: ^(sieval/community/|vendor/)``, which skips every hook and so + every rule: we drop the style half and keep the layer, sub-package and + private-module rules over ``community/``, which pre-commit does not check + at all. Do not "finish the job" by widening this to ``_check_file``. """ - if _get_layer(path) is None: + layer = _get_layer(path) + if layer is None: return [] if "tests" in path.parts: return [] + if layer == "community": + return [] file_pkg = _file_package(path) errors: list[str] = [] diff --git a/scripts/check_preflight.py b/scripts/check_preflight.py index 8018e27c..56b4c4ad 100644 --- a/scripts/check_preflight.py +++ b/scripts/check_preflight.py @@ -1164,6 +1164,35 @@ def lock_drift( return report +# Python's own warning output, which lands on the same stderr the import +# checker writes violations to: `:: : `. +_PY_WARNING_LINE = re.compile(r":\d+: \w*Warning: ") + + +def _import_violation_lines(stderr: str) -> list[str]: + """Return only the lines of *stderr* that are import-policy violations. + + ``check_layer_imports.py`` parses every file it is handed, and several + vendored modules raise ``SyntaxWarning`` on parse. Those warnings land on + stderr on **every** run, passing or failing, so counting raw stderr lines + made two real violations read as twenty-six and turned a two-line + regression into an apparently unfixable pile. + + A warning occupies two lines — the ``:: :`` header + and an indented echo of the offending source. Violations are always printed + flush-left, so dropping indented lines is safe and also handles a + multi-line warning body. + """ + violations: list[str] = [] + for line in stderr.strip().splitlines(): + if not line.strip(): + continue + if line[:1].isspace() or _PY_WARNING_LINE.search(line): + continue + violations.append(line) + return violations + + class PreflightRunner: """Orchestrates preflight checks.""" @@ -3047,13 +3076,15 @@ def check_imports(self) -> list[CheckResult]: # # KNOWN divergence, not parity: pre-commit additionally applies the # global `exclude: ^(sieval/community/|vendor/)`, so it skips - # `sieval/community/` while this wrapper checks it. Inert today (every - # relative import under `community/` is a bare level-1 `from . import - # x`), but a future vendored drop using `from ..x import y` would pass - # pre-commit and fail preflight, and the only offered fix would be to - # edit code kept byte-identical to upstream. Fixing it is a design call - # — hoisting the exemption into `_check_file` would also drop the - # private-access check's coverage of `community/`. + # `sieval/community/` for every hook while this wrapper checks it. The + # vendored drop this comment used to predict has arrived — a file using + # `from ..x import y`, whose only offered fix was to edit code kept + # byte-identical to upstream. The call was made in + # `_check_relative_scope`: `community/` is exempt from the + # relative-import rule and from nothing else, so the layer, sub-package + # and private-module rules still cover a tree pre-commit does not check + # at all. Keep the exemption there. Hoisting it here — or into + # `_check_file` — would silently drop those other three. enforced_py = [ f for f in self._git_tracked_files(".py") @@ -3072,7 +3103,12 @@ def check_imports(self) -> list[CheckResult]: if result.returncode == 0: return [CheckResult("PASS", "check_imports", "no import-policy violations")] - errors = [line for line in result.stderr.strip().splitlines() if line.strip()] + errors = _import_violation_lines(result.stderr) + if not errors: + # Non-zero exit with nothing recognisable as a violation means the + # checker's output shape moved. Show the raw stderr rather than + # reporting a reassuring "0 violation(s)" on a failing run. + errors = [ln for ln in result.stderr.strip().splitlines() if ln.strip()] return [ CheckResult( "FAIL", diff --git a/tests/unit/scripts/test_check_layer_imports.py b/tests/unit/scripts/test_check_layer_imports.py index 3a171031..dd04350e 100644 --- a/tests/unit/scripts/test_check_layer_imports.py +++ b/tests/unit/scripts/test_check_layer_imports.py @@ -800,6 +800,17 @@ def test_scripts_tree_is_out_of_scope(self, tmp_path: Path): f = self._write(tmp_path, "scripts/x.py", "from ..y import z\n") assert _check_relative_scope(f, ast.parse(f.read_text())) == [] + def test_community_tree_is_exempt(self, tmp_path: Path): + # Vendored files mirror upstream byte-for-byte and a hash test pins + # them, so the only fix this rule can offer — rewrite the import — is + # the one edit that tree forbids. + f = self._write( + tmp_path, + "sieval/community/bfcl_v3/type_convertor/java_type_converter.py", + "from ..type_mappings import JAVA_TYPE_CONVERSION\n", + ) + assert _check_relative_scope(f, ast.parse(f.read_text())) == [] + class TestCheckFileRelativeScopeIntegration: """Rule 3 wired into _check_file, and its interaction with rule 2.""" @@ -942,6 +953,23 @@ def test_level_1_undotted_subpackage_private_name_stays_exempt( ) assert _check_file(f) == [] + def test_community_exempt_from_rule_3_only(self, tmp_path: Path): + # The vendored carve-out is scoped to the relative-import rule and to + # nothing else. Hoisting it into `_check_file` — or excluding + # `community/` from the preflight wrapper's file list, which is what + # pre-commit's global `exclude` effectively does — would also drop the + # private-access, layer and sub-package rules over the whole tree. This + # is the half that stops a later widening. + f = self._write( + tmp_path, + "sieval/community/bfcl_v3/type_convertor/java_type_converter.py", + "from ..type_mappings import JAVA_TYPE_CONVERSION\n" + "from sieval.core.utils import _private_helper\n", + ) + errors = _check_file(f) + assert not any("cross-package relative import" in e for e in errors) + assert any("import of private name '_private_helper'" in e for e in errors) + class TestCheckSubpackageImports: """Sub-package boundary check — check 1 at dotted granularity. diff --git a/tests/unit/scripts/test_check_preflight.py b/tests/unit/scripts/test_check_preflight.py index 295d1b80..f3836c32 100644 --- a/tests/unit/scripts/test_check_preflight.py +++ b/tests/unit/scripts/test_check_preflight.py @@ -384,6 +384,68 @@ def test_fail_when_script_exits_nonzero(self, tmp_path: Path): assert "2 import-policy violation(s)" in results[0].message assert len(results[0].details) == 2 + def test_warning_noise_is_shown_but_not_counted(self, tmp_path: Path): + # `check_layer_imports.py` parses every file it is handed, and several + # vendored modules raise SyntaxWarning on parse — on every run, passing + # or failing. Counting raw stderr lines reported two real violations as + # twenty-six, which reads as an unfixable pile rather than a two-line + # regression. + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + (scripts_dir / "check_layer_imports.py").write_text("") + sieval_dir = tmp_path / "sieval" + sieval_dir.mkdir() + (sieval_dir / "example.py").write_text("") + + runner = PreflightRunner(project_root=tmp_path) + mock_result = subprocess.CompletedProcess( + args=[], + returncode=1, + stdout="", + stderr=( + "sieval/community/deepseek_math.py:192: SyntaxWarning: " + "invalid escape sequence '\\%'\n" + ' string = string.replace("\\%", "%")\n' + "sieval/core/x.py:7: core/ must not import tasks/ (sieval.tasks)\n" + ), + ) + side_effect = self._make_layers_side_effect(mock_result) + with patch("check_preflight.subprocess.run", side_effect=side_effect): + results = runner.check_imports() + + assert results[0].status == "FAIL" + assert "1 import-policy violation(s)" in results[0].message + assert results[0].details == [ + "sieval/core/x.py:7: core/ must not import tasks/ (sieval.tasks)" + ] + + def test_unrecognised_stderr_still_reported_on_failure(self, tmp_path: Path): + # If the checker's output shape ever moves, a "0 violation(s)" FAIL + # would read as reassuring noise. Fall back to the raw lines. + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + (scripts_dir / "check_layer_imports.py").write_text("") + sieval_dir = tmp_path / "sieval" + sieval_dir.mkdir() + (sieval_dir / "example.py").write_text("") + + runner = PreflightRunner(project_root=tmp_path) + mock_result = subprocess.CompletedProcess( + args=[], + returncode=1, + stdout="", + stderr="sieval/community/x.py:1: SyntaxWarning: invalid escape\n", + ) + side_effect = self._make_layers_side_effect(mock_result) + with patch("check_preflight.subprocess.run", side_effect=side_effect): + results = runner.check_imports() + + assert results[0].status == "FAIL" + assert "1 import-policy violation(s)" in results[0].message + assert results[0].details == [ + "sieval/community/x.py:1: SyntaxWarning: invalid escape" + ] + def test_fail_when_script_not_found(self, tmp_path: Path): # tmp_path has no scripts/ directory at all runner = PreflightRunner(project_root=tmp_path) From 838d6ce436ff6aba0168b8ff5692a4e09029371c Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 21:45:29 +0800 Subject: [PATCH 20/28] docs(scripts): say which rule the community exemption actually protects Both comments added with the exemption claimed it preserves "the layer, sub-package and private-module rules" over `sieval/community/`. Only the third of those binds there: `FORBIDDEN` and `FORBIDDEN_SUBPACKAGE` have no `community` entry, so a vendored file may import across layers freely and always could. The decision the comments defend is unchanged and still right -- hoisting the exemption into `_check_file` or the wrapper would drop private-module protection over a tree pre-commit does not check at all. But a reader weighing a wider exemption against three rules would be weighing it against two that cost nothing, and would reasonably conclude the guard is over-engineered. Name the one rule instead. Co-Authored-By: Claude Opus 5 --- scripts/check_layer_imports.py | 13 ++++++++++--- scripts/check_preflight.py | 10 ++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/scripts/check_layer_imports.py b/scripts/check_layer_imports.py index 76937cc8..e5a58b88 100644 --- a/scripts/check_layer_imports.py +++ b/scripts/check_layer_imports.py @@ -462,9 +462,16 @@ def _check_relative_scope(path: Path, tree: ast.AST) -> list[str]: this rule can offer there — rewrite the import — is the one edit that tree forbids. The exemption is deliberately narrower than pre-commit's global ``exclude: ^(sieval/community/|vendor/)``, which skips every hook and so - every rule: we drop the style half and keep the layer, sub-package and - private-module rules over ``community/``, which pre-commit does not check - at all. Do not "finish the job" by widening this to ``_check_file``. + every rule: we drop the style half and keep the other three over + ``community/``, a tree pre-commit does not check at all. + + Be precise about what that is worth, because the number is one and not + three: only **private-module protection** actually binds under + ``community/``. The layer and sub-package rules are vacuous there — + ``FORBIDDEN`` and ``FORBIDDEN_SUBPACKAGE`` have no ``community`` entry, so + a vendored file may import anything it likes. Losing the private-module + rule is therefore the entire cost of a wider exemption, and it is reason + enough: do not "finish the job" by widening this to ``_check_file``. """ layer = _get_layer(path) if layer is None: diff --git a/scripts/check_preflight.py b/scripts/check_preflight.py index 56b4c4ad..00e3c4bb 100644 --- a/scripts/check_preflight.py +++ b/scripts/check_preflight.py @@ -3081,10 +3081,12 @@ def check_imports(self) -> list[CheckResult]: # `from ..x import y`, whose only offered fix was to edit code kept # byte-identical to upstream. The call was made in # `_check_relative_scope`: `community/` is exempt from the - # relative-import rule and from nothing else, so the layer, sub-package - # and private-module rules still cover a tree pre-commit does not check - # at all. Keep the exemption there. Hoisting it here — or into - # `_check_file` — would silently drop those other three. + # relative-import rule and from nothing else, so private-module + # protection still covers a tree pre-commit does not check at all. + # That one rule is the whole of what is protected here — the layer and + # sub-package rules have no `community` entry and are vacuous on it — + # and it is enough. Keep the exemption there. Hoisting it here, or into + # `_check_file`, would silently drop it. enforced_py = [ f for f in self._git_tracked_files(".py") From 984190775ee9aa6fa0bc29d4656415b3b69d0b2b Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 22:29:11 +0800 Subject: [PATCH 21/28] fix(tasks): apply upstream's model-facing preprocessing on both BFCL protocols Upstream runs every function schema through `func_doc_language_specific_pre_processing` before it reaches the model, on both the prompting and the native-function-calling path, and merges the schema block into the row's own system turn via `system_prompt_pre_processing_chat_model`. The port did neither, so what the model saw diverged from what upstream sends on 3637 of 3641 rows -- the four exceptions being live_irrelevance rows carrying an empty schema list, where both renderings are the string `[]`. Two concrete divergences: * the 150 Java and JavaScript rows reached the model with their real parameter types (`array`, `integer`, `object`, ...) and no language hint, where upstream restates every one as `string` and appends "Note that the provided function is in Java 8 SDK syntax." The model was being asked for Java source against a schema that never said so. * the 92 live rows that open with their own system turn received a second prepended one. Two system turns is a shape every provider accepts and answers, so the run completed and only the score was wrong. Grading is deliberately untouched. Upstream's `ast_file_runner` reads `function` back out of the dataset and hands it to `ast_checker` unprocessed, so `preprocess` still stores the raw schema for `feedback` -- verified byte-identical on all 3641 rows under both protocols, and the gold-replay anchor still grades 2351/2351. The two vendored helpers rewrite their arguments in place, so the schema list is re-parsed per call and the question turns are copied; without the copy all 92 system-turn rows corrupt the stored sample on the second pass. Co-Authored-By: Claude Opus 5 --- sieval/community/bfcl_v3/__init__.py | 10 ++- sieval/community/bfcl_v3/preprocess.py | 118 +++++++++++++++++++++++++ sieval/tasks/bfcl_v3/_base.py | 43 ++++++++- tests/unit/tasks/bfcl_v3/conftest.py | 40 +++++++++ tests/unit/tasks/bfcl_v3/test__base.py | 84 ++++++++++++++++++ 5 files changed, 289 insertions(+), 6 deletions(-) create mode 100644 sieval/community/bfcl_v3/preprocess.py diff --git a/sieval/community/bfcl_v3/__init__.py b/sieval/community/bfcl_v3/__init__.py index 638c61fa..d03d8aa9 100644 --- a/sieval/community/bfcl_v3/__init__.py +++ b/sieval/community/bfcl_v3/__init__.py @@ -18,8 +18,8 @@ Partial vendored modules (named subsets of a larger upstream file -- see each module's own docstring for exactly which symbols were taken and why): -`parser.py`, `tool_convert.py`, `output_checks.py`, `aggregate.py`. `prompts.py` -is `constants/default_prompts.py` copied whole. +`parser.py`, `tool_convert.py`, `output_checks.py`, `preprocess.py`, +`aggregate.py`. `prompts.py` is `constants/default_prompts.py` copied whole. First-party modules: `_model_config.py` supplies the `MODEL_CONFIG_MAPPING` global `ast_checker.convert_func_name` reads, since upstream's own 2044-line @@ -44,6 +44,10 @@ from .aggregate import calculate_unweighted_accuracy, calculate_weighted_accuracy from .ast_checker import ast_checker from .output_checks import is_empty_output, is_function_calling_format_output +from .preprocess import ( + func_doc_language_specific_pre_processing, + system_prompt_pre_processing_chat_model, +) from .prompts import DEFAULT_SYSTEM_PROMPT from .tool_convert import convert_to_tool @@ -62,6 +66,8 @@ "convert_to_tool", "is_function_calling_format_output", "is_empty_output", + "func_doc_language_specific_pre_processing", + "system_prompt_pre_processing_chat_model", "calculate_weighted_accuracy", "calculate_unweighted_accuracy", "DEFAULT_SYSTEM_PROMPT", diff --git a/sieval/community/bfcl_v3/preprocess.py b/sieval/community/bfcl_v3/preprocess.py new file mode 100644 index 00000000..ede4d52a --- /dev/null +++ b/sieval/community/bfcl_v3/preprocess.py @@ -0,0 +1,118 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/model_handler/utils.py +Upstream blob: f78acc240bee3c43b18de3d4195fbf6ee1a207f1 +License: Apache-2.0 + +Symbols taken (bodies unchanged): `system_prompt_pre_processing_chat_model`, +`_get_language_specific_hint`, `func_doc_language_specific_pre_processing`. +This is a partial extraction, not a whole-file copy -- the rest of +`bfcl_eval/model_handler/utils.py` (multi-turn state helpers, response +formatters, the other model families' prompt builders) is out of scope. + +Both public symbols run on the way OUT, shaping what the model is shown. They +are deliberately not applied on the way back in: upstream's `ast_file_runner` +grades against the *unprocessed* schema, so the checker still sees the real +declared types. + +Only deviation: upstream imports `DEFAULT_SYSTEM_PROMPT` from +`bfcl_eval.constants.default_prompts`; that is rewritten to the relative import +of this package's copy. +""" + +import json + +from .prompts import DEFAULT_SYSTEM_PROMPT + + +def system_prompt_pre_processing_chat_model(prompts, function_docs, test_category): + """ + Add a system prompt to the chat model to instruct the model on the available functions and the expected response format. + If the prompts list already contains a system prompt, append the additional system prompt content to the existing system prompt. + """ + assert type(prompts) == list + + system_prompt_template = DEFAULT_SYSTEM_PROMPT + + system_prompt = system_prompt_template.format(functions=function_docs) + + # System prompt must be in the first position + # If the question comes with a system prompt, append its content at the end of the chat template. + if prompts[0]["role"] == "system": + prompts[0]["content"] = system_prompt + "\n\n" + prompts[0]["content"] + # Otherwise, use the system prompt template to create a new system prompt. + else: + prompts.insert( + 0, + {"role": "system", "content": system_prompt}, + ) + + return prompts + + +def _get_language_specific_hint(test_category): + if test_category == "java": + return " Note that the provided function is in Java 8 SDK syntax." + elif test_category == "javascript": + return " Note that the provided function is in JavaScript syntax." + else: + return " Note that the provided function is in Python 3 syntax." + + +def func_doc_language_specific_pre_processing(function, test_category): + if len(function) == 0: + return function + + assert type(function) == list + for item in function: + # Add language specific hints to the function description + func_description = item["description"] + item["description"] = item["description"] + _get_language_specific_hint( + test_category + ) + # Process the parameters + properties = item["parameters"]["properties"] + if test_category == "java": + for key, value in properties.items(): + if value["type"] == "any": + properties[key][ + "description" + ] += " This parameter can be of any type of Java object in string representation." + else: + value[ + "description" + ] += f" This is Java {value['type']} type parameter in string representation." + if value["type"] == "ArrayList" or value["type"] == "Array": + value[ + "description" + ] += f" The list elements are of type {value['items']['type']}; they are not in string representation." + del value["items"] + + value["type"] = "string" + + elif test_category == "javascript": + for key, value in properties.items(): + if value["type"] == "any": + properties[key][ + "description" + ] += " This parameter can be of any type of JavaScript object in string representation." + else: + value[ + "description" + ] += f" This is JavaScript {value['type']} type parameter in string representation." + if value["type"] == "array": + value[ + "description" + ] += f" The list elements are of type {value['items']['type']}; they are not in string representation." + del value["items"] + + if value["type"] == "dict": + if "properties" in value: # not every dict has properties + value[ + "description" + ] += f" The dictionary entries have the following schema; they are not in string representation. {json.dumps(value['properties'])}" + del value["properties"] + + value["type"] = "string" + + return function diff --git a/sieval/tasks/bfcl_v3/_base.py b/sieval/tasks/bfcl_v3/_base.py index ee80abf3..f820051a 100644 --- a/sieval/tasks/bfcl_v3/_base.py +++ b/sieval/tasks/bfcl_v3/_base.py @@ -37,7 +37,6 @@ from sieval.community.bfcl_v3 import ( CALL_EXPECTED, - DEFAULT_SYSTEM_PROMPT, GOLDLESS_CATEGORIES, LIVE_COUNTS, NON_LIVE_COUNTS, @@ -45,9 +44,11 @@ calculate_unweighted_accuracy, calculate_weighted_accuracy, convert_to_tool, + func_doc_language_specific_pre_processing, is_empty_output, is_function_calling_format_output, set_underscore_to_dot, + system_prompt_pre_processing_chat_model, ) from sieval.community.bfcl_v3.tool_convert import ModelStyle from sieval.community.bfcl_v3.type_mappings import GORILLA_TO_OPENAPI @@ -459,14 +460,48 @@ def _call_arguments(call) -> dict: return dict(arguments) if arguments else {} +def _model_facing_functions(raw) -> list: + """The function schemas as upstream shows them to the model. + + Both protocols run this, and they must run the *same* one: it is what makes + the Prompt and FC columns two readings of one benchmark rather than two + benchmarks. It rewrites Java and JavaScript parameter types to ``string`` + and appends a language hint to every description, so a model asked for a + `java` row is told it is reading Java 8 -- without it, the 150 Java and + JavaScript rows are posed in a language the schema never names. + + It does not touch grading. Upstream's `ast_file_runner` reads `function` + back out of the dataset and hands it to `ast_checker` unprocessed, so the + checker still compares against the real declared types -- which is why the + preprocessed list is built here, per protocol, instead of replacing what + `preprocess` stores for `feedback`. + + `json.loads` is not incidental: the vendored helper mutates the list it is + given, so it needs a private copy and gets one for free by re-parsing the + row's JSON. + """ + return func_doc_language_specific_pre_processing( + json.loads(raw["function"]), raw["category"] + ) + + class BfclV3PromptMixin: """Upstream's prompting protocol: schemas in the system turn, calls in text.""" UNDERSCORE_TO_DOT: ClassVar[bool] = False def _build_messages(self, raw) -> tuple[list, list[dict] | None]: - system = DEFAULT_SYSTEM_PROMPT.format(functions=raw["function"]) - return [{"role": "system", "content": system}, *raw["question"]], None + # Upstream merges rather than prepends: a row that already opens with a + # system turn keeps its own text, with the schema block in front of it. + # 92 live rows do, and prepending a second system turn instead is a + # shape no upstream run ever sent. Copy each message -- the helper + # rewrites `content` in place, and `raw` is the stored sample. + messages = system_prompt_pre_processing_chat_model( + [dict(message) for message in raw["question"]], + _model_facing_functions(raw), + raw["category"], + ) + return messages, None def _decode(self, output: ModelOutput, language: str) -> list | None: # Deferred on purpose: `ast_parse` is the one symbol in the vendored @@ -494,7 +529,7 @@ class BfclV3FCMixin: def _build_messages(self, raw) -> tuple[list, list[dict] | None]: tools = convert_to_tool( - json.loads(raw["function"]), + _model_facing_functions(raw), GORILLA_TO_OPENAPI, ModelStyle.OpenAI_Completions, ) diff --git a/tests/unit/tasks/bfcl_v3/conftest.py b/tests/unit/tasks/bfcl_v3/conftest.py index c960533c..1eed9320 100644 --- a/tests/unit/tasks/bfcl_v3/conftest.py +++ b/tests/unit/tasks/bfcl_v3/conftest.py @@ -54,6 +54,46 @@ def dotted_row(simple_row): return row +@pytest.fixture +def java_row(simple_row): + """A `java` row -- the cohort upstream's schema preprocessing rewrites. + + 100 non-live rows are Java and 50 more are JavaScript; on those, upstream + restates every parameter as a `string` before the schema reaches the model, + because the model is being asked for Java source rather than for JSON. + """ + row = dict(simple_row) + row["id"] = "java_0" + row["category"] = "java" + row["language"] = "Java" + functions = json.loads(row["function"]) + functions[0]["parameters"]["properties"]["tags"] = { + "type": "ArrayList", + "description": "tags", + "items": {"type": "String"}, + } + row["function"] = json.dumps(functions) + return row + + +@pytest.fixture +def system_turn_row(simple_row): + """A row that already opens with its own system turn. + + 92 live rows do. Upstream merges the schema block into that turn; a port + that prepends a second one instead sends a two-system-turn conversation no + upstream run ever produced. + """ + row = dict(simple_row) + row["id"] = "live_simple_0" + row["category"] = "live_simple" + row["question"] = [ + {"role": "system", "content": "You are a geometry tutor."}, + {"role": "user", "content": "Area of a 10x5 triangle?"}, + ] + return row + + @pytest.fixture def irrelevance_row(simple_row): row = dict(simple_row) diff --git a/tests/unit/tasks/bfcl_v3/test__base.py b/tests/unit/tasks/bfcl_v3/test__base.py index 13dd7791..34074eb8 100644 --- a/tests/unit/tasks/bfcl_v3/test__base.py +++ b/tests/unit/tasks/bfcl_v3/test__base.py @@ -13,6 +13,11 @@ BfclV3PromptMixin, grade_single_turn, ) +from sieval.tasks.bfcl_v3.bfcl_v3_non_live_0shot_gen import BfclV3NonLiveZeroShotGenTask +from sieval.tasks.bfcl_v3.bfcl_v3_non_live_0shot_gen_fc import ( + BfclV3NonLiveZeroShotGenFCTask, +) +from tests.conftest import MockChatModel, MockDataset _META: ModelMeta = {"model": "mock-chat", "api_base": None, "default_params": {}} @@ -216,6 +221,85 @@ def test_the_prompt_mixin_puts_the_schemas_in_a_system_turn_and_sends_no_tools( assert messages[1:] == dotted_row["question"] +def test_the_prompt_mixin_merges_into_a_row_that_already_has_a_system_turn( + system_turn_row, +): + """Upstream merges; prepending a second system turn is a different payload. + + 92 live rows open with their own system turn. What makes this worth a test + of its own is that the wrong shape is not visibly wrong: two system turns + is a conversation every provider accepts and answers, so the run completes + and the score is merely somewhat off. + """ + messages, _ = BfclV3PromptMixin()._build_messages(system_turn_row) + assert [m["role"] for m in messages] == ["system", "user"] + content = messages[0]["content"] + assert "calculate_triangle_area" in content + # The row's own instruction survives, after the schema block rather than + # in a turn of its own. + assert content.endswith("You are a geometry tutor.") + + +def test_building_messages_leaves_the_stored_row_untouched(system_turn_row, java_row): + """Both vendored helpers rewrite their argument in place. + + Reached without a copy, they would edit the sample the runner holds, and + the damage is invisible on a first pass -- it surfaces only on the second + (a resume, a retry, `max_iterations > 1`), as a schema block appended twice + or a type already flattened. Idempotence is the discriminating check: one + call looks correct either way. + """ + for mixin in (BfclV3PromptMixin(), BfclV3FCMixin()): + for row in (system_turn_row, java_row): + before = json.dumps(row, sort_keys=True) + first = mixin._build_messages(row) + second = mixin._build_messages(row) + assert first == second + assert json.dumps(row, sort_keys=True) == before + + +def test_both_protocols_restate_java_parameters_as_strings(java_row): + """Upstream's language preprocessing, which both columns share. + + It is shared on purpose: the Prompt and FC numbers are two readings of one + benchmark only while the schemas they carry agree. So this asserts the same + property through both protocols rather than trusting one to stand for both. + """ + messages, _ = BfclV3PromptMixin()._build_messages(java_row) + prompt_text = messages[0]["content"] + _, tools = BfclV3FCMixin()._build_messages(java_row) + assert tools is not None + fc_properties = tools[0]["function"]["parameters"]["properties"] + + assert [v["type"] for v in fc_properties.values()] == ["string", "string", "string"] + assert "'type': 'string'" in prompt_text + # The Java 8 hint, and the element type carried over from the deleted + # `items` -- both protocols, because both call the same helper. + for rendered in (prompt_text, json.dumps(tools)): + assert "Java 8 SDK syntax" in rendered + assert "The list elements are of type String" in rendered + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "task_cls", [BfclV3NonLiveZeroShotGenTask, BfclV3NonLiveZeroShotGenFCTask] +) +async def test_preprocess_stores_the_unprocessed_schema_for_grading(task_cls, java_row): + """Preprocessing is model-facing only; the grader must see the real types. + + Upstream's `ast_file_runner` reads `function` back out of the dataset and + hands it to `ast_checker` unprocessed, so a port that stored the flattened + copy would compare a Java row's real `ArrayList` gold against a schema + claiming every parameter is a `string`. + """ + task = task_cls(MockDataset(), MockChatModel()) + record = await task.preprocess(java_row, None) + assert record["extra"]["function"] == java_row["function"] + assert json.loads(record["extra"]["function"])[0]["parameters"]["properties"][ + "tags" + ] == {"type": "ArrayList", "description": "tags", "items": {"type": "String"}} + + def test_the_fc_mixin_sends_tools_and_renames_the_dotted_function(dotted_row): """Upstream's own rewrite -- the one `UNDERSCORE_TO_DOT` later reverses.""" messages, tools = BfclV3FCMixin()._build_messages(dotted_row) From 48a86037f560353f02e3d15a37c66cea7e45746c Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 22:58:19 +0800 Subject: [PATCH 22/28] fix(community): stop the BFCL v3 Prompt decoder from executing model output `resolve_ast_by_type`'s `ast.BinOp` branch resolved a decoded call's keyword argument with `eval(ast.unparse(value))`. That node is the model's reply, so a reply carrying `f(x=__import__('os').system('...') + 0)` parses to a `BinOp`, the decoder runs the payload, and it returns an ordinary-looking number. The sample grades, the score is plausible, and nothing in the run looks unusual afterwards. Established by side effect rather than by inspection: a probe payload wrote a marker file, and the file appeared. `_safe_eval.safe_eval` walks the node instead of unparsing it, so there is no string and no namespace to escape from, and it admits only literals, their containers, and the arithmetic/bitwise operators. Every expression upstream's `eval` computes *without* executing something is computed here to the same value: 19/19 benign expressions agree, and the 18 fidelity cases in the suite pass under upstream's own `eval` too, so they encode upstream's values rather than the port's. It also refuses two shapes that do not execute. An f-string, where `ast.literal_eval` draws the same line. And an expression whose cost is unbounded -- decoding runs inline on the session's event loop with no timeout around it (the grade timeout is in the `feedback` worker, a stage later), so `f(x=9**9**9)` would not return a wrong answer, it would stall every other sample in the run. Restoring upstream's `eval` makes that case run past a 40 s kill; the guarded version returns in under 0.1 s. Evidence no bound binds on the pinned data: all 2351 gradeable Python rows decode, 0 `BinOp` nodes reach the branch at all, and the largest integer literal across the gold answers is 13 digits against a 2**20 cap. Reverse mutation confirms the tests discriminate -- restoring upstream's `eval` fails exactly the 10 safety tests and leaves every fidelity test passing. The `ast.Lambda` branch keeps upstream's second `eval` byte-identical, because it cannot reach it: `Lambda.body` is a single expression node, so `value.body[0]` raises `TypeError` while the argument is still being built. Hardening a line that cannot execute would be a deviation bought for nothing. A test pins the `TypeError` and so fails if someone later "repairs" the subscript, which would install a live `eval` over model output. The guard is held outside `parser.py` because that file is vendored: inlining it would enlarge a diff that exists to be compared against upstream, and would put a security boundary inside a file no linter runs over. `community/`'s CLAUDE.md records the hand-lint command, since the package-wide exclusions cover it. Separately, `preprocess.py` had been vendored without being added to the identity map, so a vendored file went unprotected by the drift test for a commit. Added, along with a test that each file's recorded provenance is resolvable, and a note that `Upstream path:` is relative to `berkeley-function-call-leaderboard/` rather than the repo root. All 12 blob SHAs re-verified against gorilla@ea13468e, 0 mismatches. Co-Authored-By: Claude Opus 5 --- sieval/community/CLAUDE.md | 20 +- sieval/community/bfcl_v3/__init__.py | 11 +- sieval/community/bfcl_v3/_safe_eval.py | 178 ++++++++++++++++++ sieval/community/bfcl_v3/parser.py | 16 +- .../unit/community/bfcl_v3/test__safe_eval.py | 152 +++++++++++++++ tests/unit/community/bfcl_v3/test_identity.py | 74 +++++++- 6 files changed, 439 insertions(+), 12 deletions(-) create mode 100644 sieval/community/bfcl_v3/_safe_eval.py create mode 100644 tests/unit/community/bfcl_v3/test__safe_eval.py diff --git a/sieval/community/CLAUDE.md b/sieval/community/CLAUDE.md index 7137ec37..9d474bfd 100644 --- a/sieval/community/CLAUDE.md +++ b/sieval/community/CLAUDE.md @@ -26,7 +26,21 @@ inline where it would swamp a diff against upstream. Do not add more original code here without the same argument; a helper with one caller belongs in that caller's module. +`bfcl_v3/_safe_eval.py` is the second, and it has exactly **one** caller — so it +is the case that shows what "the same argument" means. The caller is a *vendored* +file (`bfcl_v3/parser.py`, whose `ast.BinOp` branch upstream resolves with `eval` +over model output). Inlining the guard there would enlarge a diff that exists to +be compared against upstream, and would put a security boundary inside a file +that cannot be linted — the surrounding vendored code does not pass lint, so +there would be no path to checking the boundary itself. "One caller" is the rule +for ordinary helpers; a boundary that has to stay separately auditable is not +one. + The package-wide `ruff` / `mypy` / `pre-commit` exclusions exist to keep vendored -code byte-identical and cover this file too, which is the wrong default for a -security boundary. Until they are narrowed, lint it by hand: -`ruff check --config 'exclude=["vendor"]' sieval/community/_sympy_guards.py`. +code byte-identical and cover both files too, which is the wrong default for a +security boundary. Until they are narrowed, lint them by hand: + +``` +ruff check --config 'exclude=["vendor"]' sieval/community/_sympy_guards.py +ruff check --config 'exclude=["vendor"]' sieval/community/bfcl_v3/_safe_eval.py +``` diff --git a/sieval/community/bfcl_v3/__init__.py b/sieval/community/bfcl_v3/__init__.py index d03d8aa9..0784c3c3 100644 --- a/sieval/community/bfcl_v3/__init__.py +++ b/sieval/community/bfcl_v3/__init__.py @@ -8,9 +8,11 @@ imports its own submodules by absolute path (`bfcl_eval.constants...`, `bfcl_eval.model_handler...`). A vendored copy cannot keep those imports, since `bfcl_eval` itself is not installed here -- rewriting them to relative imports -within this package is the only edit made to any vendored file. Each vendored -module's docstring records upstream's blob SHA at the pin and the exact rewrite -applied, so a reviewer can re-derive the copy from a fresh checkout. +within this package is the only edit made to any vendored file, with one +exception: `parser.py`'s `ast.BinOp` branch resolves a model-authored expression +with `eval`, and calls `_safe_eval.safe_eval` instead. Each vendored module's +docstring records upstream's blob SHA at the pin and the exact edits applied, so +a reviewer can re-derive the copy from a fresh checkout. Whole-file vendored modules: `ast_checker.py`, `type_mappings.py`, `type_convertor/java_type_converter.py`, `type_convertor/js_type_converter.py`, @@ -27,6 +29,9 @@ category tables (row counts, which categories have no gold answer file, which are non-Python) shared by the dataset loader and the tasks -- neither of which may reach into the other's private module, so the tables live here instead. +`_safe_eval.py` evaluates a literal expression without executing it, and is the +one deviation `parser.py` carries; it is held outside that file so the boundary +stays lintable, which vendored code is not. AI-Generated Code - Claude Opus 5 (Anthropic) """ diff --git a/sieval/community/bfcl_v3/_safe_eval.py b/sieval/community/bfcl_v3/_safe_eval.py new file mode 100644 index 00000000..5fd8ac04 --- /dev/null +++ b/sieval/community/bfcl_v3/_safe_eval.py @@ -0,0 +1,178 @@ +"""Evaluate a literal arithmetic expression without executing anything. + +`resolve_ast_by_type` resolves each keyword argument of a decoded function call +into a Python value, and for `ast.BinOp` upstream does it with +``eval(ast.unparse(value))``. The node comes from a model's reply, so that line +runs model-authored code: a reply carrying +``f(x=__import__('os').system('...') + 0)`` parses to a `BinOp`, `eval` runs the +call, and the decoder returns an ordinary-looking number. Nothing in the run +looks unusual afterwards -- the sample grades, the score is plausible, and the +only trace is whatever the payload did. + +:func:`safe_eval` replaces that one call. It walks the node instead of unparsing +it, so there is no string and no namespace to escape from, and it admits only +literals, their containers, and the arithmetic/bitwise operators. A `Name`, +`Call`, `Attribute`, `Subscript` or comprehension is refused outright, which is +the entire divergence: every expression upstream's `eval` computes *without* +executing something is computed here too, to the same value. + +Two shapes are refused that do not execute, and both are deliberate: + +* An f-string, even one with no placeholders. `ast.literal_eval` draws the line + in the same place, and the alternative is a `FormattedValue` walker guarding a + shape no function argument has been observed to carry. +* An expression whose *cost* is unbounded -- see :data:`MAX_RESULT_SIZE`. + Decoding runs inline on the session's event loop with no timeout around it + (the grade timeout is in the `feedback` worker, a stage later), so + ``f(x=9**9**9)`` would not return a wrong answer; it would stall every other + sample in the run. + +Refusal raises `ValueError`, which reaches the decoder's caller as a decode +failure and scores the sample wrong. That is the right outcome and not a +softening of it: a reply that puts executable code in an argument slot has not +produced the function call it was asked for, and upstream scores every other +unparseable reply the same way. + +This is a restriction on what is evaluated, not a sandbox. It holds because the +accepted node types cannot name anything outside the expression; it would stop +holding the moment one that can is added. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import ast +import operator + +#: Largest intermediate any step may produce, counted in bits for integers and +#: in items for everything sized. A bound is needed because the operators are +#: eager: `9 ** 9 ** 9` asks for a 369-million-digit integer and does not +#: return, and `'a' * 10**9` allocates a gigabyte before anything can inspect +#: it. 2**20 leaves a million-bit integer and a million-character string +#: admissible, against a largest integer literal of 13 digits (about 41 bits) +#: across the gold answers of all 2351 gradeable Python rows in the pinned +#: snapshot -- and no argument there reaches this module at all, since none of +#: them is written as an expression. +MAX_RESULT_SIZE = 1 << 20 + +_BIN_OPS = { + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.Div: operator.truediv, + ast.FloorDiv: operator.floordiv, + ast.Mod: operator.mod, + ast.Pow: operator.pow, + ast.LShift: operator.lshift, + ast.RShift: operator.rshift, + ast.BitOr: operator.or_, + ast.BitXor: operator.xor, + ast.BitAnd: operator.and_, +} + +_UNARY_OPS = { + ast.UAdd: operator.pos, + ast.USub: operator.neg, + ast.Invert: operator.invert, + ast.Not: operator.not_, +} + + +def _size(value) -> int: + """Footprint of an intermediate: bits for an integer, items for anything sized. + + `bool` is excluded from the integer arm on purpose -- it is an `int` + subclass whose `bit_length` says nothing useful, and it is never the thing + that grows. + """ + if isinstance(value, int) and not isinstance(value, bool): + return value.bit_length() + try: + return len(value) + except TypeError: + return 0 + + +def _refuse_if_unbounded(op, left, right) -> None: + """Screen the operators whose cost their operands' size does not bound. + + Checked *before* the operator runs, which is the only time it can be: the + cost of `9 ** 9 ** 9` cannot be measured after the fact. Addition and the + bitwise operators need no entry -- their result is at most as large as the + sum of their inputs, so :func:`safe_eval`'s check on the value that comes + back catches them one step later. + """ + grown = None + if isinstance(op, (ast.Pow, ast.LShift)) and isinstance(right, int): + if isinstance(right, bool) or right < 0: + return + # A float base overflows to `inf` cheaply, so only an integer one grows. + if isinstance(op, ast.LShift): + grown = _size(left) + right + elif isinstance(left, int): + grown = _size(left) * right + elif isinstance(op, ast.Mult): + # Sequence repetition. Two integers multiplied only *add* bit lengths, + # so that case is left to the post-hoc check. + for value, count in ((left, right), (right, left)): + if ( + isinstance(count, int) + and not isinstance(count, bool) + and hasattr(value, "__len__") + ): + grown = len(value) * max(count, 0) + break + if grown is not None and grown > MAX_RESULT_SIZE: + raise ValueError( + f"refusing an expression whose result would exceed " + f"{MAX_RESULT_SIZE} units ({type(op).__name__})" + ) + + +def safe_eval(node: ast.expr): + """Value of a literal expression, computed without executing anything. + + Raises `ValueError` on any node that could name or call something outside + the expression, and on any expression whose cost is unbounded. + """ + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Tuple): + return tuple(safe_eval(element) for element in node.elts) + if isinstance(node, ast.List): + return [safe_eval(element) for element in node.elts] + if isinstance(node, ast.Set): + return {safe_eval(element) for element in node.elts} + if isinstance(node, ast.Dict): + # A `**spread` entry has a `None` key and names something to spread, so + # it is refused with everything else that names. + if any(key is None for key in node.keys): + raise ValueError("refusing a dict spread in a function argument") + return { + safe_eval(key): safe_eval(value) + for key, value in zip(node.keys, node.values, strict=True) + if key is not None # unreachable past the guard; narrows the type + } + if isinstance(node, ast.UnaryOp) and type(node.op) in _UNARY_OPS: + return _bounded(_UNARY_OPS[type(node.op)](safe_eval(node.operand))) + if isinstance(node, ast.BinOp) and type(node.op) in _BIN_OPS: + left = safe_eval(node.left) + right = safe_eval(node.right) + _refuse_if_unbounded(node.op, left, right) + return _bounded(_BIN_OPS[type(node.op)](left, right)) + raise ValueError( + f"refusing to evaluate {type(node).__name__} in a function argument" + ) + + +def _bounded(value): + """Return `value`, or refuse it for being larger than :data:`MAX_RESULT_SIZE`. + + The companion to :func:`_refuse_if_unbounded`: that one screens the + operators that can outrun their inputs, this one catches growth by + accumulation, where every single step looked affordable. + """ + if _size(value) > MAX_RESULT_SIZE: + raise ValueError( + f"refusing an intermediate larger than {MAX_RESULT_SIZE} units" + ) + return value diff --git a/sieval/community/bfcl_v3/parser.py b/sieval/community/bfcl_v3/parser.py index 4cfcfde3..3488f9fd 100644 --- a/sieval/community/bfcl_v3/parser.py +++ b/sieval/community/bfcl_v3/parser.py @@ -18,6 +18,19 @@ parse_java_function_call` and the `js_parser` twin -- same rename as `source_parser/`'s own docstrings explain (this package's `parser.py` already occupies upstream's directory name). +3. `resolve_ast_by_type`'s `ast.BinOp` branch calls `safe_eval` where upstream + calls `eval(ast.unparse(value))`. That node is model output, so upstream's + line executes model-authored code -- `f(x=__import__('os').system('...') + + 0)` runs the call and hands back an ordinary-looking number. `safe_eval` + computes the same value for every expression that does not execute + something, and refuses the rest; the argument for where it lives, and for + the two non-executing shapes it also refuses, is in `_safe_eval.py`. + + The `ast.Lambda` branch below keeps upstream's `eval` untouched, because it + cannot reach it: `Lambda.body` is a single expression node, so `value.body[0]` + raises `TypeError` while the argument is still being built. Hardening a + branch that cannot execute would be a deviation bought for nothing, and the + raise it produces today is already the decode failure upstream scores. The Java/JavaScript branches of `ast_parse` dispatch to those two functions: a `java`/`javascript`-category sample's model output is Java/JavaScript @@ -27,6 +40,7 @@ import ast +from ._safe_eval import safe_eval from .source_parser.java_parser import parse_java_function_call from .source_parser.js_parser import parse_javascript_function_call @@ -92,7 +106,7 @@ def resolve_ast_by_type(value): elif isinstance( value, ast.BinOp ): # Added this condition to handle function calls as arguments - output = eval(ast.unparse(value)) + output = safe_eval(value) # DEVIATION 3 -- upstream: eval(ast.unparse(value)) elif isinstance(value, ast.Name): output = value.id elif isinstance(value, ast.Call): diff --git a/tests/unit/community/bfcl_v3/test__safe_eval.py b/tests/unit/community/bfcl_v3/test__safe_eval.py new file mode 100644 index 00000000..26c490a9 --- /dev/null +++ b/tests/unit/community/bfcl_v3/test__safe_eval.py @@ -0,0 +1,152 @@ +"""Cover the guard that replaces `eval` in the BFCL v3 argument resolver. + +Two properties, and neither is worth much alone: the guard must refuse what +executes (or it is not a guard) *and* agree with upstream on everything that +does not (or it is a scoring change wearing a safety label). + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import ast + +import pytest + +from sieval.community.bfcl_v3._safe_eval import MAX_RESULT_SIZE, safe_eval +from sieval.community.bfcl_v3.parser import ast_parse, resolve_ast_by_type + + +def _argument(source: str) -> ast.expr: + """The `x=` argument node of a one-call snippet, as the resolver sees it.""" + return ast.parse(source, mode="eval").body.keywords[0].value # ty: ignore + + +def test_a_payload_in_an_argument_does_not_run(tmp_path): + """The whole point, observed as a side effect rather than as a refusal. + + Asserting only that a `ValueError` comes back would pass against a guard + that refuses *after* evaluating. The marker file is what distinguishes + "refused" from "ran, then complained": under the `eval` this replaces, the + file exists and the decoder returns an ordinary-looking number. + """ + marker = tmp_path / "payload-ran" + source = f"f(x=__import__('pathlib').Path({str(marker)!r}).write_text('x') + 0)" + + with pytest.raises(ValueError, match="refusing to evaluate Call"): + resolve_ast_by_type(_argument(source)) + + assert not marker.exists(), "the payload executed before being refused" + + +def test_the_decoder_refuses_rather_than_executing_a_hostile_reply(tmp_path): + """Through the real entry point, because that is what `postprocess` calls. + + The refusal has to surface as a decode failure -- an exception out of + `ast_parse` -- so the sample scores wrong. A reply that puts executable code + in an argument slot did not produce the function call it was asked for. + """ + marker = tmp_path / "payload-ran" + reply = f"[f(x=__import__('pathlib').Path({str(marker)!r}).write_text('x') + 0)]" + + with pytest.raises(ValueError, match="refusing"): + ast_parse(reply, "Python") + + assert not marker.exists() + + +#: Every shape upstream's `eval(ast.unparse(node))` computes without executing +#: anything. The expected values are upstream's, so a divergence shows up as a +#: wrong value rather than as a refusal. +@pytest.mark.parametrize( + ("source", "expected"), + [ + ("f(x=1 + 2)", 3), + ("f(x=60 * 60)", 3600), + ("f(x=10 / 4)", 2.5), + ("f(x=7 // 2)", 3), + ("f(x=7 % 3)", 1), + ("f(x=2 ** 10)", 1024), + ("f(x=-3 + 1)", -2), + ("f(x='a' + 'b')", "ab"), + ("f(x=[1] + [2])", [1, 2]), + ("f(x=(1,) + (2,))", (1, 2)), + ("f(x=1 << 4)", 16), + ("f(x=6 & 3)", 2), + ("f(x=6 | 3)", 7), + ("f(x=6 ^ 3)", 5), + ("f(x=1.5 * 2)", 3.0), + ("f(x=(1 + 2) * (3 - 1))", 6), + ("f(x=[1 + 1, 2 * 2])", [2, 4]), + ("f(x={'a': 1 + 1})", {"a": 2}), + ], +) +def test_a_non_executing_expression_keeps_upstreams_value(source, expected): + node = _argument(source) + result = resolve_ast_by_type(node) + assert result == expected + # `1 == 1.0` and `True == 1`, so equality alone would let a type change + # through -- and the checker compares types. + assert type(result) is type(expected) + + +@pytest.mark.parametrize( + "source", + [ + "f(x=__import__('os').getpid() + 0)", + "f(x=open('/etc/passwd').read() + '')", + "f(x=[].__class__ + 0)", + "f(x=undefined_name + 1)", + "f(x=(lambda: 1)() + 1)", + "f(x=[i for i in range(3)] + [])", + # The spread must sit inside a BinOp to reach this guard at all: a bare + # `{**a}` argument is handled by upstream's own `ast.Dict` branch, which + # refuses it a different way (`Exception: Unsupported AST type`). + "f(x={**{'a': 1}} | {'b': 2})", + "f(x=f'a' + 'b')", + ], +) +def test_an_expression_that_could_name_anything_is_refused(source): + with pytest.raises(ValueError, match="refusing"): + resolve_ast_by_type(_argument(source)) + + +@pytest.mark.parametrize( + "source", + [ + "f(x=9 ** 9 ** 9)", # 369 million digits + "f(x='a' * 10 ** 9)", # a gigabyte of string + "f(x=1 << 10 ** 9)", + "f(x=[0] * 10 ** 9)", + ], +) +def test_an_unbounded_expression_is_refused_before_it_is_computed(source): + """A hang here is the failure, not an error. + + Decoding runs inline on the session's event loop with no timeout around it, + so computing any of these would stall every other sample in the run. The + test can only assert that it returns at all -- which it does only because + the cost is screened before the operator runs. + """ + refused = "refusing an expression|refusing an intermediate" + with pytest.raises(ValueError, match=refused): + resolve_ast_by_type(_argument(source)) + + +def test_the_bound_admits_far_more_than_any_real_argument(): + """Evidence the cap does not bind: the pinned largest literal is 13 digits.""" + assert safe_eval(_argument("f(x=2 ** 4096)")) == 2**4096 + assert safe_eval(_argument("f(x=1617262800000 * 2)")) == 3234525600000 + assert MAX_RESULT_SIZE >= 1 << 20 + + +def test_the_lambda_branch_still_cannot_reach_its_eval(): + """Upstream's second `eval` is dead, and this fails if it stops being dead. + + `ast.Lambda.body` is a single expression node, so upstream's + `value.body[0].value` raises while the argument to `eval` is still being + built. That is why the branch was left byte-identical: hardening a line that + cannot execute buys nothing. If someone later "repairs" the subscript, this + test fails -- which is the point, because the repair would install a live + `eval` over model output. + """ + with pytest.raises(TypeError, match="not subscriptable"): + resolve_ast_by_type(_argument("f(x=lambda a: a + 1)")) diff --git a/tests/unit/community/bfcl_v3/test_identity.py b/tests/unit/community/bfcl_v3/test_identity.py index 3886bf3f..c2b66f23 100644 --- a/tests/unit/community/bfcl_v3/test_identity.py +++ b/tests/unit/community/bfcl_v3/test_identity.py @@ -1,13 +1,42 @@ +"""Pin the vendored BFCL v3 files, and keep their provenance resolvable. + +A sha256 taken over our own copy pins whatever was copied -- it detects drift +after vendoring and proves nothing about what was vendored. The non-circular +anchor is the `Upstream blob:` line in each vendored module's docstring: a git +blob id, resolvable in upstream's repository by anyone, independent of this +checkout. All twelve were resolved against `ShishirPatil/gorilla` at +`ea13468e`, and all twelve matched. + +So the two tests here divide the work deliberately: + +* :func:`test_vendored_file_has_not_drifted` is the drift detector. Its digests + are read off the copy, on purpose -- that is the only thing they can be. +* :func:`test_vendored_file_records_resolvable_provenance` keeps the real anchor + present. It cannot resolve the blob (that needs upstream's repository, which + no test may reach for), so it checks the claim is *recorded* in a shape a + reader can resolve. Without it, a re-vendoring that drops the docstring leaves + the first test passing over bytes with no stated origin. + +`Upstream path:` is relative to `berkeley-function-call-leaderboard/`, the +package root inside upstream's monorepo -- not to the repository root, where +none of these paths exist. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + import hashlib +import re from pathlib import Path import pytest import sieval.community.bfcl_v3 as pkg -#: sha256 of each vendored file AS COMMITTED (upstream's bytes plus the -#: documented import rewrite and provenance docstring). Fill these in from the -#: failing run's output, once, after reviewing the diff against upstream. +#: sha256 of each vendored file AS COMMITTED -- upstream's bytes plus the +#: documented import rewrites, provenance docstring, and for `parser.py` the +#: annotated safety deviation. Updated only alongside a change to that file's +#: docstring deviation list, in the same commit, so the reason a digest moved is +#: always readable next to the new digest. VENDORED_SHA256 = { "ast_checker.py": ( "66ba103f5dd17d9e2b4ee597ebfdb013486add739412366e65c4cb6691acb936" @@ -27,7 +56,7 @@ "source_parser/js_parser.py": ( "3a096d914cc54514009e11915e8d4cc4c35af26b5a1c9f21ced640cd896dc737" ), - "parser.py": ("ae3863501c133eea661120d455cce3c27fe8709e01532cba639e45693f8b8c1b"), + "parser.py": ("a3e1606a074e664c2fe180afad853c53c4635feb0e5b99044d5e8740fd163683"), "tool_convert.py": ( "11b758572a09fca738479243e4cc3005387def672786b3231c868e6d73ad386e" ), @@ -38,8 +67,14 @@ "b7abb648a15492db0101d7a0628b8cc1fdfa2d1542992e0819a44911729c9472" ), "prompts.py": ("fae6ed78b354c3e1a355795569a78ba18f2b0826f52bfa1dca35a2454655b78f"), + "preprocess.py": ( + "ce638e4bee853d12644528e728475025810b719287631f0ea68e9197dfde763f" + ), } +#: Upstream revision every `Upstream blob:` line above was resolved against. +UPSTREAM_REVISION = "ea13468e" + @pytest.mark.parametrize("relative_path", sorted(VENDORED_SHA256)) def test_vendored_file_has_not_drifted(relative_path: str): @@ -52,7 +87,36 @@ def test_vendored_file_has_not_drifted(relative_path: str): ) -def test_category_tables_match_the_pinned_snapshot(): +@pytest.mark.parametrize("relative_path", sorted(VENDORED_SHA256)) +def test_vendored_file_records_resolvable_provenance(relative_path: str): + """Every vendored file states where it came from, resolvably. + + Three files legitimately share one blob (`parser.py`, `tool_convert.py` and + `preprocess.py` are all partial extractions from upstream's + `model_handler/utils.py`), so the blob ids are not asserted unique. + """ + text = (Path(pkg.__file__).parent / relative_path).read_text() + assert re.search(r"^Upstream path: \S+\.py$", text, re.M), ( + f"{relative_path} records no `Upstream path:`. The sha256 above pins " + "these bytes but says nothing about their origin; this line is what a " + "reader follows to check the copy against upstream." + ) + assert re.search(r"^Upstream blob: [0-9a-f]{40}$", text, re.M), ( + f"{relative_path} records no `Upstream blob:` git object id. That id is " + f"the only anchor outside this checkout -- resolvable at " + f"{UPSTREAM_REVISION} as " + f"`berkeley-function-call-leaderboard/`." + ) + + +def test_category_tables_total_the_pinned_row_counts(): + """The per-category tables are constants; these are the totals they must sum to. + + Named for what it does: it reads no snapshot. The check that the tables + match the downloaded data is the acceptance replay, which is skipped + wherever the snapshot is absent -- so this one has to stand on its own as a + guard against an edited table, not be mistaken for the one that walks rows. + """ from sieval.community.bfcl_v3 import LIVE_COUNTS, NON_LIVE_COUNTS assert sum(NON_LIVE_COUNTS.values()) == 1390 From 985c73cf83a1d3b058e860d91b8e3787aa354566 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 22:58:45 +0800 Subject: [PATCH 23/28] fix(tasks): stop the BFCL v3 decode guard from scoring a broken environment `_decode` defers its import of the vendored parser, because tree-sitter lives behind the optional `bfcl-v3` extra and importing it eagerly makes that extra a hard requirement. The deferred import therefore raises from inside the `try` whose `except Exception` exists to turn an undecodable reply into a score. An environment without the extra was being graded, not reported: every row of the run scores wrong, `fails` stays 0, and the result is indistinguishable from a model that cannot call a function. `ImportError` now propagates, which costs the run and names the cause. Two tests pin the split -- one that an undecodable reply is still scored rather than raised, and one that a missing parser dependency propagates. Separately, two docstrings justified `underscore_to_dot` being a call argument with "grading runs in a worker process, so a global set in the parent never arrives." That is false: `run_cpu_bound` falls back to running inline on the event loop whenever no worker pool is available, so the global would arrive fine. The argument-passing is still right, but for a different reason -- `grade_single_turn` writes the global the vendored checker reads and consumes it with no `await` in between, which is what makes the pair atomic. A caller that set the global and then awaited would let another sample's grade land between the set and the use. A reader who checked the stated reason would find it wrong and could reasonably conclude the constraint was imaginary, so the justification is corrected to the one that holds. Co-Authored-By: Claude Opus 5 --- sieval/tasks/bfcl_v3/_base.py | 33 ++++++++++++++++------- tests/unit/tasks/bfcl_v3/test__base.py | 36 ++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/sieval/tasks/bfcl_v3/_base.py b/sieval/tasks/bfcl_v3/_base.py index f820051a..fbe4f993 100644 --- a/sieval/tasks/bfcl_v3/_base.py +++ b/sieval/tasks/bfcl_v3/_base.py @@ -21,11 +21,16 @@ reverses the rewrite before comparing against gold. That reversal is the ``underscore_to_dot`` flag: true for FC, false for Prompt. -**The flag travels as a call argument, never as a module global.** Grading runs -in a worker process, so a global set in the parent never arrives: an FC run -would then be graded under the Prompt convention, and the symptom is a -plausible-looking score rather than an error. That is the whole reason grading -goes through one module-level function instead of a method. +**The flag travels as a call argument, never as a module global set by the +caller.** The vendored checker reads it from a global, and `grade_single_turn` +writes that global and consumes it with no `await` in between -- which is what +makes the pair atomic, and is the whole reason grading goes through one +module-level function instead of a method. Offloading does not supply that +guarantee on its own: `run_cpu_bound` falls back to running inline on the event +loop whenever no worker pool is available, so a caller that set the global and +then awaited would let another sample's grade land between the set and the use. +Cross-wiring the two protocols is silent -- every one of the 734 gradeable rows +whose gold name carries a dot grades wrong, with nothing reported. AI-Generated Code - Claude Opus 5 (Anthropic) """ @@ -89,10 +94,11 @@ def grade_single_turn( ) -> bool: """Grade one decoded reply. Module-level so `run_cpu_bound` can pickle it. - `underscore_to_dot` is an ARGUMENT, not a module global set by the caller: - this runs in a worker process, so a global set in the parent never arrives. - Getting that wrong grades an FC run under the Prompt convention and produces - a plausible score rather than an error. + `underscore_to_dot` is an ARGUMENT, not a module global set by the caller, + so that the write below and the read inside `ast_checker` cannot be + separated: this function has no `await` in it, and an async caller setting + the global before awaiting would. Getting it wrong grades an FC run under + the Prompt convention and produces a plausible score rather than an error. `decoded` is None when the reply could not be decoded into calls at all. """ @@ -218,6 +224,15 @@ async def postprocess(self, inf, ctx) -> PredictionRecord: language = ctx.preprocess_result["extra"]["language"] try: decoded = self._decode(inf, language) + except ImportError: + # NOT a decode failure. `_decode` defers its import of the vendored + # parser, which lives behind the optional `bfcl-v3` extra, so an + # environment missing tree-sitter raises from inside this `try`. + # Scoring that as an undecodable reply is the worst available + # outcome: every row of the run grades wrong, `fails` stays 0, and + # the run looks like a model that cannot call a function at all. + # Propagating costs the sample and names the cause. + raise except Exception as exc: # noqa: BLE001 -- a decode failure is a SCORE # Upstream's `ast_decoder:decoder_failed`: the model produced # something undecodable, which is the model's outcome, not a fault. diff --git a/tests/unit/tasks/bfcl_v3/test__base.py b/tests/unit/tasks/bfcl_v3/test__base.py index 34074eb8..a3eec168 100644 --- a/tests/unit/tasks/bfcl_v3/test__base.py +++ b/tests/unit/tasks/bfcl_v3/test__base.py @@ -300,6 +300,42 @@ async def test_preprocess_stores_the_unprocessed_schema_for_grading(task_cls, ja ] == {"type": "ArrayList", "description": "tags", "items": {"type": "String"}} +class _Ctx: + preprocess_result = {"extra": {"language": "Python", "category": "simple"}} + + +@pytest.mark.anyio +async def test_an_undecodable_reply_is_scored_not_raised(): + """Upstream's `ast_decoder:decoder_failed` -- the model's outcome.""" + task = BfclV3NonLiveZeroShotGenTask(MockDataset(), MockChatModel()) + reply = _output("this is not a function call at all") + record = await task.postprocess(reply, _Ctx()) + (rollout,) = record["rollouts"] + assert rollout.get("prediction") is None + assert rollout["extracted"] is False + assert "decode_error" in rollout["extra"] + + +@pytest.mark.anyio +async def test_a_missing_parser_dependency_propagates_instead_of_scoring(monkeypatch): + """An absent optional extra is a broken environment, not a bad answer. + + `_decode` defers its import of the vendored parser (tree-sitter lives + behind the `bfcl-v3` extra), so the `ModuleNotFoundError` is raised from + inside the same `try` that turns a decode failure into a score. Swallowed, + it grades every row of the run wrong with `fails` at 0 -- indistinguishable + from a model that cannot call a function, and the logs go with the run. + """ + task = BfclV3NonLiveZeroShotGenTask(MockDataset(), MockChatModel()) + + def no_tree_sitter(_self, _output, _language): + raise ModuleNotFoundError("No module named 'tree_sitter'") + + monkeypatch.setattr(BfclV3PromptMixin, "_decode", no_tree_sitter) + with pytest.raises(ModuleNotFoundError, match="tree_sitter"): + await task.postprocess(_output("[f(x=1)]"), _Ctx()) + + def test_the_fc_mixin_sends_tools_and_renames_the_dotted_function(dotted_row): """Upstream's own rewrite -- the one `UNDERSCORE_TO_DOT` later reverses.""" messages, tools = BfclV3FCMixin()._build_messages(dotted_row) From 324615906c0e017da1387c1130bf20479fa24399 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 23:08:21 +0800 Subject: [PATCH 24/28] fix(tasks): stop BFCL v3's shipped notes from claiming an unmodified grader `BFCL_V3_SHARED_NOTES` said the vendored parsers were "vendored verbatim ... only `bfcl_eval.*` imports were rewritten." That stopped being true one commit ago, when `parser.py`'s `ast.BinOp` branch was changed to walk the argument's AST instead of `eval`-ing it. `parser.py`'s own docstring records the deviation; the notes did not, and the notes are the copy that ships -- they reach all four tasks' `reference_impl` and are embedded verbatim in `sieval/meta/index.json`, so an archived run's `meta.json` asserted an unmodified grading path. That claim is exactly the one a reader consults before arguing a divergence, so a false "verbatim" is worse than no note at all: it answers the question wrongly instead of prompting a look. The text now names the deviation, what it computes instead, and that the FC protocol never reaches it -- FC parses no source text, so two of the four tasks carry the note for a path they do not execute. `meta/index.json` regenerated; the delta is the four BFCL entries and nothing else. Co-Authored-By: Claude Opus 5 --- sieval/meta/index.json | 8 ++++---- sieval/tasks/bfcl_v3/_base.py | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index a5e67674..27446750 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -1794,7 +1794,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored verbatim from gorilla v1.3 (Apache-2.0); only `bfcl_eval.*` imports were rewritten. exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." }, "status": "experimental", "reference_kind": "value" @@ -1815,7 +1815,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored verbatim from gorilla v1.3 (Apache-2.0); only `bfcl_eval.*` imports were rewritten. exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." }, "status": "experimental", "reference_kind": "value" @@ -1836,7 +1836,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored verbatim from gorilla v1.3 (Apache-2.0); only `bfcl_eval.*` imports were rewritten. exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." }, "status": "experimental", "reference_kind": "value" @@ -1857,7 +1857,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored verbatim from gorilla v1.3 (Apache-2.0); only `bfcl_eval.*` imports were rewritten. exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." }, "status": "experimental", "reference_kind": "value" diff --git a/sieval/tasks/bfcl_v3/_base.py b/sieval/tasks/bfcl_v3/_base.py index fbe4f993..4efd8391 100644 --- a/sieval/tasks/bfcl_v3/_base.py +++ b/sieval/tasks/bfcl_v3/_base.py @@ -146,8 +146,12 @@ def grade_single_turn( #: group and protocol it is. The specific half is prepended by each leaf. BFCL_V3_SHARED_NOTES = ( "AST checker, source parsers, type converters, prompt and aggregation " - "vendored verbatim from gorilla v1.3 (Apache-2.0); only `bfcl_eval.*` " - "imports were rewritten. " + "vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for " + "`bfcl_eval.*` import rewrites and one execution-safety deviation -- the " + "Prompt decoder resolves an argument's arithmetic by walking the AST " + "rather than by `eval`-ing model output, which computes the same value " + "for every expression that does not execute something (the FC protocol " + "parses no source text, so it never reaches that code). " "exec_*/rest/sql/chatable are excluded -- upstream has them commented out of " "its own TEST_FILE_MAPPING and they need live API keys." ) From 6b83864e720a232d8eb935b85fa89d8013228571 Mon Sep 17 00:00:00 2001 From: Ethan Date: Sat, 12 Sep 2026 00:22:56 +0800 Subject: [PATCH 25/28] fix(tasks): reword BFCL v3's excluded-category list so it stops reading as a path The shared notes named the four skipped groups as `exec_*/rest/sql/chatable`, which carries a `/rest/sql` substring. `scripts/sanitize.sh`'s config scan flags any absolute-looking path whose first segment is not a system root, and the `*` before the first slash clears its lookbehind, so the regex matched -- on all four BFCL entries of `sieval/meta/index.json`, which embeds these notes verbatim. CI's sanitize job failed on those four lines. Reworded to a comma list instead of adding an ALLOWLIST entry: the pattern is a security gate, an exemption for it would have to be scoped by file, and the four categories were never a path to begin with -- they are BFCL test-file group names. `sieval/meta/index.json` regenerated; the delta is those four `notes` strings and nothing else. Co-Authored-By: Claude Opus 5 --- sieval/meta/index.json | 8 ++++---- sieval/tasks/bfcl_v3/_base.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 27446750..439d11fa 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -1794,7 +1794,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." }, "status": "experimental", "reference_kind": "value" @@ -1815,7 +1815,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." }, "status": "experimental", "reference_kind": "value" @@ -1836,7 +1836,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." }, "status": "experimental", "reference_kind": "value" @@ -1857,7 +1857,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*/rest/sql/chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." }, "status": "experimental", "reference_kind": "value" diff --git a/sieval/tasks/bfcl_v3/_base.py b/sieval/tasks/bfcl_v3/_base.py index 4efd8391..9f8e7c1a 100644 --- a/sieval/tasks/bfcl_v3/_base.py +++ b/sieval/tasks/bfcl_v3/_base.py @@ -152,8 +152,8 @@ def grade_single_turn( "rather than by `eval`-ing model output, which computes the same value " "for every expression that does not execute something (the FC protocol " "parses no source text, so it never reaches that code). " - "exec_*/rest/sql/chatable are excluded -- upstream has them commented out of " - "its own TEST_FILE_MAPPING and they need live API keys." + "exec_*, rest, sql and chatable are excluded -- upstream has them commented " + "out of its own TEST_FILE_MAPPING and they need live API keys." ) From a8a7bd7d5724ab25733f1c3eef639fa865631e03 Mon Sep 17 00:00:00 2001 From: Ethan Date: Sat, 12 Sep 2026 01:21:00 +0800 Subject: [PATCH 26/28] fix(community): spell BFCL v3's cross-package import absolutely, and bound string `%` Two review findings, plus two doc nits. `..type_mappings` in the two vendored type converters was the only thing that needed `sieval/community/` exempted from the cross-package relative-import rule. Upstream's own import was absolute (`bfcl_eval.constants.type_mappings`) and a vendored copy cannot keep it, so that line was already rewritten -- spelling the replacement absolutely costs the same one line and satisfies the rule as it stands. The exemption is reverted: preflight is the only surface that checks `community/` at all, since pre-commit's global `exclude` skips it, so a carve-out there would have taken private-module protection with it for every future vendored drop. The mirror tests now pin the opposite decision. `_safe_eval` screened `Pow`, `LShift` and `Mult` but not `Mod`, so `%` formatting on a string was caught only by the post-hoc size check -- after the allocation. `"%.400000000f" % 1.0` asks for a 400MB string from two tiny operands, which is the class the guard exists to close; a precision field sets the result size independently of both operands, so there is nothing to measure and the shape is refused outright. Integer `%` is bounded by its right operand and stays admissible. Not a regression -- upstream's `eval` has the same exposure -- and still score-neutral, since no gold row reaches `safe_eval`. Also: the dataset docstrings pinned the harness to the `v1.3` tag while `_base.py` pinned the commit; both now carry the SHA. And `tree_sitter` is spelled `tree-sitter` to match its two siblings, re-locked with `--update-reuse` (content_hash only, no version drift). Verified: 7575 unit tests pass, 25/25 preflight checks PASS, ruff and ty clean, and the gold replay still walks 2351 rows failing only the 7 documented upstream gold defects. The new `%` test is reverse-mutation checked -- its 22-char case, which a post-hoc size check admits, returns a value with the guard deleted. Co-Authored-By: Claude Opus 5 (1M context) --- pdm.lock | 2 +- pyproject.toml | 2 +- scripts/check_layer_imports.py | 28 +++++---------- scripts/check_preflight.py | 23 ++++++------ sieval/community/bfcl_v3/_safe_eval.py | 20 ++++++++++- sieval/community/bfcl_v3/parser.py | 2 +- .../type_convertor/java_type_converter.py | 12 ++++--- .../type_convertor/js_type_converter.py | 12 ++++--- sieval/datasets/bfcl_v3_live.py | 2 +- sieval/datasets/bfcl_v3_non_live.py | 2 +- .../unit/community/bfcl_v3/test__safe_eval.py | 28 +++++++++++++++ tests/unit/community/bfcl_v3/test_identity.py | 6 ++-- .../unit/scripts/test_check_layer_imports.py | 35 ++++++++++++------- 13 files changed, 115 insertions(+), 59 deletions(-) diff --git a/pdm.lock b/pdm.lock index 14cf586b..6473a983 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "bfcl-v3", "dev", "drop", "hle", "ifbench", "ifeval", "iheval", "math", "multi-if", "nl2sh-alfa", "ruler", "ruler-gen", "scicode", "spider", "spider2", "t-eval", "test", "wikisql"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:f23e18a1776f70c47e1953d93775d51c9aa69cec822c0398116c3752eb787bfc" +content_hash = "sha256:762c06f720accabcee892c676f5a69a3273b7fd9b991f94e53d28a481e4e2da4" [[metadata.targets]] requires_python = ">=3.12,<3.15" diff --git a/pyproject.toml b/pyproject.toml index 06157b49..0cb8cdfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ bfcl-v3 = [ # break -- the vendored parsers call `Language(ptr, name)` and # `Parser.set_language()`, both removed later -- so this is a hard pin, not a # floor. It has no wheel for Python 3.13 and builds from sdist. - "tree_sitter==0.21.3", + "tree-sitter==0.21.3", "tree-sitter-java==0.21.0", "tree-sitter-javascript==0.21.4", ] diff --git a/scripts/check_layer_imports.py b/scripts/check_layer_imports.py index e5a58b88..9d38a123 100644 --- a/scripts/check_layer_imports.py +++ b/scripts/check_layer_imports.py @@ -457,29 +457,19 @@ def _check_relative_scope(path: Path, tree: ast.AST) -> list[str]: not a package, so a relative import there fails at runtime and needs no lint. (The pre-commit hook feeds both trees; this check narrows on purpose.) - ``sieval/community/`` is exempt from **this rule only**. Vendored files - mirror upstream byte-for-byte and a hash test pins them, so the only fix - this rule can offer there — rewrite the import — is the one edit that tree - forbids. The exemption is deliberately narrower than pre-commit's global - ``exclude: ^(sieval/community/|vendor/)``, which skips every hook and so - every rule: we drop the style half and keep the other three over - ``community/``, a tree pre-commit does not check at all. - - Be precise about what that is worth, because the number is one and not - three: only **private-module protection** actually binds under - ``community/``. The layer and sub-package rules are vacuous there — - ``FORBIDDEN`` and ``FORBIDDEN_SUBPACKAGE`` have no ``community`` entry, so - a vendored file may import anything it likes. Losing the private-module - rule is therefore the entire cost of a wider exemption, and it is reason - enough: do not "finish the job" by widening this to ``_check_file``. + ``sieval/community/`` is **not** exempt, and a vendored drop is not a reason + to make it one. A vendored file that reaches across packages has already had + that import rewritten — upstream's own was absolute (``from + bfcl_eval.constants...``), and a copy cannot keep it — so spelling the + replacement absolutely costs exactly what spelling it ``..x`` costs: one + line, already being edited. Exempting the tree to permit the relative form + would buy nothing and would drop this rule over every future drop, which is + a ratchet: lint exemptions are not narrowed back. """ - layer = _get_layer(path) - if layer is None: + if _get_layer(path) is None: return [] if "tests" in path.parts: return [] - if layer == "community": - return [] file_pkg = _file_package(path) errors: list[str] = [] diff --git a/scripts/check_preflight.py b/scripts/check_preflight.py index 00e3c4bb..47ef77fd 100644 --- a/scripts/check_preflight.py +++ b/scripts/check_preflight.py @@ -3076,17 +3076,18 @@ def check_imports(self) -> list[CheckResult]: # # KNOWN divergence, not parity: pre-commit additionally applies the # global `exclude: ^(sieval/community/|vendor/)`, so it skips - # `sieval/community/` for every hook while this wrapper checks it. The - # vendored drop this comment used to predict has arrived — a file using - # `from ..x import y`, whose only offered fix was to edit code kept - # byte-identical to upstream. The call was made in - # `_check_relative_scope`: `community/` is exempt from the - # relative-import rule and from nothing else, so private-module - # protection still covers a tree pre-commit does not check at all. - # That one rule is the whole of what is protected here — the layer and - # sub-package rules have no `community` entry and are vacuous on it — - # and it is enough. Keep the exemption there. Hoisting it here, or into - # `_check_file`, would silently drop it. + # `sieval/community/` for every hook while this wrapper checks it. That + # makes this wrapper the only enforcement `community/` gets, which is + # the reason to keep feeding it the tree rather than to carve it out. + # + # The vendored drop this comment used to predict has arrived — a file + # reaching across packages, where upstream's own import was absolute and + # the copy could not keep it. It is spelled absolutely, so the rule is + # satisfied without an exemption: rewriting that import was unavoidable + # either way, and the relative spelling was the only version that would + # have needed one. Do not add the carve-out here, in `_check_file`, or + # in `_check_relative_scope` — it would silently drop private-module + # protection over a tree nothing else checks. enforced_py = [ f for f in self._git_tracked_files(".py") diff --git a/sieval/community/bfcl_v3/_safe_eval.py b/sieval/community/bfcl_v3/_safe_eval.py index 5fd8ac04..1d40b979 100644 --- a/sieval/community/bfcl_v3/_safe_eval.py +++ b/sieval/community/bfcl_v3/_safe_eval.py @@ -16,7 +16,7 @@ the entire divergence: every expression upstream's `eval` computes *without* executing something is computed here too, to the same value. -Two shapes are refused that do not execute, and both are deliberate: +Three shapes are refused that do not execute, and all three are deliberate: * An f-string, even one with no placeholders. `ast.literal_eval` draws the line in the same place, and the alternative is a `FormattedValue` walker guarding a @@ -26,6 +26,14 @@ (the grade timeout is in the `feedback` worker, a stage later), so ``f(x=9**9**9)`` would not return a wrong answer; it would stall every other sample in the run. +* ``%`` formatting on a string or bytes left operand. It belongs to the class + above, but it is the one member of it that cannot be *screened*: every other + eager operator's cost is a function of its operands' sizes, while a printf + precision field sets the result size on its own -- ``"%.400000000f" % 1.0`` + asks for a 400MB string from two tiny inputs. There is nothing to measure + before the fact, and :func:`_bounded` would see it only after the allocation + it exists to prevent, so the shape is refused outright. Integer ``%`` is + bounded by its right operand and stays admissible. Refusal raises `ValueError`, which reaches the decoder's caller as a decode failure and scores the sample wrong. That is the right outcome and not a @@ -100,8 +108,18 @@ def _refuse_if_unbounded(op, left, right) -> None: bitwise operators need no entry -- their result is at most as large as the sum of their inputs, so :func:`safe_eval`'s check on the value that comes back catches them one step later. + + String `%` is the one shape here that is refused rather than screened, + because its result size comes from the format spec instead of from the + operands -- see the module docstring. """ grown = None + if isinstance(op, ast.Mod) and isinstance(left, (str, bytes)): + raise ValueError( + "refusing `%` formatting on a str/bytes operand: a precision field " + "sets the result size independently of both operands, so the cost " + "cannot be bounded before the allocation" + ) if isinstance(op, (ast.Pow, ast.LShift)) and isinstance(right, int): if isinstance(right, bool) or right < 0: return diff --git a/sieval/community/bfcl_v3/parser.py b/sieval/community/bfcl_v3/parser.py index 3488f9fd..3b5982d6 100644 --- a/sieval/community/bfcl_v3/parser.py +++ b/sieval/community/bfcl_v3/parser.py @@ -24,7 +24,7 @@ + 0)` runs the call and hands back an ordinary-looking number. `safe_eval` computes the same value for every expression that does not execute something, and refuses the rest; the argument for where it lives, and for - the two non-executing shapes it also refuses, is in `_safe_eval.py`. + the non-executing shapes it also refuses, is in `_safe_eval.py`. The `ast.Lambda` branch below keeps upstream's `eval` untouched, because it cannot reach it: `Lambda.body` is a single expression node, so `value.body[0]` diff --git a/sieval/community/bfcl_v3/type_convertor/java_type_converter.py b/sieval/community/bfcl_v3/type_convertor/java_type_converter.py index 98a09564..74e6d9fd 100644 --- a/sieval/community/bfcl_v3/type_convertor/java_type_converter.py +++ b/sieval/community/bfcl_v3/type_convertor/java_type_converter.py @@ -5,14 +5,18 @@ License: Apache-2.0 Only deviation: `from bfcl_eval.constants.type_mappings import JAVA_TYPE_CONVERSION` -rewritten to `from ..type_mappings import JAVA_TYPE_CONVERSION` (this module lives -one package level deeper than `type_mappings.py`, hence the double dot). No -other edit. +rewritten to `from sieval.community.bfcl_v3.type_mappings import +JAVA_TYPE_CONVERSION`. Spelled absolutely rather than as the shorter +`..type_mappings`: this module sits one package deeper than `type_mappings.py`, +and the import policy reserves relative imports for the same package. Upstream's +line was absolute too, so this is one rewritten import either way -- and writing +it relatively would have cost the whole `community/` tree its exemption from +that rule. No other edit. """ import re from typing import List, Dict, Union -from ..type_mappings import JAVA_TYPE_CONVERSION +from sieval.community.bfcl_v3.type_mappings import JAVA_TYPE_CONVERSION def java_type_converter(value, expected_type, nested_type=None): diff --git a/sieval/community/bfcl_v3/type_convertor/js_type_converter.py b/sieval/community/bfcl_v3/type_convertor/js_type_converter.py index 7189664a..748e5321 100644 --- a/sieval/community/bfcl_v3/type_convertor/js_type_converter.py +++ b/sieval/community/bfcl_v3/type_convertor/js_type_converter.py @@ -5,13 +5,17 @@ License: Apache-2.0 Only deviation: `from bfcl_eval.constants.type_mappings import JS_TYPE_CONVERSION` -rewritten to `from ..type_mappings import JS_TYPE_CONVERSION` (this module lives -one package level deeper than `type_mappings.py`, hence the double dot). No -other edit. +rewritten to `from sieval.community.bfcl_v3.type_mappings import +JS_TYPE_CONVERSION`. Spelled absolutely rather than as the shorter +`..type_mappings`: this module sits one package deeper than `type_mappings.py`, +and the import policy reserves relative imports for the same package. Upstream's +line was absolute too, so this is one rewritten import either way -- and writing +it relatively would have cost the whole `community/` tree its exemption from +that rule. No other edit. """ import re -from ..type_mappings import JS_TYPE_CONVERSION +from sieval.community.bfcl_v3.type_mappings import JS_TYPE_CONVERSION def js_type_converter(value, expected_type, nested_type=None): diff --git a/sieval/datasets/bfcl_v3_live.py b/sieval/datasets/bfcl_v3_live.py index 2256fd7b..368d75f5 100644 --- a/sieval/datasets/bfcl_v3_live.py +++ b/sieval/datasets/bfcl_v3_live.py @@ -10,7 +10,7 @@ References: * Blog: -* Harness: +* Harness: * Dataset: AI-Generated Code - Claude Opus 5 (Anthropic) diff --git a/sieval/datasets/bfcl_v3_non_live.py b/sieval/datasets/bfcl_v3_non_live.py index d47d917c..92dbeb6d 100644 --- a/sieval/datasets/bfcl_v3_non_live.py +++ b/sieval/datasets/bfcl_v3_non_live.py @@ -10,7 +10,7 @@ References: * Blog: -* Harness: +* Harness: * Dataset: AI-Generated Code - Claude Opus 5 (Anthropic) diff --git a/tests/unit/community/bfcl_v3/test__safe_eval.py b/tests/unit/community/bfcl_v3/test__safe_eval.py index 26c490a9..a3f413f5 100644 --- a/tests/unit/community/bfcl_v3/test__safe_eval.py +++ b/tests/unit/community/bfcl_v3/test__safe_eval.py @@ -131,6 +131,34 @@ def test_an_unbounded_expression_is_refused_before_it_is_computed(source): resolve_ast_by_type(_argument(source)) +@pytest.mark.parametrize( + "source", + [ + "f(x='%.400000000f' % 1.0)", # ~400MB of string from two tiny operands + "f(x='%.20f' % 1.0)", # 22 chars -- admissible by SIZE, refused anyway + "f(x=b'%.400000000f' % 1.0)", + ], +) +def test_percent_formatting_on_a_string_is_refused_outright(source): + """Refused by SHAPE, not by size -- which is the whole point. + + A precision field sets the result size independently of both operands, so + there is nothing for `_refuse_if_unbounded` to measure and `_bounded` would + see the value only after the allocation it exists to prevent. The 22-char + case is the discriminating one: it is far under `MAX_RESULT_SIZE`, so a + post-hoc size check admits it. Only a categorical refusal rejects it, and + only a categorical refusal bounds the 400MB sibling before it is built. + """ + with pytest.raises(ValueError, match="refusing `%` formatting"): + resolve_ast_by_type(_argument(source)) + + +def test_integer_modulo_is_still_admissible(): + """`%` on integers is bounded by its right operand, so it keeps working.""" + assert safe_eval(_argument("f(x=10 % 3)")) == 1 + assert safe_eval(_argument("f(x=10 ** 100 % 7)")) == 10**100 % 7 + + def test_the_bound_admits_far_more_than_any_real_argument(): """Evidence the cap does not bind: the pinned largest literal is 13 digits.""" assert safe_eval(_argument("f(x=2 ** 4096)")) == 2**4096 diff --git a/tests/unit/community/bfcl_v3/test_identity.py b/tests/unit/community/bfcl_v3/test_identity.py index c2b66f23..e6b87445 100644 --- a/tests/unit/community/bfcl_v3/test_identity.py +++ b/tests/unit/community/bfcl_v3/test_identity.py @@ -45,10 +45,10 @@ "cebc6f9a1b28e277483dd13b2a3fd3ceb95ad8a4631daf50933ad1af3015c698" ), "type_convertor/java_type_converter.py": ( - "d33be3f7e65f0302ef729b0ae328e4d7395758b9b92d8d4562365f97dbc84942" + "4b450f96bfe528823de681e9d6b0b7fb748ca51a534b2d7a59d76d1047dfebf2" ), "type_convertor/js_type_converter.py": ( - "dfb0373b58b51c45e63346e1a893c055168dd6ae39e27307436fefa45981e6cc" + "ef9562e296bcd7aeeff8b2b88f0fba1c996951b91b229fc91a03bf0f855fcfe2" ), "source_parser/java_parser.py": ( "024dfc247dbb7890c4ed969ab48a9fab3212b85bdb74402528a5ce7b8abab5fe" @@ -56,7 +56,7 @@ "source_parser/js_parser.py": ( "3a096d914cc54514009e11915e8d4cc4c35af26b5a1c9f21ced640cd896dc737" ), - "parser.py": ("a3e1606a074e664c2fe180afad853c53c4635feb0e5b99044d5e8740fd163683"), + "parser.py": ("bed1fd6c50b4164dd6f4277ef54b1eae908491f5476e7830f742f614fe7266f9"), "tool_convert.py": ( "11b758572a09fca738479243e4cc3005387def672786b3231c868e6d73ad386e" ), diff --git a/tests/unit/scripts/test_check_layer_imports.py b/tests/unit/scripts/test_check_layer_imports.py index dd04350e..075b40f1 100644 --- a/tests/unit/scripts/test_check_layer_imports.py +++ b/tests/unit/scripts/test_check_layer_imports.py @@ -800,15 +800,27 @@ def test_scripts_tree_is_out_of_scope(self, tmp_path: Path): f = self._write(tmp_path, "scripts/x.py", "from ..y import z\n") assert _check_relative_scope(f, ast.parse(f.read_text())) == [] - def test_community_tree_is_exempt(self, tmp_path: Path): - # Vendored files mirror upstream byte-for-byte and a hash test pins - # them, so the only fix this rule can offer — rewrite the import — is - # the one edit that tree forbids. + def test_community_tree_is_not_exempt(self, tmp_path: Path): + # Vendored code gets no carve-out from this rule. A copy cannot keep + # upstream's absolute `bfcl_eval.*` import, so the line is rewritten + # either way and can be spelled absolutely for free; exempting the tree + # to allow `..x` would drop the rule over every future drop. f = self._write( tmp_path, "sieval/community/bfcl_v3/type_convertor/java_type_converter.py", "from ..type_mappings import JAVA_TYPE_CONVERSION\n", ) + errors = _check_relative_scope(f, ast.parse(f.read_text())) + assert len(errors) == 1 + assert "cross-package relative import '..type_mappings'" in errors[0] + + def test_community_absolute_import_passes(self, tmp_path: Path): + """The spelling the vendored converters actually ship.""" + f = self._write( + tmp_path, + "sieval/community/bfcl_v3/type_convertor/java_type_converter.py", + "from sieval.community.bfcl_v3.type_mappings import JAVA_TYPE_CONVERSION\n", + ) assert _check_relative_scope(f, ast.parse(f.read_text())) == [] @@ -953,13 +965,12 @@ def test_level_1_undotted_subpackage_private_name_stays_exempt( ) assert _check_file(f) == [] - def test_community_exempt_from_rule_3_only(self, tmp_path: Path): - # The vendored carve-out is scoped to the relative-import rule and to - # nothing else. Hoisting it into `_check_file` — or excluding - # `community/` from the preflight wrapper's file list, which is what - # pre-commit's global `exclude` effectively does — would also drop the - # private-access, layer and sub-package rules over the whole tree. This - # is the half that stops a later widening. + def test_community_gets_every_rule(self, tmp_path: Path): + # `community/` has no carve-out from any of the four rules. Preflight is + # the only surface that checks this tree at all — pre-commit's global + # `exclude` skips it for every hook — so a carve-out added here, in + # `_check_relative_scope`, or in the preflight wrapper's file list would + # take private-module protection with it. This pins both halves. f = self._write( tmp_path, "sieval/community/bfcl_v3/type_convertor/java_type_converter.py", @@ -967,7 +978,7 @@ def test_community_exempt_from_rule_3_only(self, tmp_path: Path): "from sieval.core.utils import _private_helper\n", ) errors = _check_file(f) - assert not any("cross-package relative import" in e for e in errors) + assert any("cross-package relative import" in e for e in errors) assert any("import of private name '_private_helper'" in e for e in errors) From e167624f5f3843a10f8bfb6d3afde32a7a075536 Mon Sep 17 00:00:00 2001 From: Ethan Date: Sat, 12 Sep 2026 02:45:34 +0800 Subject: [PATCH 27/28] feat(tasks): promote BFCL v3 to stable, anchored on upstream's released rollouts BFCL publishes an evaluation archive beside its leaderboard (HuanzhiMao/BFCL-Result): per model, per protocol, per category, both the model's recorded replies AND upstream's own per-row verdicts. Replaying gpt-4.1-2025-04-14's rollouts through our grader reproduces every published cell exactly and agrees with upstream's verdict on every row: protocol group rows agreement Prompt non-live 1390 1390/1390 Prompt live 2251 2251/2251 FC non-live 1390 1390/1390 FC live 2251 2251/2251 All 26 per-category accuracies match to the digit, so the four leaves now record their own published numbers the way the other reproduction tasks do, and ship `stable`. The sentence they carried -- "experimental until an alignment run against a published BFCL v3 number lands" -- was the only thing blocking it, and it is now false. This is a strictly stronger anchor than the gold replay, which feeds gold back and so only ever exercises the correct path: it never sees a wrong answer, a decode failure, or either protocol's decoder. The new acceptance test drives the production `_decode` of both mixins over output that is 9% wrong. `java` at 64% Prompt / 61% FC is the point -- a third of those rows are wrong and we have to agree about which third. It also pins the two rollups against their own documented claims: live's weighted mean reconstructs the pooled rate over all 2251 rows, and non-live's mean-of-means is asserted NOT to equal the pooled rate, so a "fix" toward pooling fails loudly. Both mutations it is meant to catch were checked by reverse-mutation: flipping FC's `UNDERSCORE_TO_DOT` (the silent cross-wiring `_base` warns about) and forcing the Prompt decoder to language="Python" (so Java/JavaScript stop reaching tree-sitter) each fail the test. What the anchor does NOT measure is stated in both the test and the shipped notes: the replies are upstream's, so nothing here proves our prompt is upstream's. That rests on the two model-facing helpers being byte-identical to upstream and running on a row-for-row verified snapshot -- a construction argument, not a measurement. The archive is ~6MB, so it is staged on demand rather than vendored, and the test skips with a reason naming the layout. No file hashes are pinned: every score file's header is asserted against the pinned counts, so a swapped archive fails, and a doctored result file fails the comparison it feeds. Verified: 7575 unit tests pass, 9 acceptance tests pass with the archive staged and skip cleanly without it, 25/25 preflight checks PASS (meta/index.json regenerated), ruff and ty clean. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/meta/index.json | 16 +- sieval/tasks/bfcl_v3/_base.py | 12 +- .../tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py | 11 +- .../bfcl_v3/bfcl_v3_live_0shot_gen_fc.py | 11 +- .../bfcl_v3/bfcl_v3_non_live_0shot_gen.py | 11 +- .../bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py | 12 +- .../bfcl_v3/test_upstream_rollout_replay.py | 348 ++++++++++++++++++ 7 files changed, 399 insertions(+), 22 deletions(-) create mode 100644 tests/acceptance/bfcl_v3/test_upstream_rollout_replay.py diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 439d11fa..09a6c009 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -1794,9 +1794,9 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. Replaying upstream's released gpt-4.1-2025-04-14 (Prompt) rollouts reproduces all six published category accuracies exactly and agrees with upstream's own verdict on 2251/2251 rows: live_simple 85.66, live_multiple 76.54, live_parallel 93.75, live_parallel_multiple 75.00, live_irrelevance 77.89, live_relevance 88.89 -- so ast_summary 78.46 and Live Overall Acc 78.32. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's own released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14 snapshot): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every row's verdict compared against upstream's own per-row score file. That pins the grader against a real model's output -- wrong answers, decode failures and the FC name rewrite included -- none of which feeding gold back can reach. The anchor does NOT measure prompt construction: the two model-facing helpers are byte-identical to upstream and run on a row-for-row verified snapshot, so prompt fidelity follows by construction, not by measurement." }, - "status": "experimental", + "status": "stable", "reference_kind": "value" }, { @@ -1815,9 +1815,9 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. Replaying upstream's released gpt-4.1-2025-04-14-FC rollouts reproduces all six published category accuracies exactly and agrees with upstream's own verdict on 2251/2251 rows: live_simple 80.23, live_multiple 78.35, live_parallel 68.75, live_parallel_multiple 66.67, live_irrelevance 82.31, live_relevance 77.78 -- so ast_summary 78.39 and Live Overall Acc 79.92. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's own released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14 snapshot): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every row's verdict compared against upstream's own per-row score file. That pins the grader against a real model's output -- wrong answers, decode failures and the FC name rewrite included -- none of which feeding gold back can reach. The anchor does NOT measure prompt construction: the two model-facing helpers are byte-identical to upstream and run on a row-for-row verified snapshot, so prompt fidelity follows by construction, not by measurement." }, - "status": "experimental", + "status": "stable", "reference_kind": "value" }, { @@ -1836,9 +1836,9 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. Replaying upstream's released gpt-4.1-2025-04-14 (Prompt) rollouts reproduces all seven published category accuracies exactly and agrees with upstream's own verdict on 1390/1390 rows: simple 95.50, multiple 94.00, parallel 93.00, parallel_multiple 87.50, java 64.00, javascript 82.00, irrelevance 88.75 -- so simple_ast 80.50 and Non_Live Overall Acc 88.75. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's own released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14 snapshot): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every row's verdict compared against upstream's own per-row score file. That pins the grader against a real model's output -- wrong answers, decode failures and the FC name rewrite included -- none of which feeding gold back can reach. The anchor does NOT measure prompt construction: the two model-facing helpers are byte-identical to upstream and run on a row-for-row verified snapshot, so prompt fidelity follows by construction, not by measurement." }, - "status": "experimental", + "status": "stable", "reference_kind": "value" }, { @@ -1857,9 +1857,9 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. experimental until an alignment run against a published BFCL v3 number lands. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys." + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. Replaying upstream's released gpt-4.1-2025-04-14-FC rollouts reproduces all seven published category accuracies exactly and agrees with upstream's own verdict on 1390/1390 rows: simple 93.50, multiple 90.50, parallel 91.00, parallel_multiple 86.00, java 61.00, javascript 68.00, irrelevance 89.58 -- so simple_ast 74.17 and Non_Live Overall Acc 86.25. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's own released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14 snapshot): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every row's verdict compared against upstream's own per-row score file. That pins the grader against a real model's output -- wrong answers, decode failures and the FC name rewrite included -- none of which feeding gold back can reach. The anchor does NOT measure prompt construction: the two model-facing helpers are byte-identical to upstream and run on a row-for-row verified snapshot, so prompt fidelity follows by construction, not by measurement." }, - "status": "experimental", + "status": "stable", "reference_kind": "value" }, { diff --git a/sieval/tasks/bfcl_v3/_base.py b/sieval/tasks/bfcl_v3/_base.py index 9f8e7c1a..bd50aace 100644 --- a/sieval/tasks/bfcl_v3/_base.py +++ b/sieval/tasks/bfcl_v3/_base.py @@ -153,7 +153,17 @@ def grade_single_turn( "for every expression that does not execute something (the FC protocol " "parses no source text, so it never reaches that code). " "exec_*, rest, sql and chatable are excluded -- upstream has them commented " - "out of its own TEST_FILE_MAPPING and they need live API keys." + "out of its own TEST_FILE_MAPPING and they need live API keys. " + "Anchored on upstream's own released evaluation archive " + "(HuanzhiMao/BFCL-Result, 2025-06-14 snapshot): its recorded " + "gpt-4.1-2025-04-14 replies are replayed through this grader and every " + "row's verdict compared against upstream's own per-row score file. That " + "pins the grader against a real model's output -- wrong answers, decode " + "failures and the FC name rewrite included -- none of which feeding gold " + "back can reach. The anchor does NOT measure prompt construction: the two " + "model-facing helpers are byte-identical to upstream and run on a " + "row-for-row verified snapshot, so prompt fidelity follows by " + "construction, not by measurement." ) diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py index bf1508ae..7bb0890b 100644 --- a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py +++ b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py @@ -53,7 +53,7 @@ # inference, and then every sample dies at postprocess. deps_group="bfcl-v3", model_type="chat", - status="experimental", + status="stable", reference_kind="value", reference_impl=ReferenceImpl( source="gorilla/berkeley-function-call-leaderboard", @@ -66,8 +66,13 @@ "`ast_summary` (weighted over the four AST categories) carries one " "on `n_ast` for the same reason. underscore_to_dot=False: schemas " "reach the model verbatim, so dotted function names survive and " - "gold needs no rewrite. experimental until an alignment run " - f"against a published BFCL v3 number lands. {BFCL_V3_SHARED_NOTES}" + "gold needs no rewrite. Replaying upstream's released " + "gpt-4.1-2025-04-14 (Prompt) rollouts reproduces all six published " + "category accuracies exactly and agrees with upstream's own " + "verdict on 2251/2251 rows: live_simple 85.66, live_multiple " + "76.54, live_parallel 93.75, live_parallel_multiple 75.00, " + "live_irrelevance 77.89, live_relevance 88.89 -- so ast_summary " + f"78.46 and Live Overall Acc 78.32. {BFCL_V3_SHARED_NOTES}" ), ), ) diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py index 1239ee7a..490fae10 100644 --- a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py +++ b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py @@ -54,7 +54,7 @@ n_shot=0, tags=("english", "function-calling"), model_type="chat", - status="experimental", + status="stable", reference_kind="value", reference_impl=ReferenceImpl( source="gorilla/berkeley-function-call-leaderboard", @@ -72,8 +72,13 @@ "`function_tools` capability; a binding that does not offer it is " "rejected when the task is constructed, which prelaunch " "reconciliation reaches before the first request and before a " - "result directory exists. experimental until an alignment run " - f"against a published BFCL v3 number lands. {BFCL_V3_SHARED_NOTES}" + "result directory exists. Replaying upstream's released " + "gpt-4.1-2025-04-14-FC rollouts reproduces all six published " + "category accuracies exactly and agrees with upstream's own " + "verdict on 2251/2251 rows: live_simple 80.23, live_multiple " + "78.35, live_parallel 68.75, live_parallel_multiple 66.67, " + "live_irrelevance 82.31, live_relevance 77.78 -- so ast_summary " + f"78.39 and Live Overall Acc 79.92. {BFCL_V3_SHARED_NOTES}" ), ), ) diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py index 4dd9c7e3..4b35def8 100644 --- a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py +++ b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py @@ -53,7 +53,7 @@ # for inference, and then every sample dies at postprocess. deps_group="bfcl-v3", model_type="chat", - status="experimental", + status="stable", reference_kind="value", reference_impl=ReferenceImpl( source="gorilla/berkeley-function-call-leaderboard", @@ -69,8 +69,13 @@ "sample-clustered CI would bracket a different statistic. Every " "per-category rate carries its own. underscore_to_dot=False: " "schemas reach the model verbatim, so dotted function names survive " - "and gold needs no rewrite. experimental until an alignment run " - f"against a published BFCL v3 number lands. {BFCL_V3_SHARED_NOTES}" + "and gold needs no rewrite. Replaying upstream's released " + "gpt-4.1-2025-04-14 (Prompt) rollouts reproduces all seven " + "published category accuracies exactly and agrees with upstream's " + "own verdict on 1390/1390 rows: simple 95.50, multiple 94.00, " + "parallel 93.00, parallel_multiple 87.50, java 64.00, javascript " + "82.00, irrelevance 88.75 -- so simple_ast 80.50 and Non_Live " + f"Overall Acc 88.75. {BFCL_V3_SHARED_NOTES}" ), ), ) diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py index 09943fed..c7952936 100644 --- a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py +++ b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py @@ -54,7 +54,7 @@ n_shot=0, tags=("english", "function-calling"), model_type="chat", - status="experimental", + status="stable", reference_kind="value", reference_impl=ReferenceImpl( source="gorilla/berkeley-function-call-leaderboard", @@ -74,9 +74,13 @@ "comparison. Requires the `function_tools` capability; a binding " "that does not offer it is rejected when the task is constructed, " "which prelaunch reconciliation reaches before the first request " - "and before a result directory exists. experimental until an " - f"alignment run against a published BFCL v3 number lands. " - f"{BFCL_V3_SHARED_NOTES}" + "and before a result directory exists. Replaying upstream's " + "released gpt-4.1-2025-04-14-FC rollouts reproduces all seven " + "published category accuracies exactly and agrees with upstream's " + "own verdict on 1390/1390 rows: simple 93.50, multiple 90.50, " + "parallel 91.00, parallel_multiple 86.00, java 61.00, javascript " + "68.00, irrelevance 89.58 -- so simple_ast 74.17 and Non_Live " + f"Overall Acc 86.25. {BFCL_V3_SHARED_NOTES}" ), ), ) diff --git a/tests/acceptance/bfcl_v3/test_upstream_rollout_replay.py b/tests/acceptance/bfcl_v3/test_upstream_rollout_replay.py new file mode 100644 index 00000000..e7e867ae --- /dev/null +++ b/tests/acceptance/bfcl_v3/test_upstream_rollout_replay.py @@ -0,0 +1,348 @@ +"""Anchor BFCL v3 grading on upstream's own released rollouts and verdicts. + +BFCL publishes an evaluation archive beside the leaderboard -- per model, per +protocol, per category, it releases both the model's recorded replies and its +own per-row verdicts. That is per-prediction ground truth, so the port can be +pinned exactly, with no model spend. + +This is strictly stronger than `test_gold_replay.py`, which feeds gold back as +the reply. Gold replay can only ever exercise the *correct* path: it never sees +a wrong answer, a decode failure, or -- because it drives the checker directly +-- either protocol's decoder. This module replays a real model's output, of +which 9% is wrong, through the production `_decode` of both mixins. `java` at +64% is the point: a third of those rows are graded wrong by upstream, and we +have to agree about which third. + +What it does NOT cover is prompt construction: the replies are upstream's, so +nothing here proves our prompt is upstream's prompt. That rests on +`system_prompt_pre_processing_chat_model` and +`func_doc_language_specific_pre_processing` being byte-identical to upstream +(pinned by `tests/unit/community/bfcl_v3/test_identity.py`) and running on a +row-for-row verified snapshot -- a construction argument, not a measurement. + +Staging (opt-in, like the gold replay -- ~6MB, so it is not vendored): + + R=https://raw.githubusercontent.com/HuanzhiMao/BFCL-Result/main/2025-06-14 + D="$SIEVAL_DATA_DIR/HuanzhiMao/BFCL-Result/2025-06-14" + for m in gpt-4.1-2025-04-14 gpt-4.1-2025-04-14-FC; do + for k in result score; do + mkdir -p "$D/$k/$m" + for c in simple multiple parallel parallel_multiple java javascript \ + irrelevance live_simple live_multiple live_parallel \ + live_parallel_multiple live_irrelevance live_relevance; do + curl -sSf -o "$D/$k/$m/BFCL_v3_${c}_${k}.json" \ + "$R/$k/$m/BFCL_v3_${c}_${k}.json" + done + done + done + +No file hashes are pinned, because they would be redundant: every score file's +header is asserted against :data:`PUBLISHED` below, so a swapped archive fails; +and a doctored `result` file fails the row-for-row comparison it feeds. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import collections +import json +import pathlib + +import pytest + +from sieval.core.models import FunctionToolCall, GenModel, ModelOutput +from sieval.core.utils.hf import maybe_resolve_hf_path +from sieval.datasets import BfclV3LiveDataset, BfclV3NonLiveDataset +from sieval.tasks.bfcl_v3._base import ( + BfclV3FCMixin, + BfclV3LiveTask, + BfclV3NonLiveTask, + BfclV3PromptMixin, + grade_single_turn, +) + +#: The archive snapshot these counts were read from. It predates the pinned +#: harness revision (v1.3, 2025-07-17) by a month, and agrees anyway -- which is +#: itself evidence the checker did not move for these categories in between. +ARCHIVE = "HuanzhiMao/BFCL-Result" +SNAPSHOT = "2025-06-14" + +DATA_SNAPSHOT = "gorilla-llm/Berkeley-Function-Calling-Leaderboard" + +NON_LIVE = ( + "simple", + "multiple", + "parallel", + "parallel_multiple", + "java", + "javascript", + "irrelevance", +) +LIVE = ( + "live_simple", + "live_multiple", + "live_parallel", + "live_parallel_multiple", + "live_irrelevance", + "live_relevance", +) + +#: `(correct_count, total_count)` off each score file's header line -- upstream's +#: published cell, as integers rather than the rounded percentage the +#: leaderboard renders. Asserted both against the staged file (archive +#: integrity) and against our own replay (the actual anchor). +PUBLISHED = { + "gpt-4.1-2025-04-14": { + "simple": (382, 400), + "multiple": (188, 200), + "parallel": (186, 200), + "parallel_multiple": (175, 200), + "java": (64, 100), + "javascript": (41, 50), + "irrelevance": (213, 240), + "live_simple": (221, 258), + "live_multiple": (806, 1053), + "live_parallel": (15, 16), + "live_parallel_multiple": (18, 24), + "live_irrelevance": (687, 882), + "live_relevance": (16, 18), + }, + "gpt-4.1-2025-04-14-FC": { + "simple": (374, 400), + "multiple": (181, 200), + "parallel": (182, 200), + "parallel_multiple": (172, 200), + "java": (61, 100), + "javascript": (34, 50), + "irrelevance": (215, 240), + "live_simple": (207, 258), + "live_multiple": (825, 1053), + "live_parallel": (11, 16), + "live_parallel_multiple": (16, 24), + "live_irrelevance": (726, 882), + "live_relevance": (14, 18), + }, +} + +#: Protocol under test: archive model dir, and the mixin whose production +#: `_decode` and `UNDERSCORE_TO_DOT` are exercised. +PROTOCOLS = { + "prompt": ("gpt-4.1-2025-04-14", BfclV3PromptMixin), + "fc": ("gpt-4.1-2025-04-14-FC", BfclV3FCMixin), +} + +_ROOT = pathlib.Path(maybe_resolve_hf_path(ARCHIVE)) / SNAPSHOT + +requires_archive = pytest.mark.skipif( + not _ROOT.is_dir(), + reason=f"needs the staged BFCL result archive ({ARCHIVE} @ {SNAPSHOT})", +) + + +def _jsonl(path: pathlib.Path) -> list[dict]: + return [json.loads(x) for x in path.open(encoding="utf-8") if x.strip()] + + +def _model_output(protocol: str, recorded) -> ModelOutput: + """Upstream's recorded reply, in the shape the mixin's `_decode` reads. + + Built as a real `ModelOutput` so the production decoders run on the type + they run on in a live eval, rather than on a stand-in that happens to have + the two attributes they touch. + + The FC archive stores each call as `{name: arguments-JSON-string}`, which is + exactly what the OpenAI dialects put on `FunctionToolCall.arguments`, so the + calls are rebuilt rather than pre-parsed -- `_call_arguments` is then part of + what this anchors. + """ + meta = GenModel(model="archive-replay", api_key="none").meta() + if protocol == "prompt": + text = recorded if isinstance(recorded, str) else "" + return ModelOutput(model=meta, texts=[text]) + calls = [] + for item in recorded if isinstance(recorded, list) else []: + if not isinstance(item, dict): + continue + for index, (name, arguments) in enumerate(item.items()): + calls.append( + FunctionToolCall( + f"call_{len(calls)}_{index}", + name, + arguments if isinstance(arguments, str) else json.dumps(arguments), + ) + ) + return ModelOutput(model=meta, texts=[""], tool_calls=tuple(calls)) + + +def _decode_or_fail(decoder, protocol: str, recorded, language): + """The mixin's production `_decode`, under `postprocess`'s failure semantic. + + `postprocess` wraps the call in `except Exception -> [None]`, because an + undecodable reply is upstream's `ast_decoder:decoder_failed` -- a SCORE, not + a fault -- and that boundary has to be mirrored here or a real reply takes + the test out. It is reached often on this data: a model that declines in + prose ("None of the functions can be used, as ...") raises `SyntaxError` + straight out of `ast.parse`. Only the two lines of that semantic are + mirrored; the decoder itself is the shipped one, and `postprocess`'s + narrower re-raise of `ImportError` is pinned in the unit tests. + """ + try: + return decoder._decode(_model_output(protocol, recorded), language) + except Exception: # noqa: BLE001 -- a decode failure is a score + return None + + +def _replay(protocol: str, categories, dataset) -> dict[str, tuple[int, int]]: + """Grade every recorded reply; return `{category: (ours, total)}`. + + Asserts row-for-row agreement with upstream's verdicts on the way, so a + disagreement names the row rather than only moving a total. + """ + model_dir, mixin = PROTOCOLS[protocol] + decoder = mixin() + + # `live_relevance` ships one id twice (18 rows, 17 distinct ids), so rows and + # recorded replies are both consumed from per-id queues. Keying either by id + # would drop that row and quietly score 17. + rows = collections.defaultdict(collections.deque) + for row in dataset.test_set: + rows[row["id"]].append(row) + + out: dict[str, tuple[int, int]] = {} + for category in categories: + base = _ROOT / "{kind}" / model_dir / f"BFCL_v3_{category}_{{kind}}.json" + results = _jsonl(pathlib.Path(str(base).format(kind="result"))) + scored = _jsonl(pathlib.Path(str(base).format(kind="score"))) + + header = scored[0] + expected = PUBLISHED[model_dir][category] + assert (header["correct_count"], header["total_count"]) == expected, ( + f"{model_dir}/{category}: staged archive publishes " + f"{header['correct_count']}/{header['total_count']}, pinned " + f"{expected[0]}/{expected[1]} -- the archive moved, so every number " + "below was measured against a different corpus." + ) + wrong = collections.Counter(r["id"] for r in scored[1:] if "id" in r) + + pool = {k: collections.deque(v) for k, v in rows.items()} + correct = walked = 0 + disagreed = [] + for record in results: + queue = pool.get(record["id"]) + if not queue: + continue + row = queue.popleft() + walked += 1 + ours = grade_single_turn( + row["function"], + _decode_or_fail(decoder, protocol, record["result"], row["language"]), + row["ground_truth"], + row["language"], + category, + mixin.UNDERSCORE_TO_DOT, + ) + correct += int(ours) + if ours != (wrong[record["id"]] == 0): + disagreed.append(record["id"]) + + assert walked == expected[1], ( + f"{model_dir}/{category}: replayed {walked} of {expected[1]} rows. " + "A short walk scores the missing rows as absent, not as wrong." + ) + assert not disagreed, ( + f"{model_dir}/{category}: our verdict differs from upstream's on " + f"{len(disagreed)} row(s), e.g. {disagreed[:5]}" + ) + out[category] = (correct, expected[1]) + return out + + +@requires_archive +@pytest.mark.parametrize("protocol", sorted(PROTOCOLS)) +@pytest.mark.parametrize( + ("group", "categories"), [("non_live", NON_LIVE), ("live", LIVE)] +) +def test_replaying_upstream_rollouts_reproduces_every_published_cell( + protocol: str, group: str, categories +): + dataset = ( + BfclV3NonLiveDataset(DATA_SNAPSHOT) + if group == "non_live" + else BfclV3LiveDataset(DATA_SNAPSHOT) + ) + model_dir, _ = PROTOCOLS[protocol] + replayed = _replay(protocol, categories, dataset) + assert replayed == {c: PUBLISHED[model_dir][c] for c in categories} + + +@requires_archive +@pytest.mark.parametrize("protocol", sorted(PROTOCOLS)) +def test_the_live_rollup_is_the_pooled_rate_over_every_live_row(protocol: str): + """The weighted rollup's own claim, checked rather than asserted in prose. + + `BfclV3LiveTask` documents its sample-count-weighted mean as algebraically + the pooled rate over the union of the six categories' rows. If that holds, + the headline is reconstructible from the published counts alone -- which is + what makes it a genuine per-sample rate and so interval-bearing. + """ + model_dir, _ = PROTOCOLS[protocol] + cells = {c: PUBLISHED[model_dir][c] for c in LIVE} + + def cell(category: str) -> dict: + correct, total = cells[category] + accuracy = 100.0 * correct / total + return { + "accuracy": accuracy, + "total_count": total, + "display_accuracy": accuracy, + } + + # Built without `__init__`, which would demand a live model binding: the + # rollup reads only ClassVars. This runs the PRODUCTION aggregation rather + # than a second copy of its arithmetic. + task = object.__new__(BfclV3LiveTask) + rollup = task._aggregate(cell) # noqa: SLF001 -- tests are the carve-out + pooled = ( + 100.0 * sum(c for c, _ in cells.values()) / sum(t for _, t in cells.values()) + ) + assert rollup["live_overall_acc"] == pytest.approx(pooled, abs=1e-9) + + +@requires_archive +@pytest.mark.parametrize("protocol", sorted(PROTOCOLS)) +def test_the_non_live_rollup_is_the_documented_mean_of_means(protocol: str): + """Non-live nests `simple_ast` inside a five-way unweighted mean. + + Deliberately NOT the pooled rate: javascript's 50 rows weigh as much as + simple's 400, which is why the task publishes no interval on this headline. + Pinned here so a "fix" toward pooling fails loudly. + """ + model_dir, _ = PROTOCOLS[protocol] + cells = {c: PUBLISHED[model_dir][c] for c in NON_LIVE} + + def rate(category: str) -> float: + correct, total = cells[category] + return 100.0 * correct / total + + def cell(category: str) -> dict: + return { + "accuracy": rate(category), + "total_count": cells[category][1], + "display_accuracy": rate(category), + } + + task = object.__new__(BfclV3NonLiveTask) + rollup = task._aggregate(cell) # noqa: SLF001 -- tests are the carve-out + simple_ast = sum(rate(c) for c in ("simple", "java", "javascript")) / 3 + expected = ( + simple_ast + + sum( + rate(c) + for c in ("multiple", "parallel", "parallel_multiple", "irrelevance") + ) + ) / 5 + assert rollup["simple_ast"] == pytest.approx(simple_ast, abs=1e-9) + assert rollup["non_live_overall_acc"] == pytest.approx(expected, abs=1e-9) + pooled = ( + 100.0 * sum(c for c, _ in cells.values()) / sum(t for _, t in cells.values()) + ) + assert rollup["non_live_overall_acc"] != pytest.approx(pooled, abs=1e-6) From 56ecec382dcb9aeef45f9ad523028f383c1edf40 Mon Sep 17 00:00:00 2001 From: Ethan Date: Sat, 12 Sep 2026 03:10:36 +0800 Subject: [PATCH 28/28] docs(tasks): cut the BFCL v3 prose back to what the code does not already say MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The port shipped 1751 lines of comment and docstring against 3689 of code, and the bulk of the excess was one fact stated in several places rather than any single essay. Net -140 lines over 18 files, with no fact dropped. Give each fact one owner: - The four leaves each said the same thing twice -- a module docstring and a `reference_impl.notes` restating the headline formula and the interval policy in different words. `notes` keeps them, because it is what reaches `meta/index.json` and so the only copy a reader without the source ever sees; the docstrings now say what the task is and stop. - `underscore_to_dot` was explained four times (`_base` module docstring, `grade_single_turn`, both `_fc` leaves). `_base` owns the mechanism since the flag lives there; `grade_single_turn` owns the atomicity argument since it is that function's shape being justified, and it absorbs the 734-row number that quantifies the risk. - Why the unqualified name goes to Prompt was argued in every leaf and again in `tasks/CLAUDE.md` §Variants, which is the naming vocabulary's actual home. - The `source_parser/` rename was explained three times, at length. Also drops a private reference: `tool_convert.py` cited "Step 3's whole-file modules", a step number from an internal implementation plan that means nothing in a public checkout. Kept deliberately: every non-derivable fact, including the two that read as padding and are not -- `_safe_eval`'s calibration evidence (13-digit largest gold literal across 2351 rows) and the ImportError/`.get("prediction")` rationales, both of which describe silent failures. Test docstrings are left alone after sampling them: at 8-19 lines they are dense rather than padded, and the longest states two production failure modes a reader cannot infer. Prose-only, and checked rather than asserted: with docstrings stripped and string constants normalized, every touched file's AST is identical to its previous version. The only literals that move are one `notes` per leaf, the shared notes block, and the six vendored digests whose docstrings were trimmed. Verified: 7575 unit tests pass, 9 acceptance tests pass, 25/25 preflight checks PASS (meta/index.json regenerated), ruff and ty clean. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/community/CLAUDE.md | 16 ++- sieval/community/bfcl_v3/__init__.py | 43 +++---- sieval/community/bfcl_v3/_safe_eval.py | 99 +++++++--------- sieval/community/bfcl_v3/parser.py | 39 +++---- .../bfcl_v3/source_parser/java_parser.py | 9 +- .../bfcl_v3/source_parser/js_parser.py | 9 +- sieval/community/bfcl_v3/tool_convert.py | 20 ++-- .../type_convertor/java_type_converter.py | 11 +- .../type_convertor/js_type_converter.py | 11 +- sieval/datasets/_bfcl_v3.py | 46 ++++---- sieval/meta/index.json | 8 +- sieval/tasks/CLAUDE.md | 15 ++- sieval/tasks/bfcl_v3/_base.py | 107 +++++++----------- .../tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py | 34 +++--- .../bfcl_v3/bfcl_v3_live_0shot_gen_fc.py | 47 +++----- .../bfcl_v3/bfcl_v3_non_live_0shot_gen.py | 43 +++---- .../bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py | 60 ++++------ .../bfcl_v3/test_upstream_rollout_replay.py | 85 ++++++-------- tests/unit/community/bfcl_v3/test_identity.py | 46 ++++---- 19 files changed, 304 insertions(+), 444 deletions(-) diff --git a/sieval/community/CLAUDE.md b/sieval/community/CLAUDE.md index 9d474bfd..f9044bcf 100644 --- a/sieval/community/CLAUDE.md +++ b/sieval/community/CLAUDE.md @@ -26,15 +26,13 @@ inline where it would swamp a diff against upstream. Do not add more original code here without the same argument; a helper with one caller belongs in that caller's module. -`bfcl_v3/_safe_eval.py` is the second, and it has exactly **one** caller — so it -is the case that shows what "the same argument" means. The caller is a *vendored* -file (`bfcl_v3/parser.py`, whose `ast.BinOp` branch upstream resolves with `eval` -over model output). Inlining the guard there would enlarge a diff that exists to -be compared against upstream, and would put a security boundary inside a file -that cannot be linted — the surrounding vendored code does not pass lint, so -there would be no path to checking the boundary itself. "One caller" is the rule -for ordinary helpers; a boundary that has to stay separately auditable is not -one. +`bfcl_v3/_safe_eval.py` is the second, and with exactly **one** caller it shows +what "the same argument" means. That caller is a *vendored* file +(`bfcl_v3/parser.py`, whose `ast.BinOp` branch upstream resolves with `eval` over +model output), so inlining the guard would both enlarge a diff that exists to be +compared against upstream and put a security boundary inside a file that cannot +be linted. "One caller" is the rule for ordinary helpers; a boundary that has to +stay separately auditable is not one. The package-wide `ruff` / `mypy` / `pre-commit` exclusions exist to keep vendored code byte-identical and cover both files too, which is the wrong default for a diff --git a/sieval/community/bfcl_v3/__init__.py b/sieval/community/bfcl_v3/__init__.py index 0784c3c3..07726c7d 100644 --- a/sieval/community/bfcl_v3/__init__.py +++ b/sieval/community/bfcl_v3/__init__.py @@ -4,34 +4,25 @@ Revision: v1.3 (ea13468e4423454d0c213704fb87cf7cb3990433) License: Apache-2.0 -Upstream's evaluation surface is `bfcl_eval`, an installable package that -imports its own submodules by absolute path (`bfcl_eval.constants...`, -`bfcl_eval.model_handler...`). A vendored copy cannot keep those imports, since -`bfcl_eval` itself is not installed here -- rewriting them to relative imports -within this package is the only edit made to any vendored file, with one -exception: `parser.py`'s `ast.BinOp` branch resolves a model-authored expression -with `eval`, and calls `_safe_eval.safe_eval` instead. Each vendored module's -docstring records upstream's blob SHA at the pin and the exact edits applied, so -a reviewer can re-derive the copy from a fresh checkout. +Upstream's evaluation surface is `bfcl_eval`, which imports its own submodules +by absolute path. A vendored copy cannot keep those imports, so rewriting them +is the only edit made to any vendored file -- with one exception: `parser.py`'s +`ast.BinOp` branch calls `_safe_eval.safe_eval` where upstream `eval`s a +model-authored expression. Each vendored module records upstream's blob SHA at +the pin and its exact edits, so the copy can be re-derived from a fresh checkout. -Whole-file vendored modules: `ast_checker.py`, `type_mappings.py`, -`type_convertor/java_type_converter.py`, `type_convertor/js_type_converter.py`, -`source_parser/java_parser.py`, `source_parser/js_parser.py`. +Whole-file: `ast_checker.py`, `type_mappings.py`, `type_convertor/*.py`, +`source_parser/*.py`, and `prompts.py` (upstream's `constants/default_prompts`). +Partial -- named subsets of a larger upstream file, listed in each module's own +docstring: `parser.py`, `tool_convert.py`, `output_checks.py`, `preprocess.py`, +`aggregate.py`. -Partial vendored modules (named subsets of a larger upstream file -- see each -module's own docstring for exactly which symbols were taken and why): -`parser.py`, `tool_convert.py`, `output_checks.py`, `preprocess.py`, -`aggregate.py`. `prompts.py` is `constants/default_prompts.py` copied whole. - -First-party modules: `_model_config.py` supplies the `MODEL_CONFIG_MAPPING` -global `ast_checker.convert_func_name` reads, since upstream's own 2044-line -registry has no entries for sieval's models. `_tables.py` holds the BFCL v3 -category tables (row counts, which categories have no gold answer file, which -are non-Python) shared by the dataset loader and the tasks -- neither of which -may reach into the other's private module, so the tables live here instead. -`_safe_eval.py` evaluates a literal expression without executing it, and is the -one deviation `parser.py` carries; it is held outside that file so the boundary -stays lintable, which vendored code is not. +First-party: `_model_config.py` supplies the `MODEL_CONFIG_MAPPING` global +`ast_checker.convert_func_name` reads, since upstream's 2044-line registry has no +entry for sieval's models. `_tables.py` holds the category tables shared by the +dataset loader and the tasks, neither of which may reach into the other's private +module. `_safe_eval.py` is the deviation `parser.py` carries, held outside that +file so the boundary stays lintable -- vendored code is not. AI-Generated Code - Claude Opus 5 (Anthropic) """ diff --git a/sieval/community/bfcl_v3/_safe_eval.py b/sieval/community/bfcl_v3/_safe_eval.py index 1d40b979..7aba7a02 100644 --- a/sieval/community/bfcl_v3/_safe_eval.py +++ b/sieval/community/bfcl_v3/_safe_eval.py @@ -3,47 +3,38 @@ `resolve_ast_by_type` resolves each keyword argument of a decoded function call into a Python value, and for `ast.BinOp` upstream does it with ``eval(ast.unparse(value))``. The node comes from a model's reply, so that line -runs model-authored code: a reply carrying -``f(x=__import__('os').system('...') + 0)`` parses to a `BinOp`, `eval` runs the -call, and the decoder returns an ordinary-looking number. Nothing in the run -looks unusual afterwards -- the sample grades, the score is plausible, and the -only trace is whatever the payload did. +runs model-authored code: ``f(x=__import__('os').system('...') + 0)`` parses to +a `BinOp`, `eval` runs the call, and the decoder hands back an ordinary-looking +number with nothing else in the run looking unusual. :func:`safe_eval` replaces that one call. It walks the node instead of unparsing -it, so there is no string and no namespace to escape from, and it admits only -literals, their containers, and the arithmetic/bitwise operators. A `Name`, -`Call`, `Attribute`, `Subscript` or comprehension is refused outright, which is -the entire divergence: every expression upstream's `eval` computes *without* -executing something is computed here too, to the same value. - -Three shapes are refused that do not execute, and all three are deliberate: - -* An f-string, even one with no placeholders. `ast.literal_eval` draws the line - in the same place, and the alternative is a `FormattedValue` walker guarding a - shape no function argument has been observed to carry. -* An expression whose *cost* is unbounded -- see :data:`MAX_RESULT_SIZE`. - Decoding runs inline on the session's event loop with no timeout around it - (the grade timeout is in the `feedback` worker, a stage later), so - ``f(x=9**9**9)`` would not return a wrong answer; it would stall every other - sample in the run. -* ``%`` formatting on a string or bytes left operand. It belongs to the class - above, but it is the one member of it that cannot be *screened*: every other - eager operator's cost is a function of its operands' sizes, while a printf - precision field sets the result size on its own -- ``"%.400000000f" % 1.0`` - asks for a 400MB string from two tiny inputs. There is nothing to measure - before the fact, and :func:`_bounded` would see it only after the allocation - it exists to prevent, so the shape is refused outright. Integer ``%`` is - bounded by its right operand and stays admissible. - -Refusal raises `ValueError`, which reaches the decoder's caller as a decode -failure and scores the sample wrong. That is the right outcome and not a -softening of it: a reply that puts executable code in an argument slot has not -produced the function call it was asked for, and upstream scores every other -unparseable reply the same way. - -This is a restriction on what is evaluated, not a sandbox. It holds because the -accepted node types cannot name anything outside the expression; it would stop -holding the moment one that can is added. +it, so there is no string and no namespace to escape from, and admits only +literals, their containers, and the arithmetic/bitwise operators; a `Name`, +`Call`, `Attribute`, `Subscript` or comprehension is refused. That is the entire +divergence: every expression upstream's `eval` computes *without* executing +something is computed here too, to the same value. + +Three shapes are refused that do not execute: + +* An f-string, even with no placeholders -- where `ast.literal_eval` draws the + line, and the alternative is a `FormattedValue` walker for a shape no argument + has been observed to carry. +* An expression whose *cost* is unbounded (:data:`MAX_RESULT_SIZE`). Decoding + runs inline on the event loop with no timeout -- the grade timeout is a stage + later -- so ``f(x=9**9**9)`` would stall the whole run, not one sample. +* ``%`` on a str/bytes left operand. Same class, but the one member that cannot + be *screened*: a printf precision field sets the result size independently of + both operands (``"%.400000000f" % 1.0`` is 400MB from two tiny inputs), so + there is nothing to measure and :func:`_bounded` would see it only after the + allocation. Integer ``%`` is bounded by its right operand and stays allowed. + +Refusal raises `ValueError`, which reaches the caller as a decode failure and +scores the sample wrong -- the right outcome, since a reply putting executable +code in an argument slot has not produced the call it was asked for. + +A restriction on what is evaluated, not a sandbox: it holds because the accepted +node types cannot name anything outside the expression, and would stop holding +the moment one that can is added. AI-Generated Code - Claude Opus 5 (Anthropic) """ @@ -51,15 +42,13 @@ import ast import operator -#: Largest intermediate any step may produce, counted in bits for integers and -#: in items for everything sized. A bound is needed because the operators are -#: eager: `9 ** 9 ** 9` asks for a 369-million-digit integer and does not -#: return, and `'a' * 10**9` allocates a gigabyte before anything can inspect -#: it. 2**20 leaves a million-bit integer and a million-character string -#: admissible, against a largest integer literal of 13 digits (about 41 bits) -#: across the gold answers of all 2351 gradeable Python rows in the pinned -#: snapshot -- and no argument there reaches this module at all, since none of -#: them is written as an expression. +#: Largest intermediate any step may produce -- bits for integers, items for +#: anything sized. Needed because the operators are eager: `9 ** 9 ** 9` asks +#: for a 369-million-digit integer and does not return. 2**20 admits a +#: million-bit integer and a million-character string, against a largest gold +#: literal of 13 digits (~41 bits) across all 2351 gradeable Python rows in the +#: pinned snapshot -- none of which reaches this module at all, since no gold +#: argument is written as an expression. MAX_RESULT_SIZE = 1 << 20 _BIN_OPS = { @@ -103,15 +92,11 @@ def _size(value) -> int: def _refuse_if_unbounded(op, left, right) -> None: """Screen the operators whose cost their operands' size does not bound. - Checked *before* the operator runs, which is the only time it can be: the - cost of `9 ** 9 ** 9` cannot be measured after the fact. Addition and the - bitwise operators need no entry -- their result is at most as large as the - sum of their inputs, so :func:`safe_eval`'s check on the value that comes - back catches them one step later. - - String `%` is the one shape here that is refused rather than screened, - because its result size comes from the format spec instead of from the - operands -- see the module docstring. + Checked *before* the operator runs, the only time it can be: the cost of + `9 ** 9 ** 9` cannot be measured after the fact. Addition and the bitwise + operators need no entry -- their result is at most the sum of their inputs, + so :func:`_bounded` catches them one step later. String `%` is refused + rather than screened; see the module docstring. """ grown = None if isinstance(op, ast.Mod) and isinstance(left, (str, bytes)): diff --git a/sieval/community/bfcl_v3/parser.py b/sieval/community/bfcl_v3/parser.py index 3b5982d6..dcde4a48 100644 --- a/sieval/community/bfcl_v3/parser.py +++ b/sieval/community/bfcl_v3/parser.py @@ -5,37 +5,30 @@ License: Apache-2.0 Symbols taken (bodies unchanged): `ast_parse`, `resolve_ast_call`, -`resolve_ast_by_type`. This is a partial extraction, not a whole-file copy -- -the rest of `model_handler/utils.py` (prompt construction, decoders for other -model styles, retry helpers) is out of scope for this port. +`resolve_ast_by_type`. A partial extraction -- the rest of +`model_handler/utils.py` (prompt construction, other model styles' decoders, +retry helpers) is out of scope. Deviations: -1. The module needs no `bfcl_eval.*` imports at all for these three - functions (`import ast` is stdlib), so their bodies needed no rewrite. -2. Upstream's `utils.py:13-14` imports `parse_java_function_call` and - `parse_javascript_function_call` from `bfcl_eval.model_handler.parser.*`. - Rewritten to `from .source_parser.java_parser import - parse_java_function_call` and the `js_parser` twin -- same rename as - `source_parser/`'s own docstrings explain (this package's `parser.py` - already occupies upstream's directory name). -3. `resolve_ast_by_type`'s `ast.BinOp` branch calls `safe_eval` where upstream +1. The two source-parser imports are rewritten to `.source_parser.*`, renamed + from upstream's `parser/` because this file already occupies that name. + Nothing else needed rewriting: these three functions reach no `bfcl_eval.*`. +2. `resolve_ast_by_type`'s `ast.BinOp` branch calls `safe_eval` where upstream calls `eval(ast.unparse(value))`. That node is model output, so upstream's line executes model-authored code -- `f(x=__import__('os').system('...') + 0)` runs the call and hands back an ordinary-looking number. `safe_eval` computes the same value for every expression that does not execute - something, and refuses the rest; the argument for where it lives, and for - the non-executing shapes it also refuses, is in `_safe_eval.py`. + something; what it refuses, and why it lives outside this file, is in + `_safe_eval.py`. - The `ast.Lambda` branch below keeps upstream's `eval` untouched, because it - cannot reach it: `Lambda.body` is a single expression node, so `value.body[0]` - raises `TypeError` while the argument is still being built. Hardening a - branch that cannot execute would be a deviation bought for nothing, and the - raise it produces today is already the decode failure upstream scores. + The `ast.Lambda` branch keeps upstream's `eval` because it cannot reach it: + `Lambda.body` is a single expression node, so `value.body[0]` raises + `TypeError` first -- and that raise is already the decode failure upstream + scores. Hardening an unreachable branch buys nothing. -The Java/JavaScript branches of `ast_parse` dispatch to those two functions: -a `java`/`javascript`-category sample's model output is Java/JavaScript -source text, not Python call syntax, so `ast_parse` parses it with the -matching tree-sitter grammar rather than `ast.parse`. +`ast_parse` dispatches `java`/`javascript` rows to the tree-sitter parsers +rather than `ast.parse`: their model output is source text, not Python call +syntax. """ import ast diff --git a/sieval/community/bfcl_v3/source_parser/java_parser.py b/sieval/community/bfcl_v3/source_parser/java_parser.py index 37148bab..4601211b 100644 --- a/sieval/community/bfcl_v3/source_parser/java_parser.py +++ b/sieval/community/bfcl_v3/source_parser/java_parser.py @@ -5,13 +5,10 @@ License: Apache-2.0 Only deviation: none beyond this docstring. The module imports only -`tree_sitter` and `tree_sitter_java` -- neither is `bfcl_eval.*` -- so there -was no import to rewrite. +`tree_sitter` and `tree_sitter_java`, neither of which is `bfcl_eval.*`. -Held in `source_parser/`, not upstream's `parser/`: this package's -`parser.py` (the vendored `ast_parse`) already occupies that name, so -upstream's containing directory was renamed to avoid the collision. Nothing -inside the file changed for the rename. +Held in `source_parser/` rather than upstream's `parser/`, whose name this +package's own `parser.py` already occupies. Nothing inside the file changed. """ from tree_sitter import Language, Parser diff --git a/sieval/community/bfcl_v3/source_parser/js_parser.py b/sieval/community/bfcl_v3/source_parser/js_parser.py index 0962ef2b..9421a45c 100644 --- a/sieval/community/bfcl_v3/source_parser/js_parser.py +++ b/sieval/community/bfcl_v3/source_parser/js_parser.py @@ -5,13 +5,10 @@ License: Apache-2.0 Only deviation: none beyond this docstring. The module imports only -`tree_sitter` and `tree_sitter_javascript` -- neither is `bfcl_eval.*` -- so -there was no import to rewrite. +`tree_sitter` and `tree_sitter_javascript`, neither of which is `bfcl_eval.*`. -Held in `source_parser/`, not upstream's `parser/`: this package's -`parser.py` (the vendored `ast_parse`) already occupies that name, so -upstream's containing directory was renamed to avoid the collision. Nothing -inside the file changed for the rename. +Held in `source_parser/` rather than upstream's `parser/`, whose name this +package's own `parser.py` already occupies. Nothing inside the file changed. """ from tree_sitter import Language, Parser diff --git a/sieval/community/bfcl_v3/tool_convert.py b/sieval/community/bfcl_v3/tool_convert.py index b8470cd7..d82d3fe7 100644 --- a/sieval/community/bfcl_v3/tool_convert.py +++ b/sieval/community/bfcl_v3/tool_convert.py @@ -5,23 +5,17 @@ License: Apache-2.0 Symbols taken (bodies unchanged): `convert_to_tool`, `_cast_to_openai_type`. -This is a partial extraction, not a whole-file copy -- the rest of -`model_handler/utils.py` is out of scope for this port. +A partial extraction -- the rest of `model_handler/utils.py` is out of scope. Deviations, beyond the usual import rewrite: - `_cast_to_openai_type` compares its `mapping` argument against the module - global `GORILLA_TO_OPENAPI`, so that name is imported from the already-vendored - `.type_mappings` -- the same relative-import rewrite Step 3's whole-file - modules get, just landing in a partial module instead. -- `convert_to_tool` branches on upstream's `ModelStyle` enum - (`bfcl_eval.model_handler.model_style.ModelStyle`). Rather than vendor that - whole file -- most of whose members name providers sieval does not model -- - only the members these two functions actually reference are copied below as - a plain enum: `OpenAI_Completions`, `OpenAI_Responses`, `Anthropic`, - `Mistral`, `GOOGLE`, `AMAZON`, `FIREWORK_AI`, `OSSMODEL`, `COHERE`, `WRITER`, - `NOVITA_AI`. Omitted: `Gorilla`, `NEXUS` (present upstream, never read by - either copied function). + global `GORILLA_TO_OPENAPI`, imported here from the vendored `.type_mappings`. +- `convert_to_tool` branches on upstream's `ModelStyle` + (`bfcl_eval.model_handler.model_style`). Rather than vendor that file, the + enum below copies the 11 members these two functions reference, with + upstream's values. `Gorilla` and `NEXUS` are omitted: present upstream, read + by neither copied function. """ import copy diff --git a/sieval/community/bfcl_v3/type_convertor/java_type_converter.py b/sieval/community/bfcl_v3/type_convertor/java_type_converter.py index 74e6d9fd..a558a8af 100644 --- a/sieval/community/bfcl_v3/type_convertor/java_type_converter.py +++ b/sieval/community/bfcl_v3/type_convertor/java_type_converter.py @@ -5,13 +5,10 @@ License: Apache-2.0 Only deviation: `from bfcl_eval.constants.type_mappings import JAVA_TYPE_CONVERSION` -rewritten to `from sieval.community.bfcl_v3.type_mappings import -JAVA_TYPE_CONVERSION`. Spelled absolutely rather than as the shorter -`..type_mappings`: this module sits one package deeper than `type_mappings.py`, -and the import policy reserves relative imports for the same package. Upstream's -line was absolute too, so this is one rewritten import either way -- and writing -it relatively would have cost the whole `community/` tree its exemption from -that rule. No other edit. +rewritten to the absolute `sieval.community.bfcl_v3.type_mappings` path. Absolute +rather than `..type_mappings` because the import policy reserves relative imports +for the same package, and upstream's line was absolute anyway -- one rewritten +import either way. No other edit. """ import re diff --git a/sieval/community/bfcl_v3/type_convertor/js_type_converter.py b/sieval/community/bfcl_v3/type_convertor/js_type_converter.py index 748e5321..f5f36a92 100644 --- a/sieval/community/bfcl_v3/type_convertor/js_type_converter.py +++ b/sieval/community/bfcl_v3/type_convertor/js_type_converter.py @@ -5,13 +5,10 @@ License: Apache-2.0 Only deviation: `from bfcl_eval.constants.type_mappings import JS_TYPE_CONVERSION` -rewritten to `from sieval.community.bfcl_v3.type_mappings import -JS_TYPE_CONVERSION`. Spelled absolutely rather than as the shorter -`..type_mappings`: this module sits one package deeper than `type_mappings.py`, -and the import policy reserves relative imports for the same package. Upstream's -line was absolute too, so this is one rewritten import either way -- and writing -it relatively would have cost the whole `community/` tree its exemption from -that rule. No other edit. +rewritten to the absolute `sieval.community.bfcl_v3.type_mappings` path. Absolute +rather than `..type_mappings` because the import policy reserves relative imports +for the same package, and upstream's line was absolute anyway -- one rewritten +import either way. No other edit. """ import re diff --git a/sieval/datasets/_bfcl_v3.py b/sieval/datasets/_bfcl_v3.py index 4e97a67f..b99a2689 100644 --- a/sieval/datasets/_bfcl_v3.py +++ b/sieval/datasets/_bfcl_v3.py @@ -1,33 +1,29 @@ """Shared loader for the BFCL v3 single-turn category files. One HF repo holds every category as its own JSONL file, with ground truth in a -parallel `possible_answer/` file. Three categories ship no gold at all -- -`irrelevance`, `live_irrelevance` and `live_relevance` are scored on whether a -call was produced, not on which call it was. +parallel `possible_answer/` file. Three categories ship no gold -- `irrelevance`, +`live_irrelevance` and `live_relevance` are scored on whether a call was +produced, not on which call it was. The join key is the **universal index** -- the `` of -`_[--]` -- not the whole `id` string. That is -upstream's key: its runner sorts both files on `(category, universal_index)`, -discarding the sub-indices, then pairs them positionally. The distinction is -load-bearing on exactly one row at the pinned revision: the prompt file's -`live_multiple_1052-79-0` is graded against the gold file's -`live_multiple_1052-279-0`. One of the two sub-indices is an upstream typo, but -because it never enters upstream's key, upstream grades the row and cannot see -the disagreement. Joining on the whole `id` would instead score 1052 of -`live_multiple`'s 1053 rows and report a column upstream never published. - -The key is unique within every file that is joined, which is what makes an index -join and upstream's positional join the same pairing; the loader asserts that -rather than assuming it, and so is strictly stricter than upstream, which only -checks that the two files are the same length. It is deliberately not asserted -for the goldless three, which are never joined -- `live_relevance` ships the -same row twice and is 18 rows over 17 distinct ids. - -Row counts are asserted per category, against the gold file as well as the -prompt file -- the pair of them is what covers upstream's own -`len(prompt) == len(possible_answer)`. The revision pin already prevents a -silent re-upload; what it cannot prevent is someone bumping the pin, so a count -that moves fails here rather than quietly rescoring a leaderboard column. +`_[--]` -- not the whole `id`. That is upstream's key: +its runner sorts both files on `(category, universal_index)` and pairs them +positionally, discarding the sub-indices. It is load-bearing on exactly one row +at the pinned revision, where prompt `live_multiple_1052-79-0` is graded against +gold `live_multiple_1052-279-0`: one sub-index is an upstream typo, invisible to +upstream because it never enters the key. Joining on the whole `id` would score +1052 of `live_multiple`'s 1053 rows and publish a column upstream never did. + +An index join equals upstream's positional one only while the key is unique per +joined file, so the loader asserts that instead of assuming it -- stricter than +upstream, which only checks the two files are the same length. Not asserted for +the goldless three, which are never joined: `live_relevance` ships one row twice, +18 rows over 17 distinct ids. + +Row counts are asserted per category on the gold file as well as the prompt file, +which together cover upstream's own `len(prompt) == len(possible_answer)`. The +revision pin stops a silent re-upload but not a pin bump, so a count that moves +fails here rather than quietly rescoring a leaderboard column. AI-Generated Code - Claude Opus 5 (Anthropic) """ diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 09a6c009..99133737 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -1794,7 +1794,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. Replaying upstream's released gpt-4.1-2025-04-14 (Prompt) rollouts reproduces all six published category accuracies exactly and agrees with upstream's own verdict on 2251/2251 rows: live_simple 85.66, live_multiple 76.54, live_parallel 93.75, live_parallel_multiple 75.00, live_irrelevance 77.89, live_relevance 88.89 -- so ast_summary 78.46 and Live Overall Acc 78.32. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's own released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14 snapshot): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every row's verdict compared against upstream's own per-row score file. That pins the grader against a real model's output -- wrong answers, decode failures and the FC name rewrite included -- none of which feeding gold back can reach. The anchor does NOT measure prompt construction: the two model-facing helpers are byte-identical to upstream and run on a row-for-row verified snapshot, so prompt fidelity follows by construction, not by measurement." + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. Replay of upstream's gpt-4.1-2025-04-14 (Prompt) rollouts agrees on 2251/2251 rows and reproduces every published cell: live_simple 85.66, live_multiple 76.54, live_parallel 93.75, live_parallel_multiple 75.00, live_irrelevance 77.89, live_relevance 88.89; ast_summary 78.46, Live Overall Acc 78.32. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every verdict compared against upstream's own per-row score file, which pins the grader against real output -- wrong answers, decode failures and the FC name rewrite included. It does not cover prompt construction: the replies are upstream's, so that rests on the two model-facing helpers being byte-identical to upstream rather than on a measurement." }, "status": "stable", "reference_kind": "value" @@ -1815,7 +1815,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. Replaying upstream's released gpt-4.1-2025-04-14-FC rollouts reproduces all six published category accuracies exactly and agrees with upstream's own verdict on 2251/2251 rows: live_simple 80.23, live_multiple 78.35, live_parallel 68.75, live_parallel_multiple 66.67, live_irrelevance 82.31, live_relevance 77.78 -- so ast_summary 78.39 and Live Overall Acc 79.92. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's own released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14 snapshot): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every row's verdict compared against upstream's own per-row score file. That pins the grader against a real model's output -- wrong answers, decode failures and the FC name rewrite included -- none of which feeding gold back can reach. The anchor does NOT measure prompt construction: the two model-facing helpers are byte-identical to upstream and run on a row-for-row verified snapshot, so prompt fidelity follows by construction, not by measurement." + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected at task construction, which prelaunch reconciliation reaches before the first request and before a result directory exists. Replay of upstream's gpt-4.1-2025-04-14-FC rollouts agrees on 2251/2251 rows and reproduces every published cell: live_simple 80.23, live_multiple 78.35, live_parallel 68.75, live_parallel_multiple 66.67, live_irrelevance 82.31, live_relevance 77.78; ast_summary 78.39, Live Overall Acc 79.92. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every verdict compared against upstream's own per-row score file, which pins the grader against real output -- wrong answers, decode failures and the FC name rewrite included. It does not cover prompt construction: the replies are upstream's, so that rests on the two model-facing helpers being byte-identical to upstream rather than on a measurement." }, "status": "stable", "reference_kind": "value" @@ -1836,7 +1836,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. Replaying upstream's released gpt-4.1-2025-04-14 (Prompt) rollouts reproduces all seven published category accuracies exactly and agrees with upstream's own verdict on 1390/1390 rows: simple 95.50, multiple 94.00, parallel 93.00, parallel_multiple 87.50, java 64.00, javascript 82.00, irrelevance 88.75 -- so simple_ast 80.50 and Non_Live Overall Acc 88.75. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's own released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14 snapshot): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every row's verdict compared against upstream's own per-row score file. That pins the grader against a real model's output -- wrong answers, decode failures and the FC name rewrite included -- none of which feeding gold back can reach. The anchor does NOT measure prompt construction: the two model-facing helpers are byte-identical to upstream and run on a row-for-row verified snapshot, so prompt fidelity follows by construction, not by measurement." + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. No interval on the headline, simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate (javascript's 50 rows weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. Replay of upstream's gpt-4.1-2025-04-14 (Prompt) rollouts agrees on 1390/1390 rows and reproduces every published cell: simple 95.50, multiple 94.00, parallel 93.00, parallel_multiple 87.50, java 64.00, javascript 82.00, irrelevance 88.75; simple_ast 80.50, Non_Live Overall Acc 88.75. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every verdict compared against upstream's own per-row score file, which pins the grader against real output -- wrong answers, decode failures and the FC name rewrite included. It does not cover prompt construction: the replies are upstream's, so that rests on the two model-facing helpers being byte-identical to upstream rather than on a measurement." }, "status": "stable", "reference_kind": "value" @@ -1857,7 +1857,7 @@ "reference_impl": { "source": "gorilla/berkeley-function-call-leaderboard", "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", - "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. NO interval is published on the headline, nor on simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate over the rows (javascript's 50 weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected when the task is constructed, which prelaunch reconciliation reaches before the first request and before a result directory exists. Replaying upstream's released gpt-4.1-2025-04-14-FC rollouts reproduces all seven published category accuracies exactly and agrees with upstream's own verdict on 1390/1390 rows: simple 93.50, multiple 90.50, parallel 91.00, parallel_multiple 86.00, java 61.00, javascript 68.00, irrelevance 89.58 -- so simple_ast 74.17 and Non_Live Overall Acc 86.25. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's own released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14 snapshot): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every row's verdict compared against upstream's own per-row score file. That pins the grader against a real model's output -- wrong answers, decode failures and the FC name rewrite included -- none of which feeding gold back can reach. The anchor does NOT measure prompt construction: the two model-facing helpers are byte-identical to upstream and run on a row-for-row verified snapshot, so prompt fidelity follows by construction, not by measurement." + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. No interval on the headline, simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate (javascript's 50 rows weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected at task construction, which prelaunch reconciliation reaches before the first request and before a result directory exists. Replay of upstream's gpt-4.1-2025-04-14-FC rollouts agrees on 1390/1390 rows and reproduces every published cell: simple 93.50, multiple 90.50, parallel 91.00, parallel_multiple 86.00, java 61.00, javascript 68.00, irrelevance 89.58; simple_ast 74.17, Non_Live Overall Acc 86.25. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every verdict compared against upstream's own per-row score file, which pins the grader against real output -- wrong answers, decode failures and the FC name rewrite included. It does not cover prompt construction: the replies are upstream's, so that rests on the two model-facing helpers being byte-identical to upstream rather than on a measurement." }, "status": "stable", "reference_kind": "value" diff --git a/sieval/tasks/CLAUDE.md b/sieval/tasks/CLAUDE.md index 1ce09813..2b58d6e7 100644 --- a/sieval/tasks/CLAUDE.md +++ b/sieval/tasks/CLAUDE.md @@ -46,14 +46,13 @@ name, so it is the only place the distinction can live. not for an extractor we added: the second column has to be upstream's. - **`_fc` is licensed by a capability, not a prompt.** It exists where upstream publishes a native-function-calling column beside its prompting one, and it is - a different axis from `_parse`: `_parse` layers a second extractor over the - same model interface, while `_fc` changes *which model capability is - exercised* — it gates on `function_tools`, so it changes which models can run - the task at all. The unqualified name goes to the prompting reading, because - every model on a function-calling leaderboard has that row while only some - have an FC row, and a default name that half the fleet cannot run is the - narrow one. `bfcl_v3_non_live_0shot_gen` / `_fc` is the case that earned the - row. + a different axis from `_parse`: that one layers a second extractor over the + same model interface, while `_fc` changes *which capability is exercised* — it + gates on `function_tools`, so it changes which models can run the task at all. + The unqualified name goes to the prompting reading, since every model on such a + leaderboard has that row while only some have an FC row, and a default name + half the fleet cannot run is the narrow one. + `bfcl_v3_non_live_0shot_gen` / `_fc` earned the row. - The mode is read positionally, so a variant may not spell one: `foo_0shot_clp_gen.py` has two readings and is rejected. - The table is the current vocabulary, not the limit — a new variant earns a row diff --git a/sieval/tasks/bfcl_v3/_base.py b/sieval/tasks/bfcl_v3/_base.py index bd50aace..54d72a01 100644 --- a/sieval/tasks/bfcl_v3/_base.py +++ b/sieval/tasks/bfcl_v3/_base.py @@ -1,36 +1,23 @@ """Shared stages for the BFCL v3 single-turn task family. -Four leaves come out of two independent choices, and this module holds what -every one of them shares. +Four leaves come out of two independent choices, and this module holds what they +share. -**Group** (``non_live`` / ``live``) selects the dataset and the aggregation -rule. It is the generic base, because it is what binds the concrete sample type -and what picks the rollup. +**Group** (``non_live`` / ``live``) selects the dataset and the rollup, so it is +the generic base. -**Protocol** (``Prompt`` / ``FC``) is upstream's own pair of published columns, -and it is the axis the two mixins here carry. They differ in exactly two -places: how the function schemas reach the model -- rendered into a system -turn, or sent as native tools -- and how calls are read back -- parsed out of -the reply text, or read off structured tool calls. Nothing else about the two -runs differs, which is why the protocol is a stateless mixin rather than a -second class hierarchy. +**Protocol** (``Prompt`` / ``FC``) is upstream's own pair of published columns. +The two mixins differ in exactly two places -- how the schemas reach the model +(a rendered system turn, or native tools) and how calls are read back (parsed +out of the reply, or off structured tool calls) -- which is why the protocol is +a stateless mixin rather than a second class hierarchy. **One divergence follows from the protocol, and it is upstream's.** A native tool name cannot carry a dot in the OpenAI dialects, so upstream rewrites ``geometry.triangle_area`` to ``geometry_triangle_area`` on the way out and reverses the rewrite before comparing against gold. That reversal is the -``underscore_to_dot`` flag: true for FC, false for Prompt. - -**The flag travels as a call argument, never as a module global set by the -caller.** The vendored checker reads it from a global, and `grade_single_turn` -writes that global and consumes it with no `await` in between -- which is what -makes the pair atomic, and is the whole reason grading goes through one -module-level function instead of a method. Offloading does not supply that -guarantee on its own: `run_cpu_bound` falls back to running inline on the event -loop whenever no worker pool is available, so a caller that set the global and -then awaited would let another sample's grade land between the set and the use. -Cross-wiring the two protocols is silent -- every one of the 734 gradeable rows -whose gold name carries a dot grades wrong, with nothing reported. +``underscore_to_dot`` flag -- true for FC, false for Prompt -- and +:func:`grade_single_turn` says why it travels as an argument. AI-Generated Code - Claude Opus 5 (Anthropic) """ @@ -97,8 +84,10 @@ def grade_single_turn( `underscore_to_dot` is an ARGUMENT, not a module global set by the caller, so that the write below and the read inside `ast_checker` cannot be separated: this function has no `await` in it, and an async caller setting - the global before awaiting would. Getting it wrong grades an FC run under - the Prompt convention and produces a plausible score rather than an error. + the global before awaiting would. Cross-wiring the two protocols is silent + -- all 734 gradeable rows whose gold name carries a dot grade wrong, with + nothing reported -- so the atomicity is the whole reason grading goes + through one module-level function rather than a method. `decoded` is None when the reply could not be decoded into calls at all. """ @@ -154,16 +143,14 @@ def grade_single_turn( "parses no source text, so it never reaches that code). " "exec_*, rest, sql and chatable are excluded -- upstream has them commented " "out of its own TEST_FILE_MAPPING and they need live API keys. " - "Anchored on upstream's own released evaluation archive " - "(HuanzhiMao/BFCL-Result, 2025-06-14 snapshot): its recorded " - "gpt-4.1-2025-04-14 replies are replayed through this grader and every " - "row's verdict compared against upstream's own per-row score file. That " - "pins the grader against a real model's output -- wrong answers, decode " - "failures and the FC name rewrite included -- none of which feeding gold " - "back can reach. The anchor does NOT measure prompt construction: the two " - "model-facing helpers are byte-identical to upstream and run on a " - "row-for-row verified snapshot, so prompt fidelity follows by " - "construction, not by measurement." + "Anchored on upstream's released evaluation archive (HuanzhiMao/BFCL-Result, " + "2025-06-14): its recorded gpt-4.1-2025-04-14 replies are replayed through " + "this grader and every verdict compared against upstream's own per-row score " + "file, which pins the grader against real output -- wrong answers, decode " + "failures and the FC name rewrite included. It does not cover prompt " + "construction: the replies are upstream's, so that rests on the two " + "model-facing helpers being byte-identical to upstream rather than on a " + "measurement." ) @@ -239,13 +226,11 @@ async def postprocess(self, inf, ctx) -> PredictionRecord: try: decoded = self._decode(inf, language) except ImportError: - # NOT a decode failure. `_decode` defers its import of the vendored - # parser, which lives behind the optional `bfcl-v3` extra, so an - # environment missing tree-sitter raises from inside this `try`. - # Scoring that as an undecodable reply is the worst available - # outcome: every row of the run grades wrong, `fails` stays 0, and - # the run looks like a model that cannot call a function at all. - # Propagating costs the sample and names the cause. + # NOT a decode failure: `_decode` defers its import of the vendored + # parser, which lives behind the optional `bfcl-v3` extra. Scoring a + # missing tree-sitter as an undecodable reply would grade every row + # of the run wrong with `fails` still 0; propagating costs the one + # sample and names the cause. raise except Exception as exc: # noqa: BLE001 -- a decode failure is a SCORE # Upstream's `ast_decoder:decoder_failed`: the model produced @@ -295,13 +280,11 @@ async def feedback(self, post, ctx) -> tuple[bool, JudgementRecord]: ) async def report(self, finals: list, fails: list) -> dict: - # One 0/1 per sample that came back, NOT one percentage point: this list - # is both what the cells average and what the interval estimators are - # handed, and those read `sum(values) / denominator` as a probability. - # Hand them percent and any rate above a single point reads as a - # saturated set, which still publishes a whole triple -- an interval - # bracketing 100 beside a rate that is nothing of the kind. The rate is - # scaled to percentage points once, in `cell`, so the two agree. + # One 0/1 per sample that came back, NOT one percentage point: the same + # list feeds the cells and the interval estimators, and those read + # `sum(values) / denominator` as a probability -- hand them percent and + # any rate above 1 point reads as saturated. Scaling happens once, in + # `cell`, so the rate and its bounds agree. correct_by_category: dict[str, list[float]] = defaultdict(list) for ctx in finals: fb = ctx.feedback_result @@ -314,21 +297,16 @@ def cell(category: str) -> dict: `total_count` is the category's row count, not the number that came back: `DENOMINATOR_REQUESTED` charges a missing sample as wrong, and - a sample that failed before `preprocess` has no category to be - attributed to anyway. - - All three of upstream's keys, because both aggregation helpers read - `display_accuracy` unconditionally and a two-key cell raises - `KeyError`. It is never "N/A" here: that marks a category upstream - did not evaluate, and this task always scores the full declared set. - Their return value carries the same three keys, which is what lets - `simple_ast` nest straight back in as a cell. + a sample that failed before `preprocess` has no category anyway. + + All three of upstream's keys: both aggregation helpers read + `display_accuracy` unconditionally, and they return the same three, + which is what lets `simple_ast` nest back in as a cell. Never "N/A" + -- that marks a category upstream did not evaluate. """ values = correct_by_category.get(category, []) denominator = self.CATEGORY_COUNTS[category] - # Percentage points -- the units every rate here is published in, - # and the units the estimators return their bounds in. - accuracy = 100.0 * sum(values) / denominator + accuracy = 100.0 * sum(values) / denominator # percentage points return { "accuracy": accuracy, "total_count": denominator, @@ -455,9 +433,8 @@ def _group_intervals( for value in correct_by_category.get(category, []) ] return [ - # The headline and `live_overall_acc` are ONE number under two key - # names, so the alias rides along on the same call -- never a second - # call with the same arguments. This emits `n_problems` itself. + # `score` and `live_overall_acc` are ONE number, so the alias rides + # along on this call. Emits `n_problems` itself. interval_metrics( all_values, denominator=sum(self.CATEGORY_COUNTS.values()), diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py index 7bb0890b..8bb021c0 100644 --- a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py +++ b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py @@ -1,20 +1,15 @@ """BFCL v3 live categories, 0-shot, upstream's prompting protocol. Six user-contributed categories, 2251 rows: four Python AST categories plus -irrelevance and relevance detection. The headline is upstream's `Live Overall -Acc` -- the SAMPLE-COUNT-WEIGHTED mean of the six, which is algebraically the -pooled rate over all 2251 rows. That is why this group publishes an interval -where the non-live group cannot: a weighted mean of category rates over their -own row counts is the rate over their union, so a problem-clustered interval -brackets the number printed beside it. `ast_summary`, weighted over the four AST -categories, carries one on `n_ast` for the same reason. +irrelevance and relevance detection. Unlike the non-live group this one carries +intervals, because its weighted rollup IS the pooled rate over the union of its +rows -- see `BfclV3LiveTask`. -This is the `(Prompt)` column: the function schemas arrive in the system turn -and the reply is parsed with `ast_parse`. The `_fc` sibling is the `(FC)` -column, where the same schemas arrive as tools. The unqualified name is this one -because every model on the leaderboard has a Prompt row, while an FC row exists -only where the provider exposes a tools API -- and because FC gates on the -`function_tools` capability, which would make the default name the narrow one. +Upstream's `(Prompt)` column: schemas arrive in the system turn and the reply is +parsed with `ast_parse`. The `_fc` sibling is the `(FC)` column, where the same +schemas arrive as tools. Headline, interval policy and the published numbers are +in `reference_impl.notes`; why the unqualified name is this one is in the +variants table. References: @@ -66,13 +61,12 @@ "`ast_summary` (weighted over the four AST categories) carries one " "on `n_ast` for the same reason. underscore_to_dot=False: schemas " "reach the model verbatim, so dotted function names survive and " - "gold needs no rewrite. Replaying upstream's released " - "gpt-4.1-2025-04-14 (Prompt) rollouts reproduces all six published " - "category accuracies exactly and agrees with upstream's own " - "verdict on 2251/2251 rows: live_simple 85.66, live_multiple " - "76.54, live_parallel 93.75, live_parallel_multiple 75.00, " - "live_irrelevance 77.89, live_relevance 88.89 -- so ast_summary " - f"78.46 and Live Overall Acc 78.32. {BFCL_V3_SHARED_NOTES}" + "gold needs no rewrite. Replay of upstream's gpt-4.1-2025-04-14 " + "(Prompt) rollouts agrees on 2251/2251 rows and reproduces every " + "published cell: live_simple 85.66, live_multiple 76.54, " + "live_parallel 93.75, live_parallel_multiple 75.00, " + "live_irrelevance 77.89, live_relevance 88.89; ast_summary 78.46, " + f"Live Overall Acc 78.32. {BFCL_V3_SHARED_NOTES}" ), ), ) diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py index 490fae10..23ab97d4 100644 --- a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py +++ b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py @@ -1,28 +1,16 @@ """BFCL v3 live categories, 0-shot, upstream's function-calling protocol. Six user-contributed categories, 2251 rows: four Python AST categories plus -irrelevance and relevance detection. The headline is upstream's `Live Overall -Acc` -- the SAMPLE-COUNT-WEIGHTED mean of the six, which is algebraically the -pooled rate over all 2251 rows. That is why this group publishes an interval -where the non-live group cannot: a weighted mean of category rates over their -own row counts is the rate over their union, so a problem-clustered interval -brackets the number printed beside it. `ast_summary`, weighted over the four AST -categories, carries one on `n_ast` for the same reason. +irrelevance and relevance detection. Unlike the non-live group this one carries +intervals, because its weighted rollup IS the pooled rate over the union of its +rows -- see `BfclV3LiveTask`. -This is the `(FC)` column: the same schemas reach the model through the -provider's tools API, and the calls are read back off structured tool calls -rather than parsed out of the reply. The sibling without the suffix is the -`(Prompt)` column, and it holds the unqualified name because every model on the -leaderboard has a Prompt row, while an FC row exists only where the provider -exposes a tools API -- and because this task gates on the `function_tools` -capability, which would make the default name the narrow one. - -One scoring divergence follows from the transport and it is upstream's. A native -tool name cannot carry a dot in the OpenAI dialects, so `convert_to_tool` -rewrites `geometry.triangle_area` to `geometry_triangle_area` on the way out; -gold still spells the dot, so the comparison rewrites gold to match. That is -`underscore_to_dot=True`, and it is the only behavioural difference from the -Prompt sibling beyond how the schemas and the calls travel. +Upstream's `(FC)` column: the same schemas reach the model through the provider's +tools API and the calls are read off structured `tool_calls` rather than parsed +out of the reply. Beyond that transport, the only behavioural difference from +the Prompt sibling is `underscore_to_dot=True` -- upstream's own reversal of the +`.`->`_` rewrite a native tool name forces, explained in `_base`. Headline, +interval policy and the published numbers are in `reference_impl.notes`. References: @@ -70,15 +58,14 @@ "way out, because a native tool name cannot carry a dot, so gold is " "rewritten to match before comparison. Requires the " "`function_tools` capability; a binding that does not offer it is " - "rejected when the task is constructed, which prelaunch " - "reconciliation reaches before the first request and before a " - "result directory exists. Replaying upstream's released " - "gpt-4.1-2025-04-14-FC rollouts reproduces all six published " - "category accuracies exactly and agrees with upstream's own " - "verdict on 2251/2251 rows: live_simple 80.23, live_multiple " - "78.35, live_parallel 68.75, live_parallel_multiple 66.67, " - "live_irrelevance 82.31, live_relevance 77.78 -- so ast_summary " - f"78.39 and Live Overall Acc 79.92. {BFCL_V3_SHARED_NOTES}" + "rejected at task construction, which prelaunch reconciliation " + "reaches before the first request and before a result directory " + "exists. Replay of upstream's gpt-4.1-2025-04-14-FC rollouts agrees " + "on 2251/2251 rows and reproduces every published cell: live_simple " + "80.23, live_multiple 78.35, live_parallel 68.75, " + "live_parallel_multiple 66.67, live_irrelevance 82.31, " + "live_relevance 77.78; ast_summary 78.39, Live Overall Acc " + f"79.92. {BFCL_V3_SHARED_NOTES}" ), ), ) diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py index 4b35def8..38993e7a 100644 --- a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py +++ b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py @@ -1,20 +1,13 @@ """BFCL v3 non-live categories, 0-shot, upstream's prompting protocol. Seven hand-written categories, 1390 rows: four Python AST categories, Java and -JavaScript simple-call categories, and irrelevance detection. The headline is -upstream's `Non_Live Overall Acc` -- the UNWEIGHTED mean of five numbers, one of -which (`simple_ast`) is itself the unweighted mean of three. It publishes no -confidence interval for that reason: it is a mean of rates, not the pooled rate -over 1390 rows, and an interval computed over those rows would bracket a -different statistic than the one printed beside it. Every per-category rate does -carry one. +JavaScript simple-call categories, and irrelevance detection. -This is the `(Prompt)` column: the function schemas arrive in the system turn -and the reply is parsed with `ast_parse`. The `_fc` sibling is the `(FC)` -column, where the same schemas arrive as tools. The unqualified name is this one -because every model on the leaderboard has a Prompt row, while an FC row exists -only where the provider exposes a tools API -- and because FC gates on the -`function_tools` capability, which would make the default name the narrow one. +Upstream's `(Prompt)` column: schemas arrive in the system turn and the reply is +parsed with `ast_parse`. The `_fc` sibling is the `(FC)` column, where the same +schemas arrive as tools. Headline, interval policy and the published numbers are +in `reference_impl.notes`; why the unqualified name is this one is in the +variants table. References: @@ -62,19 +55,17 @@ "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. " "Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast " "(itself the unweighted mean of simple/java/javascript), multiple, " - "parallel, parallel_multiple and irrelevance. NO interval is " - "published on the headline, nor on simple_ast or ast_summary: an " - "unweighted mean of category rates is not the pooled rate over the " - "rows (javascript's 50 weigh as much as simple's 400), so a " - "sample-clustered CI would bracket a different statistic. Every " - "per-category rate carries its own. underscore_to_dot=False: " - "schemas reach the model verbatim, so dotted function names survive " - "and gold needs no rewrite. Replaying upstream's released " - "gpt-4.1-2025-04-14 (Prompt) rollouts reproduces all seven " - "published category accuracies exactly and agrees with upstream's " - "own verdict on 1390/1390 rows: simple 95.50, multiple 94.00, " - "parallel 93.00, parallel_multiple 87.50, java 64.00, javascript " - "82.00, irrelevance 88.75 -- so simple_ast 80.50 and Non_Live " + "parallel, parallel_multiple and irrelevance. No interval on the " + "headline, simple_ast or ast_summary: an unweighted mean of category " + "rates is not the pooled rate (javascript's 50 rows weigh as much as " + "simple's 400), so a sample-clustered CI would bracket a different " + "statistic. Every per-category rate carries its own. " + "underscore_to_dot=False: schemas reach the model verbatim, so " + "dotted function names survive and gold needs no rewrite. Replay of " + "upstream's gpt-4.1-2025-04-14 (Prompt) rollouts agrees on 1390/1390 " + "rows and reproduces every published cell: simple 95.50, multiple " + "94.00, parallel 93.00, parallel_multiple 87.50, java 64.00, " + "javascript 82.00, irrelevance 88.75; simple_ast 80.50, Non_Live " f"Overall Acc 88.75. {BFCL_V3_SHARED_NOTES}" ), ), diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py index c7952936..8f964f05 100644 --- a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py +++ b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py @@ -1,28 +1,14 @@ """BFCL v3 non-live categories, 0-shot, upstream's function-calling protocol. Seven hand-written categories, 1390 rows: four Python AST categories, Java and -JavaScript simple-call categories, and irrelevance detection. The headline is -upstream's `Non_Live Overall Acc` -- the UNWEIGHTED mean of five numbers, one of -which (`simple_ast`) is itself the unweighted mean of three. It publishes no -confidence interval for that reason: it is a mean of rates, not the pooled rate -over 1390 rows, and an interval computed over those rows would bracket a -different statistic than the one printed beside it. Every per-category rate does -carry one. +JavaScript simple-call categories, and irrelevance detection. -This is the `(FC)` column: the same schemas reach the model through the -provider's tools API, and the calls are read back off structured tool calls -rather than parsed out of the reply. The sibling without the suffix is the -`(Prompt)` column, and it holds the unqualified name because every model on the -leaderboard has a Prompt row, while an FC row exists only where the provider -exposes a tools API -- and because this task gates on the `function_tools` -capability, which would make the default name the narrow one. - -One scoring divergence follows from the transport and it is upstream's. A native -tool name cannot carry a dot in the OpenAI dialects, so `convert_to_tool` -rewrites `geometry.triangle_area` to `geometry_triangle_area` on the way out; -gold still spells the dot, so the comparison rewrites gold to match. That is -`underscore_to_dot=True`, and it is the only behavioural difference from the -Prompt sibling beyond how the schemas and the calls travel. +Upstream's `(FC)` column: the same schemas reach the model through the provider's +tools API and the calls are read off structured `tool_calls` rather than parsed +out of the reply. Beyond that transport, the only behavioural difference from +the Prompt sibling is `underscore_to_dot=True` -- upstream's own reversal of the +`.`->`_` rewrite a native tool name forces, explained in `_base`. Headline, +interval policy and the published numbers are in `reference_impl.notes`. References: @@ -63,24 +49,22 @@ "FC protocol -- upstream's `(FC)` leaderboard column. Headline is " "`Non_Live Overall Acc`: unweighted mean of simple_ast (itself the " "unweighted mean of simple/java/javascript), multiple, parallel, " - "parallel_multiple and irrelevance. NO interval is published on the " - "headline, nor on simple_ast or ast_summary: an unweighted mean of " - "category rates is not the pooled rate over the rows (javascript's " - "50 weigh as much as simple's 400), so a sample-clustered CI would " - "bracket a different statistic. Every per-category rate carries its " - "own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in " - "every function name on the way out, because a native tool name " - "cannot carry a dot, so gold is rewritten to match before " - "comparison. Requires the `function_tools` capability; a binding " - "that does not offer it is rejected when the task is constructed, " - "which prelaunch reconciliation reaches before the first request " - "and before a result directory exists. Replaying upstream's " - "released gpt-4.1-2025-04-14-FC rollouts reproduces all seven " - "published category accuracies exactly and agrees with upstream's " - "own verdict on 1390/1390 rows: simple 93.50, multiple 90.50, " + "parallel_multiple and irrelevance. No interval on the headline, " + "simple_ast or ast_summary: an unweighted mean of category rates is " + "not the pooled rate (javascript's 50 rows weigh as much as simple's " + "400), so a sample-clustered CI would bracket a different statistic. " + "Every per-category rate carries its own. underscore_to_dot=True: " + "convert_to_tool rewrote `.` to `_` in every function name on the " + "way out, because a native tool name cannot carry a dot, so gold is " + "rewritten to match before comparison. Requires the `function_tools` " + "capability; a binding that does not offer it is rejected at task " + "construction, which prelaunch reconciliation reaches before the " + "first request and before a result directory exists. Replay of " + "upstream's gpt-4.1-2025-04-14-FC rollouts agrees on 1390/1390 rows " + "and reproduces every published cell: simple 93.50, multiple 90.50, " "parallel 91.00, parallel_multiple 86.00, java 61.00, javascript " - "68.00, irrelevance 89.58 -- so simple_ast 74.17 and Non_Live " - f"Overall Acc 86.25. {BFCL_V3_SHARED_NOTES}" + "68.00, irrelevance 89.58; simple_ast 74.17, Non_Live Overall Acc " + f"86.25. {BFCL_V3_SHARED_NOTES}" ), ), ) diff --git a/tests/acceptance/bfcl_v3/test_upstream_rollout_replay.py b/tests/acceptance/bfcl_v3/test_upstream_rollout_replay.py index e7e867ae..6929c459 100644 --- a/tests/acceptance/bfcl_v3/test_upstream_rollout_replay.py +++ b/tests/acceptance/bfcl_v3/test_upstream_rollout_replay.py @@ -1,26 +1,26 @@ """Anchor BFCL v3 grading on upstream's own released rollouts and verdicts. -BFCL publishes an evaluation archive beside the leaderboard -- per model, per -protocol, per category, it releases both the model's recorded replies and its -own per-row verdicts. That is per-prediction ground truth, so the port can be -pinned exactly, with no model spend. - -This is strictly stronger than `test_gold_replay.py`, which feeds gold back as -the reply. Gold replay can only ever exercise the *correct* path: it never sees -a wrong answer, a decode failure, or -- because it drives the checker directly --- either protocol's decoder. This module replays a real model's output, of -which 9% is wrong, through the production `_decode` of both mixins. `java` at -64% is the point: a third of those rows are graded wrong by upstream, and we -have to agree about which third. - -What it does NOT cover is prompt construction: the replies are upstream's, so -nothing here proves our prompt is upstream's prompt. That rests on -`system_prompt_pre_processing_chat_model` and -`func_doc_language_specific_pre_processing` being byte-identical to upstream -(pinned by `tests/unit/community/bfcl_v3/test_identity.py`) and running on a -row-for-row verified snapshot -- a construction argument, not a measurement. - -Staging (opt-in, like the gold replay -- ~6MB, so it is not vendored): +BFCL publishes an evaluation archive beside the leaderboard: per model, per +protocol, per category, both the model's recorded replies and its own per-row +verdicts. That is per-prediction ground truth, so the port pins exactly with no +model spend. + +Stronger than `test_gold_replay.py`, which feeds gold back and so only ever +exercises the *correct* path -- never a wrong answer, a decode failure, or +either protocol's decoder. This replays real output, 9% of it wrong, through the +production `_decode` of both mixins. `java` at 64% is the point: a third of +those rows are wrong and we have to agree about which third. + +It does NOT cover prompt construction -- the replies are upstream's. That rests +on the two model-facing helpers being byte-identical to upstream (pinned by +`test_identity.py`) and running on a row-for-row verified snapshot: a +construction argument, not a measurement. + +No file hashes are pinned, and they would be redundant: a swapped archive fails +the :data:`PUBLISHED` header assertion, and a doctored `result` file fails the +comparison it feeds. + +Staging is opt-in, like the gold replay -- ~6MB, so not vendored: R=https://raw.githubusercontent.com/HuanzhiMao/BFCL-Result/main/2025-06-14 D="$SIEVAL_DATA_DIR/HuanzhiMao/BFCL-Result/2025-06-14" @@ -36,10 +36,6 @@ done done -No file hashes are pinned, because they would be redundant: every score file's -header is asserted against :data:`PUBLISHED` below, so a swapped archive fails; -and a doctored `result` file fails the row-for-row comparison it feeds. - AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) """ @@ -145,14 +141,11 @@ def _jsonl(path: pathlib.Path) -> list[dict]: def _model_output(protocol: str, recorded) -> ModelOutput: """Upstream's recorded reply, in the shape the mixin's `_decode` reads. - Built as a real `ModelOutput` so the production decoders run on the type - they run on in a live eval, rather than on a stand-in that happens to have - the two attributes they touch. - - The FC archive stores each call as `{name: arguments-JSON-string}`, which is - exactly what the OpenAI dialects put on `FunctionToolCall.arguments`, so the - calls are rebuilt rather than pre-parsed -- `_call_arguments` is then part of - what this anchors. + A real `ModelOutput`, not a stand-in with the two attributes `_decode` + touches. The FC archive stores each call as `{name: arguments-JSON-string}`, + which is what the OpenAI dialects put on `FunctionToolCall.arguments`, so + the calls are rebuilt rather than pre-parsed and `_call_arguments` is part + of what this anchors. """ meta = GenModel(model="archive-replay", api_key="none").meta() if protocol == "prompt": @@ -176,14 +169,12 @@ def _model_output(protocol: str, recorded) -> ModelOutput: def _decode_or_fail(decoder, protocol: str, recorded, language): """The mixin's production `_decode`, under `postprocess`'s failure semantic. - `postprocess` wraps the call in `except Exception -> [None]`, because an - undecodable reply is upstream's `ast_decoder:decoder_failed` -- a SCORE, not - a fault -- and that boundary has to be mirrored here or a real reply takes - the test out. It is reached often on this data: a model that declines in - prose ("None of the functions can be used, as ...") raises `SyntaxError` - straight out of `ast.parse`. Only the two lines of that semantic are - mirrored; the decoder itself is the shipped one, and `postprocess`'s - narrower re-raise of `ImportError` is pinned in the unit tests. + `postprocess` wraps the call in `except Exception -> [None]`: an undecodable + reply is upstream's `ast_decoder:decoder_failed`, a SCORE rather than a + fault. Mirrored here or a real reply takes the test out -- a model that + declines in prose raises `SyntaxError` straight out of `ast.parse`. Only + those two lines are mirrored; the decoder is the shipped one, and + `postprocess`'s narrower `ImportError` re-raise is pinned in the unit tests. """ try: return decoder._decode(_model_output(protocol, recorded), language) @@ -279,10 +270,9 @@ def test_replaying_upstream_rollouts_reproduces_every_published_cell( def test_the_live_rollup_is_the_pooled_rate_over_every_live_row(protocol: str): """The weighted rollup's own claim, checked rather than asserted in prose. - `BfclV3LiveTask` documents its sample-count-weighted mean as algebraically - the pooled rate over the union of the six categories' rows. If that holds, - the headline is reconstructible from the published counts alone -- which is - what makes it a genuine per-sample rate and so interval-bearing. + `BfclV3LiveTask` documents its weighted mean as algebraically the pooled + rate over the union of the six categories' rows -- which is what makes the + headline a genuine per-sample rate, and so interval-bearing. """ model_dir, _ = PROTOCOLS[protocol] cells = {c: PUBLISHED[model_dir][c] for c in LIVE} @@ -296,9 +286,8 @@ def cell(category: str) -> dict: "display_accuracy": accuracy, } - # Built without `__init__`, which would demand a live model binding: the - # rollup reads only ClassVars. This runs the PRODUCTION aggregation rather - # than a second copy of its arithmetic. + # Built without `__init__` (which would demand a live model binding); the + # rollup reads only ClassVars. Runs the PRODUCTION aggregation. task = object.__new__(BfclV3LiveTask) rollup = task._aggregate(cell) # noqa: SLF001 -- tests are the carve-out pooled = ( diff --git a/tests/unit/community/bfcl_v3/test_identity.py b/tests/unit/community/bfcl_v3/test_identity.py index e6b87445..6365eaed 100644 --- a/tests/unit/community/bfcl_v3/test_identity.py +++ b/tests/unit/community/bfcl_v3/test_identity.py @@ -1,25 +1,19 @@ """Pin the vendored BFCL v3 files, and keep their provenance resolvable. -A sha256 taken over our own copy pins whatever was copied -- it detects drift -after vendoring and proves nothing about what was vendored. The non-circular -anchor is the `Upstream blob:` line in each vendored module's docstring: a git -blob id, resolvable in upstream's repository by anyone, independent of this -checkout. All twelve were resolved against `ShishirPatil/gorilla` at -`ea13468e`, and all twelve matched. - -So the two tests here divide the work deliberately: - -* :func:`test_vendored_file_has_not_drifted` is the drift detector. Its digests - are read off the copy, on purpose -- that is the only thing they can be. -* :func:`test_vendored_file_records_resolvable_provenance` keeps the real anchor - present. It cannot resolve the blob (that needs upstream's repository, which - no test may reach for), so it checks the claim is *recorded* in a shape a - reader can resolve. Without it, a re-vendoring that drops the docstring leaves - the first test passing over bytes with no stated origin. - -`Upstream path:` is relative to `berkeley-function-call-leaderboard/`, the -package root inside upstream's monorepo -- not to the repository root, where -none of these paths exist. +A sha256 over our own copy pins whatever was copied: it detects drift after +vendoring and proves nothing about what was vendored. The non-circular anchor is +each module's `Upstream blob:` line -- a git blob id anyone can resolve in +upstream's repository. All twelve were resolved against `ShishirPatil/gorilla` +at `ea13468e` and matched. + +Hence two tests: one detects drift (its digests can only be read off the copy), +the other keeps the real anchor *present*. The second cannot resolve the blob -- +no test may reach for upstream's repository -- so it checks the claim is +recorded in a resolvable shape; without it, a re-vendoring that drops the +docstring leaves the first passing over bytes with no stated origin. + +`Upstream path:` is relative to `berkeley-function-call-leaderboard/`, not to +the repository root, where none of these paths exist. AI-Generated Code - Claude Opus 5 (Anthropic) """ @@ -45,20 +39,20 @@ "cebc6f9a1b28e277483dd13b2a3fd3ceb95ad8a4631daf50933ad1af3015c698" ), "type_convertor/java_type_converter.py": ( - "4b450f96bfe528823de681e9d6b0b7fb748ca51a534b2d7a59d76d1047dfebf2" + "41082275fda22106130db71d65902786884ca58379814124291a1274ae40451b" ), "type_convertor/js_type_converter.py": ( - "ef9562e296bcd7aeeff8b2b88f0fba1c996951b91b229fc91a03bf0f855fcfe2" + "32ec360c519e3813c2159993ce78e290152b1a79abdfa3e35f03c6ed3af24202" ), "source_parser/java_parser.py": ( - "024dfc247dbb7890c4ed969ab48a9fab3212b85bdb74402528a5ce7b8abab5fe" + "e31322b491601ab0e5ca3a3ae5a227f9478ed93783f740813d51228ef20a999b" ), "source_parser/js_parser.py": ( - "3a096d914cc54514009e11915e8d4cc4c35af26b5a1c9f21ced640cd896dc737" + "abf44b3f6d38b1cfb14d57b005e0ea7d133543a393bfd54c0056c7c74482e23a" ), - "parser.py": ("bed1fd6c50b4164dd6f4277ef54b1eae908491f5476e7830f742f614fe7266f9"), + "parser.py": ("e5655d4e60f40388f7b061cd551d29769c9f1cab1e67d7c1699a556531d8d17e"), "tool_convert.py": ( - "11b758572a09fca738479243e4cc3005387def672786b3231c868e6d73ad386e" + "f6474e2a5348e521b16e51fc5a3f669aef85343a1307a723715209696c407804" ), "output_checks.py": ( "683a43d71e3e55d52b5edc7afd94def3ce449261642c3dbde543ae82ceab5268"