diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 965e2a69..43e93c64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,9 +27,13 @@ env: # is tested at all. nltk alone is not enough -- the pre-2020 grader reaches # only nltk, so without sqlparse the test-suite metric and the hardened # execution under it are silently uncollected here while passing locally. + # `bfcl-v3` is installed rather than left out like `hle`/`multi-if`: it gates + # the java/javascript decode path, the one thing the vendored source-text + # parsers exist to make work, and an untested path silently rots. The cost is + # one small C extension (tree-sitter) built from sdist per run. INSTALL_GROUPS: >- - -G drop -G ifbench -G ifeval -G iheval -G math -G scicode -G spider -G test - -G dev + -G bfcl-v3 -G drop -G ifbench -G ifeval -G iheval -G math -G scicode -G spider + -G test -G dev jobs: changes: diff --git a/pdm.lock b/pdm.lock index dd695add..6473a983 100644 --- a/pdm.lock +++ b/pdm.lock @@ -2,10 +2,10 @@ # It is not intended for manual editing. [metadata] -groups = ["default", "dev", "drop", "hle", "ifbench", "ifeval", "iheval", "math", "multi-if", "nl2sh-alfa", "ruler", "ruler-gen", "scicode", "spider", "spider2", "t-eval", "test", "wikisql"] +groups = ["default", "bfcl-v3", "dev", "drop", "hle", "ifbench", "ifeval", "iheval", "math", "multi-if", "nl2sh-alfa", "ruler", "ruler-gen", "scicode", "spider", "spider2", "t-eval", "test", "wikisql"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:2733395ff19bb311187e466cf75746088e4f331191d869ff9dedcbe5fc05bc33" +content_hash = "sha256:762c06f720accabcee892c676f5a69a3273b7fd9b991f94e53d28a481e4e2da4" [[metadata.targets]] requires_python = ">=3.12,<3.15" @@ -3748,6 +3748,56 @@ files = [ {file = "transformers-4.57.3.tar.gz", hash = "sha256:df4945029aaddd7c09eec5cad851f30662f8bd1746721b34cc031d70c65afebc"}, ] +[[package]] +name = "tree-sitter" +version = "0.21.3" +requires_python = ">=3.8" +summary = "Python bindings for the Tree-Sitter parsing library" +groups = ["bfcl-v3"] +files = [ + {file = "tree-sitter-0.21.3.tar.gz", hash = "sha256:b5de3028921522365aa864d95b3c41926e0ba6a85ee5bd000e10dc49b0766988"}, + {file = "tree_sitter-0.21.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:669b3e5a52cb1e37d60c7b16cc2221c76520445bb4f12dd17fd7220217f5abf3"}, + {file = "tree_sitter-0.21.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2aa2a5099a9f667730ff26d57533cc893d766667f4d8a9877e76a9e74f48f0d3"}, + {file = "tree_sitter-0.21.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a3e06ae2a517cf6f1abb682974f76fa760298e6d5a3ecf2cf140c70f898adf0"}, + {file = "tree_sitter-0.21.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af992dfe08b4fefcfcdb40548d0d26d5d2e0a0f2d833487372f3728cd0772b48"}, + {file = "tree_sitter-0.21.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c7cbab1dd9765138505c4a55e2aa857575bac4f1f8a8b0457744a4fefa1288e6"}, + {file = "tree_sitter-0.21.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e1e66aeb457d1529370fcb0997ae5584c6879e0e662f1b11b2f295ea57e22f54"}, + {file = "tree_sitter-0.21.3-cp312-cp312-win_amd64.whl", hash = "sha256:013c750252dc3bd0e069d82e9658de35ed50eecf31c6586d0de7f942546824c5"}, +] + +[[package]] +name = "tree-sitter-java" +version = "0.21.0" +requires_python = ">=3.8" +summary = "Java grammar for tree-sitter" +groups = ["bfcl-v3"] +files = [ + {file = "tree_sitter_java-0.21.0-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:d30fb39af6590f6be877ff8d39c7e193f37594375b8fcdda55babd264411616b"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:696391968cdc467f97e525fbe43790c2d5ba46bf446376eb30d5c20b5c9d795a"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b82318982531a473c598601d73956b0b1f9e608d39e427ba8c1a59f2d6da262"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9c01f48ec3a3d4b55fb2c56b7cad6ebf94dba568d690b45d966731859485dd4"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:d83ca51b26826550051c53468ec74f5b570b4afb4d4520df0aa2a35b52de2289"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:8afb56c90048272b079235b82fc2a4a69b058b4ace206d9475cbad2eb143f40a"}, + {file = "tree_sitter_java-0.21.0-cp38-abi3-win_amd64.whl", hash = "sha256:6534fac27a93160a2b27a0c77f22a7c91f0aee2dba9d4cdbf835bf509ddfbf4f"}, +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.21.4" +requires_python = ">=3.8" +summary = "Javascript grammar for tree-sitter" +groups = ["bfcl-v3"] +files = [ + {file = "tree-sitter-javascript-0.21.4.tar.gz", hash = "sha256:5ae97db218c22f16f1fd1108d77c4017b453addc36041136779c99800f23ff20"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:e0f1d26ec85cc66d56fb25fd41ce499dc5c4a60b478a91754684e2289433daae"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e9172dc9e59649d7598c6509103db1648e7187c2bcb39c34222fc052b9db0ac4"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d45bd39ce931b164f84cfb6a186df504e38c6f91f541e2e60d5a40eb9d574005"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00ff426e9bb552abb49ce3234b083edb1a6aa73bfeea7d9bf0f6bb3d0256c4cd"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c2945bc49985ab396773e2738ae9b1e7d06950d6ad70c535c010fb9b2816a855"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5fc87d07f4c6a7067d4e0f004956a549c05e21eeced3287f3ea2bdd04a71a97a"}, + {file = "tree_sitter_javascript-0.21.4-cp38-abi3-win_amd64.whl", hash = "sha256:2843b0e81564d8176922ef2b40db128a4de026606d6b1d22a06efb8e8a5c01b8"}, +] + [[package]] name = "triton" version = "3.5.1" diff --git a/pyproject.toml b/pyproject.toml index 2a2995eb..0cb8cdfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,21 @@ Issues = "https://github.com/scitix/sieval/issues" Changelog = "https://github.com/scitix/sieval/blob/main/CHANGELOG.md" [project.optional-dependencies] +bfcl-v3 = [ + # Reached only by the Prompt protocol, and there by every category rather + # than just `java` and `javascript`: upstream's `ast_parse` dispatches those + # two to tree-sitter parsers rather than Python's `ast`, but it imports both + # source parsers at module scope, so touching `ast_parse` at all requires + # them. The FC protocol never parses source text and does not need this. + # + # Pinned to upstream's own versions. tree_sitter 0.21.3 predates the 0.22 API + # break -- the vendored parsers call `Language(ptr, name)` and + # `Parser.set_language()`, both removed later -- so this is a hard pin, not a + # floor. It has no wheel for Python 3.13 and builds from sdist. + "tree-sitter==0.21.3", + "tree-sitter-java==0.21.0", + "tree-sitter-javascript==0.21.4", +] drop = ["numpy<=2.2", "scipy>=1.16.3"] # numpy backs the vendored HLE `calib_err` (calibration error). No grader/judge # extras: the judge is reached through the base `openai` client like any model. diff --git a/scripts/check_layer_imports.py b/scripts/check_layer_imports.py index 26d2d836..9d38a123 100644 --- a/scripts/check_layer_imports.py +++ b/scripts/check_layer_imports.py @@ -456,6 +456,15 @@ def _check_relative_scope(path: Path, tree: ast.AST) -> list[str]: Scoped to the sieval package only: ``scripts/`` files are standalone modules, not a package, so a relative import there fails at runtime and needs no lint. (The pre-commit hook feeds both trees; this check narrows on purpose.) + + ``sieval/community/`` is **not** exempt, and a vendored drop is not a reason + to make it one. A vendored file that reaches across packages has already had + that import rewritten — upstream's own was absolute (``from + bfcl_eval.constants...``), and a copy cannot keep it — so spelling the + replacement absolutely costs exactly what spelling it ``..x`` costs: one + line, already being edited. Exempting the tree to permit the relative form + would buy nothing and would drop this rule over every future drop, which is + a ratchet: lint exemptions are not narrowed back. """ if _get_layer(path) is None: return [] diff --git a/scripts/check_preflight.py b/scripts/check_preflight.py index 8018e27c..47ef77fd 100644 --- a/scripts/check_preflight.py +++ b/scripts/check_preflight.py @@ -1164,6 +1164,35 @@ def lock_drift( return report +# Python's own warning output, which lands on the same stderr the import +# checker writes violations to: `:: : `. +_PY_WARNING_LINE = re.compile(r":\d+: \w*Warning: ") + + +def _import_violation_lines(stderr: str) -> list[str]: + """Return only the lines of *stderr* that are import-policy violations. + + ``check_layer_imports.py`` parses every file it is handed, and several + vendored modules raise ``SyntaxWarning`` on parse. Those warnings land on + stderr on **every** run, passing or failing, so counting raw stderr lines + made two real violations read as twenty-six and turned a two-line + regression into an apparently unfixable pile. + + A warning occupies two lines — the ``:: :`` header + and an indented echo of the offending source. Violations are always printed + flush-left, so dropping indented lines is safe and also handles a + multi-line warning body. + """ + violations: list[str] = [] + for line in stderr.strip().splitlines(): + if not line.strip(): + continue + if line[:1].isspace() or _PY_WARNING_LINE.search(line): + continue + violations.append(line) + return violations + + class PreflightRunner: """Orchestrates preflight checks.""" @@ -3047,13 +3076,18 @@ def check_imports(self) -> list[CheckResult]: # # KNOWN divergence, not parity: pre-commit additionally applies the # global `exclude: ^(sieval/community/|vendor/)`, so it skips - # `sieval/community/` while this wrapper checks it. Inert today (every - # relative import under `community/` is a bare level-1 `from . import - # x`), but a future vendored drop using `from ..x import y` would pass - # pre-commit and fail preflight, and the only offered fix would be to - # edit code kept byte-identical to upstream. Fixing it is a design call - # — hoisting the exemption into `_check_file` would also drop the - # private-access check's coverage of `community/`. + # `sieval/community/` for every hook while this wrapper checks it. That + # makes this wrapper the only enforcement `community/` gets, which is + # the reason to keep feeding it the tree rather than to carve it out. + # + # The vendored drop this comment used to predict has arrived — a file + # reaching across packages, where upstream's own import was absolute and + # the copy could not keep it. It is spelled absolutely, so the rule is + # satisfied without an exemption: rewriting that import was unavoidable + # either way, and the relative spelling was the only version that would + # have needed one. Do not add the carve-out here, in `_check_file`, or + # in `_check_relative_scope` — it would silently drop private-module + # protection over a tree nothing else checks. enforced_py = [ f for f in self._git_tracked_files(".py") @@ -3072,7 +3106,12 @@ def check_imports(self) -> list[CheckResult]: if result.returncode == 0: return [CheckResult("PASS", "check_imports", "no import-policy violations")] - errors = [line for line in result.stderr.strip().splitlines() if line.strip()] + errors = _import_violation_lines(result.stderr) + if not errors: + # Non-zero exit with nothing recognisable as a violation means the + # checker's output shape moved. Show the raw stderr rather than + # reporting a reassuring "0 violation(s)" on a failing run. + errors = [ln for ln in result.stderr.strip().splitlines() if ln.strip()] return [ CheckResult( "FAIL", diff --git a/sieval/community/CLAUDE.md b/sieval/community/CLAUDE.md index 7137ec37..f9044bcf 100644 --- a/sieval/community/CLAUDE.md +++ b/sieval/community/CLAUDE.md @@ -26,7 +26,19 @@ inline where it would swamp a diff against upstream. Do not add more original code here without the same argument; a helper with one caller belongs in that caller's module. +`bfcl_v3/_safe_eval.py` is the second, and with exactly **one** caller it shows +what "the same argument" means. That caller is a *vendored* file +(`bfcl_v3/parser.py`, whose `ast.BinOp` branch upstream resolves with `eval` over +model output), so inlining the guard would both enlarge a diff that exists to be +compared against upstream and put a security boundary inside a file that cannot +be linted. "One caller" is the rule for ordinary helpers; a boundary that has to +stay separately auditable is not one. + The package-wide `ruff` / `mypy` / `pre-commit` exclusions exist to keep vendored -code byte-identical and cover this file too, which is the wrong default for a -security boundary. Until they are narrowed, lint it by hand: -`ruff check --config 'exclude=["vendor"]' sieval/community/_sympy_guards.py`. +code byte-identical and cover both files too, which is the wrong default for a +security boundary. Until they are narrowed, lint them by hand: + +``` +ruff check --config 'exclude=["vendor"]' sieval/community/_sympy_guards.py +ruff check --config 'exclude=["vendor"]' sieval/community/bfcl_v3/_safe_eval.py +``` diff --git a/sieval/community/bfcl_v3/__init__.py b/sieval/community/bfcl_v3/__init__.py new file mode 100644 index 00000000..07726c7d --- /dev/null +++ b/sieval/community/bfcl_v3/__init__.py @@ -0,0 +1,90 @@ +"""BFCL v3 (single-turn) grader, vendored from ShishirPatil/gorilla. + +Source: https://github.com/ShishirPatil/gorilla +Revision: v1.3 (ea13468e4423454d0c213704fb87cf7cb3990433) +License: Apache-2.0 + +Upstream's evaluation surface is `bfcl_eval`, which imports its own submodules +by absolute path. A vendored copy cannot keep those imports, so rewriting them +is the only edit made to any vendored file -- with one exception: `parser.py`'s +`ast.BinOp` branch calls `_safe_eval.safe_eval` where upstream `eval`s a +model-authored expression. Each vendored module records upstream's blob SHA at +the pin and its exact edits, so the copy can be re-derived from a fresh checkout. + +Whole-file: `ast_checker.py`, `type_mappings.py`, `type_convertor/*.py`, +`source_parser/*.py`, and `prompts.py` (upstream's `constants/default_prompts`). +Partial -- named subsets of a larger upstream file, listed in each module's own +docstring: `parser.py`, `tool_convert.py`, `output_checks.py`, `preprocess.py`, +`aggregate.py`. + +First-party: `_model_config.py` supplies the `MODEL_CONFIG_MAPPING` global +`ast_checker.convert_func_name` reads, since upstream's 2044-line registry has no +entry for sieval's models. `_tables.py` holds the category tables shared by the +dataset loader and the tasks, neither of which may reach into the other's private +module. `_safe_eval.py` is the deviation `parser.py` carries, held outside that +file so the boundary stays lintable -- vendored code is not. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import TYPE_CHECKING + +from ._model_config import set_underscore_to_dot +from ._tables import ( + CALL_EXPECTED, + GOLDLESS_CATEGORIES, + LANGUAGE_BY_CATEGORY, + LIVE_COUNTS, + NON_LIVE_COUNTS, +) +from .aggregate import calculate_unweighted_accuracy, calculate_weighted_accuracy +from .ast_checker import ast_checker +from .output_checks import is_empty_output, is_function_calling_format_output +from .preprocess import ( + func_doc_language_specific_pre_processing, + system_prompt_pre_processing_chat_model, +) +from .prompts import DEFAULT_SYSTEM_PROMPT +from .tool_convert import convert_to_tool + +if TYPE_CHECKING: + # A module `__getattr__` types *every* attribute of the package as its own + # return type, so `ast_parse` would resolve to `object` and every call site + # would read as calling a non-callable. An explicitly declared symbol takes + # precedence over the fallback; `TYPE_CHECKING` is False at runtime, so the + # import below stays deferred. + from .parser import ast_parse + + +__all__ = [ + "ast_checker", + "ast_parse", + "convert_to_tool", + "is_function_calling_format_output", + "is_empty_output", + "func_doc_language_specific_pre_processing", + "system_prompt_pre_processing_chat_model", + "calculate_weighted_accuracy", + "calculate_unweighted_accuracy", + "DEFAULT_SYSTEM_PROMPT", + "set_underscore_to_dot", + "NON_LIVE_COUNTS", + "LIVE_COUNTS", + "GOLDLESS_CATEGORIES", + "CALL_EXPECTED", + "LANGUAGE_BY_CATEGORY", +] + + +def __getattr__(name: str) -> object: + """`ast_parse` is deferred because `parser.py` is the only module here that + reaches tree-sitter, which lives in the optional `bfcl-v3` extra. Importing + it eagerly makes that extra a hard requirement for importing this package at + all -- and `sieval/datasets/` imports it for two pure-data tables, so the + whole dataset registry died on a base install. + """ + if name == "ast_parse": + from .parser import ast_parse + + return ast_parse + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/sieval/community/bfcl_v3/_model_config.py b/sieval/community/bfcl_v3/_model_config.py new file mode 100644 index 00000000..35ba731f --- /dev/null +++ b/sieval/community/bfcl_v3/_model_config.py @@ -0,0 +1,48 @@ +"""The `MODEL_CONFIG_MAPPING` global upstream's `ast_checker` expects. + +`convert_func_name` reads `MODEL_CONFIG_MAPPING[model].underscore_to_dot` to +decide whether a gold function name's dots must be rewritten to underscores +before comparison. Upstream carries a 2044-line registry of its own evaluated +models to answer that one question. sieval's models are not in it, so a verbatim +copy would `KeyError`. + +Rather than delete the call (removing a helper is as much a deviation as adding +one), supply the global the file expects. The flag is a property of the +*transport*, not of a model: it is True exactly when the function schemas +travelled through an API whose tool names cannot contain `.`. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class _ModelEntry: + underscore_to_dot: bool + + +class _ModelConfigMapping: + """Answers for any model name, because the flag does not depend on one.""" + + def __init__(self) -> None: + self._underscore_to_dot = False + + def set_underscore_to_dot(self, enabled: bool) -> None: + self._underscore_to_dot = enabled + + def __getitem__(self, _model_name: str) -> _ModelEntry: + return _ModelEntry(underscore_to_dot=self._underscore_to_dot) + + +MODEL_CONFIG_MAPPING = _ModelConfigMapping() + + +def set_underscore_to_dot(enabled: bool) -> None: + """Set the flag for the protocol about to be graded. + + Prompt protocol → False: schemas reach the model verbatim, dots survive. + FC protocol → True: `convert_to_tool` rewrote `.`→`_` on the way out, so + gold must be rewritten to match. + """ + MODEL_CONFIG_MAPPING.set_underscore_to_dot(enabled) diff --git a/sieval/community/bfcl_v3/_safe_eval.py b/sieval/community/bfcl_v3/_safe_eval.py new file mode 100644 index 00000000..7aba7a02 --- /dev/null +++ b/sieval/community/bfcl_v3/_safe_eval.py @@ -0,0 +1,181 @@ +"""Evaluate a literal arithmetic expression without executing anything. + +`resolve_ast_by_type` resolves each keyword argument of a decoded function call +into a Python value, and for `ast.BinOp` upstream does it with +``eval(ast.unparse(value))``. The node comes from a model's reply, so that line +runs model-authored code: ``f(x=__import__('os').system('...') + 0)`` parses to +a `BinOp`, `eval` runs the call, and the decoder hands back an ordinary-looking +number with nothing else in the run looking unusual. + +:func:`safe_eval` replaces that one call. It walks the node instead of unparsing +it, so there is no string and no namespace to escape from, and admits only +literals, their containers, and the arithmetic/bitwise operators; a `Name`, +`Call`, `Attribute`, `Subscript` or comprehension is refused. That is the entire +divergence: every expression upstream's `eval` computes *without* executing +something is computed here too, to the same value. + +Three shapes are refused that do not execute: + +* An f-string, even with no placeholders -- where `ast.literal_eval` draws the + line, and the alternative is a `FormattedValue` walker for a shape no argument + has been observed to carry. +* An expression whose *cost* is unbounded (:data:`MAX_RESULT_SIZE`). Decoding + runs inline on the event loop with no timeout -- the grade timeout is a stage + later -- so ``f(x=9**9**9)`` would stall the whole run, not one sample. +* ``%`` on a str/bytes left operand. Same class, but the one member that cannot + be *screened*: a printf precision field sets the result size independently of + both operands (``"%.400000000f" % 1.0`` is 400MB from two tiny inputs), so + there is nothing to measure and :func:`_bounded` would see it only after the + allocation. Integer ``%`` is bounded by its right operand and stays allowed. + +Refusal raises `ValueError`, which reaches the caller as a decode failure and +scores the sample wrong -- the right outcome, since a reply putting executable +code in an argument slot has not produced the call it was asked for. + +A restriction on what is evaluated, not a sandbox: it holds because the accepted +node types cannot name anything outside the expression, and would stop holding +the moment one that can is added. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import ast +import operator + +#: Largest intermediate any step may produce -- bits for integers, items for +#: anything sized. Needed because the operators are eager: `9 ** 9 ** 9` asks +#: for a 369-million-digit integer and does not return. 2**20 admits a +#: million-bit integer and a million-character string, against a largest gold +#: literal of 13 digits (~41 bits) across all 2351 gradeable Python rows in the +#: pinned snapshot -- none of which reaches this module at all, since no gold +#: argument is written as an expression. +MAX_RESULT_SIZE = 1 << 20 + +_BIN_OPS = { + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.Div: operator.truediv, + ast.FloorDiv: operator.floordiv, + ast.Mod: operator.mod, + ast.Pow: operator.pow, + ast.LShift: operator.lshift, + ast.RShift: operator.rshift, + ast.BitOr: operator.or_, + ast.BitXor: operator.xor, + ast.BitAnd: operator.and_, +} + +_UNARY_OPS = { + ast.UAdd: operator.pos, + ast.USub: operator.neg, + ast.Invert: operator.invert, + ast.Not: operator.not_, +} + + +def _size(value) -> int: + """Footprint of an intermediate: bits for an integer, items for anything sized. + + `bool` is excluded from the integer arm on purpose -- it is an `int` + subclass whose `bit_length` says nothing useful, and it is never the thing + that grows. + """ + if isinstance(value, int) and not isinstance(value, bool): + return value.bit_length() + try: + return len(value) + except TypeError: + return 0 + + +def _refuse_if_unbounded(op, left, right) -> None: + """Screen the operators whose cost their operands' size does not bound. + + Checked *before* the operator runs, the only time it can be: the cost of + `9 ** 9 ** 9` cannot be measured after the fact. Addition and the bitwise + operators need no entry -- their result is at most the sum of their inputs, + so :func:`_bounded` catches them one step later. String `%` is refused + rather than screened; see the module docstring. + """ + grown = None + if isinstance(op, ast.Mod) and isinstance(left, (str, bytes)): + raise ValueError( + "refusing `%` formatting on a str/bytes operand: a precision field " + "sets the result size independently of both operands, so the cost " + "cannot be bounded before the allocation" + ) + if isinstance(op, (ast.Pow, ast.LShift)) and isinstance(right, int): + if isinstance(right, bool) or right < 0: + return + # A float base overflows to `inf` cheaply, so only an integer one grows. + if isinstance(op, ast.LShift): + grown = _size(left) + right + elif isinstance(left, int): + grown = _size(left) * right + elif isinstance(op, ast.Mult): + # Sequence repetition. Two integers multiplied only *add* bit lengths, + # so that case is left to the post-hoc check. + for value, count in ((left, right), (right, left)): + if ( + isinstance(count, int) + and not isinstance(count, bool) + and hasattr(value, "__len__") + ): + grown = len(value) * max(count, 0) + break + if grown is not None and grown > MAX_RESULT_SIZE: + raise ValueError( + f"refusing an expression whose result would exceed " + f"{MAX_RESULT_SIZE} units ({type(op).__name__})" + ) + + +def safe_eval(node: ast.expr): + """Value of a literal expression, computed without executing anything. + + Raises `ValueError` on any node that could name or call something outside + the expression, and on any expression whose cost is unbounded. + """ + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Tuple): + return tuple(safe_eval(element) for element in node.elts) + if isinstance(node, ast.List): + return [safe_eval(element) for element in node.elts] + if isinstance(node, ast.Set): + return {safe_eval(element) for element in node.elts} + if isinstance(node, ast.Dict): + # A `**spread` entry has a `None` key and names something to spread, so + # it is refused with everything else that names. + if any(key is None for key in node.keys): + raise ValueError("refusing a dict spread in a function argument") + return { + safe_eval(key): safe_eval(value) + for key, value in zip(node.keys, node.values, strict=True) + if key is not None # unreachable past the guard; narrows the type + } + if isinstance(node, ast.UnaryOp) and type(node.op) in _UNARY_OPS: + return _bounded(_UNARY_OPS[type(node.op)](safe_eval(node.operand))) + if isinstance(node, ast.BinOp) and type(node.op) in _BIN_OPS: + left = safe_eval(node.left) + right = safe_eval(node.right) + _refuse_if_unbounded(node.op, left, right) + return _bounded(_BIN_OPS[type(node.op)](left, right)) + raise ValueError( + f"refusing to evaluate {type(node).__name__} in a function argument" + ) + + +def _bounded(value): + """Return `value`, or refuse it for being larger than :data:`MAX_RESULT_SIZE`. + + The companion to :func:`_refuse_if_unbounded`: that one screens the + operators that can outrun their inputs, this one catches growth by + accumulation, where every single step looked affordable. + """ + if _size(value) > MAX_RESULT_SIZE: + raise ValueError( + f"refusing an intermediate larger than {MAX_RESULT_SIZE} units" + ) + return value diff --git a/sieval/community/bfcl_v3/_tables.py b/sieval/community/bfcl_v3/_tables.py new file mode 100644 index 00000000..31f63934 --- /dev/null +++ b/sieval/community/bfcl_v3/_tables.py @@ -0,0 +1,44 @@ +"""BFCL v3 category tables, shared by the dataset loader and the tasks. + +Held here rather than in either caller because both need them and neither may +import the other's private module. Keeping them out of the vendored files is +also what lets those stay byte-identical. + +Row counts are ours, measured at the pinned revision; the category membership is +upstream's `TEST_COLLECTION_MAPPING` at v1.3. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +NON_LIVE_COUNTS: dict[str, int] = { + "simple": 400, + "multiple": 200, + "parallel": 200, + "parallel_multiple": 200, + "java": 100, + "javascript": 50, + "irrelevance": 240, +} + +LIVE_COUNTS: dict[str, int] = { + "live_simple": 258, + "live_multiple": 1053, + "live_parallel": 16, + "live_parallel_multiple": 24, + "live_irrelevance": 882, + "live_relevance": 18, +} + +#: Upstream ships no `possible_answer/` file for these: the correct outcome is +#: about whether a call was produced, not about which call it was. +GOLDLESS_CATEGORIES = frozenset( + {"irrelevance", "live_irrelevance", "live_relevance"} +) + +#: Of the goldless three, the one where producing a call is CORRECT. +CALL_EXPECTED = frozenset({"live_relevance"}) + +#: Only these two are not Python. The same string selects both the +#: type converter inside `ast_checker` and the source-text parser inside +#: `ast_parse` -- these categories' model outputs are Java/JavaScript source. +LANGUAGE_BY_CATEGORY = {"java": "Java", "javascript": "JavaScript"} diff --git a/sieval/community/bfcl_v3/aggregate.py b/sieval/community/bfcl_v3/aggregate.py new file mode 100644 index 00000000..a0a2347c --- /dev/null +++ b/sieval/community/bfcl_v3/aggregate.py @@ -0,0 +1,65 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/eval_checker/eval_runner_helper.py +Upstream blob: 1de6462be8be9cc91d5a0e95c69e2c6fd6fc1a0d +License: Apache-2.0 + +Symbols taken (bodies unchanged): `calculate_weighted_accuracy`, +`calculate_unweighted_accuracy`. This is a partial extraction, not a +whole-file copy -- the rest of `eval_runner_helper.py` (leaderboard CSV +generation, cost/latency aggregation, pandas/numpy-based reporting) is out of +scope for this port. + +Only deviation: neither function needs any import, so there was nothing to +rewrite. +""" + + +def calculate_weighted_accuracy(accuracy_dict_list, display_na_if_category_missing=True): + has_na = False + total_count = 0 + total_accuracy = 0 + for accuracy_dict in accuracy_dict_list: + accuracy = accuracy_dict["accuracy"] + count = accuracy_dict["total_count"] + if accuracy_dict["display_accuracy"] == "N/A": + has_na = True + + total_count += count + total_accuracy += accuracy * count + + result = {"accuracy": total_accuracy / total_count, "total_count": total_count} + + if has_na and display_na_if_category_missing: + result["display_accuracy"] = "N/A" + else: + result["display_accuracy"] = result["accuracy"] + + return result + + +def calculate_unweighted_accuracy(accuracy_dict_list, display_na_if_category_missing=True): + has_na = False + total_count = 0 + total_accuracy = 0 + for accuracy_dict in accuracy_dict_list: + accuracy = accuracy_dict["accuracy"] + count = accuracy_dict["total_count"] + if accuracy_dict["display_accuracy"] == "N/A": + # If a category is not being evaluated, it will still be considered 0 in the overall score calculation. + has_na = True + + total_count += count + total_accuracy += accuracy + + result = { + "accuracy": total_accuracy / len(accuracy_dict_list), + "total_count": total_count, + } + + if has_na and display_na_if_category_missing: + result["display_accuracy"] = "N/A" + else: + result["display_accuracy"] = result["accuracy"] + + return result diff --git a/sieval/community/bfcl_v3/ast_checker.py b/sieval/community/bfcl_v3/ast_checker.py new file mode 100644 index 00000000..0782092e --- /dev/null +++ b/sieval/community/bfcl_v3/ast_checker.py @@ -0,0 +1,646 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/eval_checker/ast_eval/ast_checker.py +Upstream blob: 214f3ba55c012691761913b6aca68ab7bc4aa38b +License: Apache-2.0 + +Only deviation: absolute `bfcl_eval.*` imports rewritten to relative ones +(`.type_mappings`, `.type_convertor.*`, `._model_config`). No other edit. +""" + +from ._model_config import MODEL_CONFIG_MAPPING +from .type_mappings import ( + JAVA_TYPE_CONVERSION, + JS_TYPE_CONVERSION, +) +from .type_convertor.java_type_converter import java_type_converter +from .type_convertor.js_type_converter import js_type_converter +import re + +#### Constants #### +PYTHON_TYPE_MAPPING = { + "string": str, + "integer": int, + "float": float, + "boolean": bool, + "array": list, + "tuple": list, + "dict": dict, + "any": str, +} + +# This is the list of types that we need to recursively check its values +PYTHON_NESTED_TYPE_CHECK_LIST = ["array", "tuple"] + + +NESTED_CONVERSION_TYPE_LIST = ["Array", "ArrayList", "array"] + + +#### Main function #### +def ast_checker( + func_description, model_output, possible_answer, language, test_category, model_name +): + if "parallel" in test_category: + return parallel_function_checker_no_order( + func_description, model_output, possible_answer, language, model_name + ) + + elif "multiple" in test_category: + return multiple_function_checker( + func_description, model_output, possible_answer, language, model_name + ) + + else: + if len(model_output) != 1: + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "simple_function_checker:wrong_count", + } + + return simple_function_checker( + func_description[0], model_output[0], possible_answer[0], language, model_name + ) + + +#### Helper functions for AST #### +def find_description(func_descriptions, name): + if type(func_descriptions) == list: + for func_description in func_descriptions: + if func_description["name"] == name: + return func_description + return None + else: + # it is a dict, there is only one function + return func_descriptions + + +def get_possible_answer_type(possible_answer: list): + for answer in possible_answer: + if answer != "": # Optional parameter + return type(answer) + return None + + +def convert_func_name(function_name, model_name: str): + model_name_escaped = model_name.replace("_", "/") + if "." in function_name: + if MODEL_CONFIG_MAPPING[model_name_escaped].underscore_to_dot: + # OAI does not support "." in the function name so we replace it with "_". ^[a-zA-Z0-9_-]{1,64}$ is the regex for the name. + # This happens for OpenAI, Mistral, and Google models + return re.sub(r"\.", "_", function_name) + return function_name + + +def type_checker( + param: str, + value, + possible_answer: list, + expected_type_description: str, + expected_type_converted, + nested_type_converted, +): + # NOTE: This type checker only supports nested type checking for one level deep. + # We didn't implement recursive type checking for nested types, as it's not needed for the current use case and it's very complex. + + result = { + "valid": True, + "error": [], + "is_variable": False, + "error_type": "type_error:simple", + } + + is_variable = False + # check for the case where a variable is used instead of a actual value. + # use the type in possible_answer as the expected type + possible_answer_type = get_possible_answer_type(possible_answer) + # if possible_answer only contains optional parameters, we can't determine the type + if possible_answer_type != None: + # we are being precise here. + # in fact, possible_answer_type should always be string, as that's how we treat varibale in possible_answer + if possible_answer_type != expected_type_converted: + is_variable = True + + # value is the same type as in function description + if type(value) == expected_type_converted: + # We don't need to do recursive check for simple types + if nested_type_converted == None: + result["is_variable"] = is_variable + return result + else: + for possible_answer_item in possible_answer: + flag = True # Each parameter should match to at least one possible answer type. + # Here, we assume that each item should be the same type. We could also relax it. + if type(possible_answer_item) == list: + for value_item in value: + checker_result = type_checker( + param, + value_item, + possible_answer_item, + str(nested_type_converted), + nested_type_converted, + None, + ) + if not checker_result["valid"]: + flag = False + break + + if flag: + return {"valid": True, "error": [], "is_variable": is_variable} + + result["valid"] = False + result["error"] = [ + f"Nested type checking failed for parameter {repr(param)}. Expected outer type {expected_type_description} with inner type {str(nested_type_converted)}. Parameter value: {repr(value)}." + ] + result["error_type"] = "type_error:nested" + + # value is not as expected, check for the case where a variable is used instead of a actual value + # use the type in possible_answer as the expected type + possible_answer_type = get_possible_answer_type(possible_answer) + # if possible_answer only contains optional parameters, we can't determine the type + if possible_answer_type != None: + # we are being precise here. + # in fact, possible_answer_type should always be string, as that's how we treat varibale in possible_answer + if type(value) == possible_answer_type: + result["is_variable"] = True + return result + + result["valid"] = False + result["error"].append( + f"Incorrect type for parameter {repr(param)}. Expected type {expected_type_description}, got {type(value).__name__}. Parameter value: {repr(value)}." + ) + result["error_type"] = "type_error:simple" + return result + + +def standardize_string(input_string: str): + # This function standardizes the string by removing all the spaces, ",./-_*^" punctuation, and converting it to lowercase + # It will also convert all the single quotes to double quotes + # This is used to compare the model output with the possible answers + # We don't want to punish model for answer like April 1, 2024 vs April 1,2024, vs April 1 2024 + regex_string = r"[ \,\.\/\-\_\*\^]" + return re.sub(regex_string, "", input_string).lower().replace("'", '"') + + +def string_checker(param: str, model_output: str, possible_answer: list): + standardize_possible_answer = [] + standardize_model_output = standardize_string(model_output) + for i in range(len(possible_answer)): + if type(possible_answer[i]) == str: + standardize_possible_answer.append(standardize_string(possible_answer[i])) + + if standardize_model_output not in standardize_possible_answer: + return { + "valid": False, + "error": [ + f"Invalid value for parameter {repr(param)}: {repr(model_output)}. Expected one of {possible_answer}. Case insensitive." + ], + "error_type": "value_error:string", + } + + return {"valid": True, "error": []} + + +def list_checker(param: str, model_output: list, possible_answer: list): + # Convert the tuple to a list + + standardize_model_output = list(model_output) + + # If the element in the list is a string, we need to standardize it + for i in range(len(standardize_model_output)): + if type(standardize_model_output[i]) == str: + standardize_model_output[i] = standardize_string(model_output[i]) + + standardize_possible_answer = [] + # We also need to standardize the possible answers + for i in range(len(possible_answer)): + standardize_possible_answer.append([]) + for j in range(len(possible_answer[i])): + if type(possible_answer[i][j]) == str: + standardize_possible_answer[i].append( + standardize_string(possible_answer[i][j]) + ) + else: + standardize_possible_answer[i].append(possible_answer[i][j]) + + if standardize_model_output not in standardize_possible_answer: + return { + "valid": False, + "error": [ + f"Invalid value for parameter {repr(param)}: {repr(model_output)}. Expected one of {possible_answer}." + ], + "error_type": "value_error:list/tuple", + } + + return {"valid": True, "error": []} + + +def dict_checker(param: str, model_output: dict, possible_answers: list): + # This function works for simple dictionaries, but not dictionaries with nested dictionaries. + # The current dataset only contains simple dictionaries, so this is sufficient. + + result = {"valid": False, "error": [], "error_type": "dict_checker:unclear"} + for i in range(len(possible_answers)): + + if possible_answers[i] == "": + continue + + result = {"valid": False, "error": [], "error_type": "dict_checker:unclear"} + + flag = True + + possible_answer = possible_answers[i] + # possible_anwer is a single dictionary + + for key, value in model_output.items(): + if key not in possible_answer: + result["valid"] = False + result["error"].append(f"Unexpected dict key parameter: '{key}'.") + result["error_type"] = "value_error:dict_key" + flag = False + break + + standardize_value = value + # If the value is a string, we need to standardize it + if type(value) == str: + standardize_value = standardize_string(value) + + # We also need to standardize the possible answers if they are string + standardize_possible_answer = [] + for i in range(len(possible_answer[key])): + if type(possible_answer[key][i]) == str: + standardize_possible_answer.append( + standardize_string(possible_answer[key][i]) + ) + else: + standardize_possible_answer.append(possible_answer[key][i]) + + if standardize_value not in standardize_possible_answer: + result["valid"] = False + result["error"].append( + f"Invalid value for parameter {repr(key)}: {repr(value)}. Expected one of {standardize_possible_answer}." + ) + result["error_type"] = "value_error:dict_value" + flag = False + break + + for key, value in possible_answer.items(): + if key not in model_output and "" not in value: + result["valid"] = False + result["error"].append(f"Missing dict key parameter: '{key}'.") + result["error_type"] = "value_error:dict_key" + flag = False + break + + if flag: + return {"valid": True, "error": []} + + return result + + +def list_dict_checker(param: str, model_output: list, possible_answers: list): + # This function takes in a list of dictionaries and checks if each dictionary is valid + # The order of the dictionaries in the list must match the order of the possible answers + + result = {"valid": False, "error": [], "error_type": "list_dict_checker:unclear"} + + for answer_index in range(len(possible_answers)): + flag = True # True means so far, all dictionaries are valid + + # Only proceed if the number of dictionaries in the list matches the number of dictionaries in the possible answers + if len(model_output) != len(possible_answers[answer_index]): + result["valid"] = False + result["error"] = ["Wrong number of dictionaries in the list."] + result["error_type"] = "value_error:list_dict_count" + flag = False + continue + + for dict_index in range(len(model_output)): + result = dict_checker( + param, + model_output[dict_index], + [possible_answers[answer_index][dict_index]], + ) + if not result["valid"]: + flag = False + break + if flag: + return {"valid": True, "error": []} + + return result + + +def simple_function_checker( + func_description: dict, + model_output: dict, + possible_answer: dict, + language: str, + model_name: str, +): + possible_answer = list(possible_answer.values())[0] + # Extract function name and parameters details + func_name = func_description["name"] + param_details = func_description["parameters"]["properties"] + required_params = func_description["parameters"]["required"] + + # Initialize a result dictionary + result = { + "valid": True, + "error": [], + "error_type": "simple_function_checker:unclear", + } + + func_name = convert_func_name(func_name, model_name) + + # Check if function name matches + if func_name not in model_output: + result["valid"] = False + result["error"].append( + f"Function name {repr(func_name)} not found in model output." + ) + result["error_type"] = "simple_function_checker:wrong_func_name" + return result + + model_params = model_output[func_name] + + # Check for required parameters in model output + for param in required_params: + if param not in model_params: + result["valid"] = False + result["error"].append(f"Missing required parameter: {repr(param)}.") + result["error_type"] = "simple_function_checker:missing_required" + return result + + # Validate types and values for each parameter in model output + for param, value in model_params.items(): + if param not in param_details or param not in possible_answer: + result["valid"] = False + result["error"].append(f"Unexpected parameter: {repr(param)}.") + result["error_type"] = "simple_function_checker:unexpected_param" + return result + + full_param_details = param_details[param] + expected_type_description = full_param_details["type"] # This is a string + is_variable = False + nested_type_converted = None + + if language == "Java": + expected_type_converted = JAVA_TYPE_CONVERSION[expected_type_description] + + if expected_type_description in JAVA_TYPE_CONVERSION: + if type(value) != str: + result["valid"] = False + result["error"].append( + f"Incorrect type for parameter {repr(param)}. Expected type String, got {type(value).__name__}. Parameter value: {repr(value)}." + ) + result["error_type"] = "type_error:java" + return result + + if expected_type_description in NESTED_CONVERSION_TYPE_LIST: + nested_type = param_details[param]["items"]["type"] + nested_type_converted = JAVA_TYPE_CONVERSION[nested_type] + value = java_type_converter( + value, expected_type_description, nested_type + ) + else: + value = java_type_converter(value, expected_type_description) + + elif language == "JavaScript": + expected_type_converted = JS_TYPE_CONVERSION[expected_type_description] + + if expected_type_description in JS_TYPE_CONVERSION: + if type(value) != str: + result["valid"] = False + result["error"].append( + f"Incorrect type for parameter {repr(param)}. Expected type String, got {type(value).__name__}. Parameter value: {repr(value)}." + ) + result["error_type"] = "type_error:js" + return result + + if expected_type_description in NESTED_CONVERSION_TYPE_LIST: + nested_type = param_details[param]["items"]["type"] + nested_type_converted = JS_TYPE_CONVERSION[nested_type] + value = js_type_converter( + value, expected_type_description, nested_type + ) + else: + value = js_type_converter(value, expected_type_description) + + elif language == "Python": + expected_type_converted = PYTHON_TYPE_MAPPING[expected_type_description] + if expected_type_description in PYTHON_NESTED_TYPE_CHECK_LIST: + nested_type = param_details[param]["items"]["type"] + nested_type_converted = PYTHON_TYPE_MAPPING[nested_type] + + # We convert all tuple value to list when the expected type is tuple. + # The conversion is necessary because any tuple in the possible answer would become a list after being processed through json.dump() and json.load(). + # This does introduce some false positive (eg, when the model provides a list value instead of tuple). We hope to find a better solution in the future. + if expected_type_description == "tuple" and type(value) == tuple: + value = list(value) + + # Allow python auto conversion from int to float + if ( + language == "Python" + and expected_type_description == "float" + and type(value) == int + ): + value = float(value) + + # Type checking + # In fact, we only check for Python here. + # Type check for other languages are handled by the type converter, and so their value (after conversion) is always correct. + type_check_result = type_checker( + param, + value, + possible_answer[param], + expected_type_description, + expected_type_converted, + nested_type_converted, + ) + is_variable = type_check_result["is_variable"] + if not type_check_result["valid"]: + return type_check_result + + # It doesn't make sense to special handle dictionaries and list of dictionaries if the value is a variable. + # We can just treat the variable as a string and use the normal flow. + if not is_variable: + # Special handle for dictionaries + if expected_type_converted == dict: + result = dict_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + # Special handle for list of dictionaries + elif expected_type_converted == list and nested_type_converted == dict: + result = list_dict_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + # Special handle for strings + elif expected_type_converted == str: + # We don't check for case sensitivity for string, as long as it's not a variable + result = string_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + elif expected_type_converted == list: + result = list_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + # Check if the value is within the possible answers + if value not in possible_answer[param]: + result["valid"] = False + result["error"].append( + f"Invalid value for parameter {repr(param)}: {repr(value)}. Expected one of {possible_answer[param]}." + ) + result["error_type"] = "value_error:others" + return result + + # Check for optional parameters not provided but allowed + for param in possible_answer: + if param not in model_params and "" not in possible_answer[param]: + result["valid"] = False + result["error"].append( + f"Optional parameter {repr(param)} not provided and not marked as optional." + ) + result["error_type"] = "simple_function_checker:missing_optional" + return result + + return result + + +def parallel_function_checker_enforce_order( + func_descriptions: list, + model_output: list, + possible_answers: dict, + language: str, + model_name: str, +): + if len(model_output) != len(possible_answers): + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "parallel_function_checker_enforce_order:wrong_count", + } + + func_name_list = list(possible_answers.keys()) + possible_answers_list = [] + + for key, value in possible_answers.items(): + possible_answers_list.append({key: value}) + + for i in range(len(possible_answers_list)): + func_description = find_description(func_descriptions, func_name_list[i]) + + result = simple_function_checker( + func_description, + model_output[i], + possible_answers_list[i], + language, + model_name, + ) + if not result["valid"]: + return result + + return {"valid": True, "error": []} + + +def parallel_function_checker_no_order( + func_descriptions: list, + model_output: list, + possible_answers: list, + language: str, + model_name: str, +): + if len(model_output) != len(possible_answers): + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "parallel_function_checker_no_order:wrong_count", + } + + matched_indices = [] + + # We go throught the possible answers one by one, and eliminate the model output that matches the possible answer + # It must be this way because we need ground truth to fetch the correct function description + for i in range(len(possible_answers)): + # possible_answers[i] is a dictionary with only one key + func_name_expected = list(possible_answers[i].keys())[0] + func_description = find_description(func_descriptions, func_name_expected) + + + all_errors = [] + + for index in range(len(model_output)): + if index in matched_indices: + continue + + result = simple_function_checker( + func_description, + model_output[index], + possible_answers[i], + language, + model_name, + ) + + if result["valid"]: + matched_indices.append(index) + break + else: + all_errors.append( + { + f"Model Result Index {index}": { + "sub_error": result["error"], + "sub_error_type": result["error_type"], + "model_output_item": model_output[index], + "possible_answer_item": possible_answers[i], + } + } + ) + + if not result["valid"]: + considered_indices = [ + i for i in range(len(model_output)) if i not in matched_indices + ] + all_errors.insert( + 0, + f"Could not find a matching function among index {considered_indices} of model output for index {i} of possible answers.", + ) + return { + "valid": False, + "error": all_errors, + "error_type": "parallel_function_checker_no_order:cannot_find_match", + } + + return {"valid": True, "error": []} + + +def multiple_function_checker( + func_descriptions: list, + model_output: list, + possible_answers: list, + language: str, + model_name: str, +): + if len(model_output) != len(possible_answers): + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "multiple_function_checker:wrong_count", + } + + # possible_answers is a list of only one dictionary with only one key + func_name_expected = list(possible_answers[0].keys())[0] + func_description = find_description(func_descriptions, func_name_expected) + return simple_function_checker( + func_description, + model_output[0], + possible_answers[0], + language, + model_name, + ) diff --git a/sieval/community/bfcl_v3/output_checks.py b/sieval/community/bfcl_v3/output_checks.py new file mode 100644 index 00000000..92bf012e --- /dev/null +++ b/sieval/community/bfcl_v3/output_checks.py @@ -0,0 +1,49 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/utils.py +Upstream blob: c28472cd8b24e250d3412c5edf3b7b49c748f915 +License: Apache-2.0 + +Symbols taken (bodies unchanged): `is_function_calling_format_output`, +`is_empty_output`. This is a partial extraction, not a whole-file copy -- the +rest of `bfcl_eval/utils.py` (test-category helpers, file I/O, CLI argument +parsing) is out of scope for this port. + +Only deviation: neither function needs any import, so there was nothing to +rewrite. +""" + + +def is_function_calling_format_output(decoded_output): + """ + Ensure the output is a list of dictionaries of the form: + `[{func1: {param1: val1, param2: val2, ...}}, {func2: {param1: val1, param2: val2, ...}}, ...]` + Sometimes the model handler's `decode_ast` method will return successfully, but the output is not in the correct format, and that will mess up the downstream evaluation that expects this format. + This is especially the case when the model doesn't predict any function calls, and the output is an human-readable string. + Note: Empty list `[]` is considered the correct format in this check. + """ + if type(decoded_output) != list: + return False + for item in decoded_output: + if type(item) != dict: + return False + # Check for `{func1: {param1: val1, param2: val2, ...}}`, should only have one key-value pair + if len(item) != 1: + return False + # Check for `{param1: val1, param2: val2, ...}`; the parameter-value pairs should be a dictionary + if type(list(item.values())[0]) != dict: + return False + return True + + +def is_empty_output(decoded_output): + # This function is a patch to the ast decoder for relevance detection + # Sometimes the ast decoder will parse successfully, but the input doens't really have a function call + # [], [{}], and anything that is not in function calling format is considered empty (and thus should be marked as correct) + if not is_function_calling_format_output(decoded_output): + return True + if len(decoded_output) == 0: + return True + if len(decoded_output) == 1 and len(decoded_output[0]) == 0: + return True + return False diff --git a/sieval/community/bfcl_v3/parser.py b/sieval/community/bfcl_v3/parser.py new file mode 100644 index 00000000..dcde4a48 --- /dev/null +++ b/sieval/community/bfcl_v3/parser.py @@ -0,0 +1,123 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/model_handler/utils.py +Upstream blob: f78acc240bee3c43b18de3d4195fbf6ee1a207f1 +License: Apache-2.0 + +Symbols taken (bodies unchanged): `ast_parse`, `resolve_ast_call`, +`resolve_ast_by_type`. A partial extraction -- the rest of +`model_handler/utils.py` (prompt construction, other model styles' decoders, +retry helpers) is out of scope. + +Deviations: +1. The two source-parser imports are rewritten to `.source_parser.*`, renamed + from upstream's `parser/` because this file already occupies that name. + Nothing else needed rewriting: these three functions reach no `bfcl_eval.*`. +2. `resolve_ast_by_type`'s `ast.BinOp` branch calls `safe_eval` where upstream + calls `eval(ast.unparse(value))`. That node is model output, so upstream's + line executes model-authored code -- `f(x=__import__('os').system('...') + + 0)` runs the call and hands back an ordinary-looking number. `safe_eval` + computes the same value for every expression that does not execute + something; what it refuses, and why it lives outside this file, is in + `_safe_eval.py`. + + The `ast.Lambda` branch keeps upstream's `eval` because it cannot reach it: + `Lambda.body` is a single expression node, so `value.body[0]` raises + `TypeError` first -- and that raise is already the decode failure upstream + scores. Hardening an unreachable branch buys nothing. + +`ast_parse` dispatches `java`/`javascript` rows to the tree-sitter parsers +rather than `ast.parse`: their model output is source text, not Python call +syntax. +""" + +import ast + +from ._safe_eval import safe_eval +from .source_parser.java_parser import parse_java_function_call +from .source_parser.js_parser import parse_javascript_function_call + + +def ast_parse(input_str: str, language: str="Python") -> list[dict]: + if language == "Python": + cleaned_input = input_str.strip("[]'") + parsed = ast.parse(cleaned_input, mode="eval") + extracted = [] + if isinstance(parsed.body, ast.Call): + extracted.append(resolve_ast_call(parsed.body)) + else: + for elem in parsed.body.elts: + assert isinstance(elem, ast.Call) + extracted.append(resolve_ast_call(elem)) + return extracted + elif language == "Java": + return parse_java_function_call( + input_str[1:-1] + ) # Remove the [ and ] from the string + elif language == "JavaScript": + return parse_javascript_function_call(input_str[1:-1]) + else: + raise NotImplementedError(f"Unsupported language: {language}") + + +def resolve_ast_call(elem): + # Handle nested attributes for deeply nested module paths + func_parts = [] + func_part = elem.func + while isinstance(func_part, ast.Attribute): + func_parts.append(func_part.attr) + func_part = func_part.value + if isinstance(func_part, ast.Name): + func_parts.append(func_part.id) + func_name = ".".join(reversed(func_parts)) + args_dict = {} + for arg in elem.keywords: + output = resolve_ast_by_type(arg.value) + args_dict[arg.arg] = output + return {func_name: args_dict} + + +def resolve_ast_by_type(value): + if isinstance(value, ast.Constant): + if value.value is Ellipsis: + output = "..." + else: + output = value.value + elif isinstance(value, ast.UnaryOp): + output = -value.operand.value + elif isinstance(value, ast.List): + output = [resolve_ast_by_type(v) for v in value.elts] + elif isinstance(value, ast.Dict): + output = { + resolve_ast_by_type(k): resolve_ast_by_type(v) + for k, v in zip(value.keys, value.values) + } + elif isinstance( + value, ast.NameConstant + ): # Added this condition to handle boolean values + output = value.value + elif isinstance( + value, ast.BinOp + ): # Added this condition to handle function calls as arguments + output = safe_eval(value) # DEVIATION 3 -- upstream: eval(ast.unparse(value)) + elif isinstance(value, ast.Name): + output = value.id + elif isinstance(value, ast.Call): + if len(value.keywords) == 0: + output = ast.unparse(value) + else: + output = resolve_ast_call(value) + elif isinstance(value, ast.Tuple): + output = tuple(resolve_ast_by_type(v) for v in value.elts) + elif isinstance(value, ast.Lambda): + output = eval(ast.unparse(value.body[0].value)) + elif isinstance(value, ast.Ellipsis): + output = "..." + elif isinstance(value, ast.Subscript): + try: + output = ast.unparse(value.body[0].value) + except: + output = ast.unparse(value.value) + "[" + ast.unparse(value.slice) + "]" + else: + raise Exception(f"Unsupported AST type: {type(value)}") + return output diff --git a/sieval/community/bfcl_v3/preprocess.py b/sieval/community/bfcl_v3/preprocess.py new file mode 100644 index 00000000..ede4d52a --- /dev/null +++ b/sieval/community/bfcl_v3/preprocess.py @@ -0,0 +1,118 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/model_handler/utils.py +Upstream blob: f78acc240bee3c43b18de3d4195fbf6ee1a207f1 +License: Apache-2.0 + +Symbols taken (bodies unchanged): `system_prompt_pre_processing_chat_model`, +`_get_language_specific_hint`, `func_doc_language_specific_pre_processing`. +This is a partial extraction, not a whole-file copy -- the rest of +`bfcl_eval/model_handler/utils.py` (multi-turn state helpers, response +formatters, the other model families' prompt builders) is out of scope. + +Both public symbols run on the way OUT, shaping what the model is shown. They +are deliberately not applied on the way back in: upstream's `ast_file_runner` +grades against the *unprocessed* schema, so the checker still sees the real +declared types. + +Only deviation: upstream imports `DEFAULT_SYSTEM_PROMPT` from +`bfcl_eval.constants.default_prompts`; that is rewritten to the relative import +of this package's copy. +""" + +import json + +from .prompts import DEFAULT_SYSTEM_PROMPT + + +def system_prompt_pre_processing_chat_model(prompts, function_docs, test_category): + """ + Add a system prompt to the chat model to instruct the model on the available functions and the expected response format. + If the prompts list already contains a system prompt, append the additional system prompt content to the existing system prompt. + """ + assert type(prompts) == list + + system_prompt_template = DEFAULT_SYSTEM_PROMPT + + system_prompt = system_prompt_template.format(functions=function_docs) + + # System prompt must be in the first position + # If the question comes with a system prompt, append its content at the end of the chat template. + if prompts[0]["role"] == "system": + prompts[0]["content"] = system_prompt + "\n\n" + prompts[0]["content"] + # Otherwise, use the system prompt template to create a new system prompt. + else: + prompts.insert( + 0, + {"role": "system", "content": system_prompt}, + ) + + return prompts + + +def _get_language_specific_hint(test_category): + if test_category == "java": + return " Note that the provided function is in Java 8 SDK syntax." + elif test_category == "javascript": + return " Note that the provided function is in JavaScript syntax." + else: + return " Note that the provided function is in Python 3 syntax." + + +def func_doc_language_specific_pre_processing(function, test_category): + if len(function) == 0: + return function + + assert type(function) == list + for item in function: + # Add language specific hints to the function description + func_description = item["description"] + item["description"] = item["description"] + _get_language_specific_hint( + test_category + ) + # Process the parameters + properties = item["parameters"]["properties"] + if test_category == "java": + for key, value in properties.items(): + if value["type"] == "any": + properties[key][ + "description" + ] += " This parameter can be of any type of Java object in string representation." + else: + value[ + "description" + ] += f" This is Java {value['type']} type parameter in string representation." + if value["type"] == "ArrayList" or value["type"] == "Array": + value[ + "description" + ] += f" The list elements are of type {value['items']['type']}; they are not in string representation." + del value["items"] + + value["type"] = "string" + + elif test_category == "javascript": + for key, value in properties.items(): + if value["type"] == "any": + properties[key][ + "description" + ] += " This parameter can be of any type of JavaScript object in string representation." + else: + value[ + "description" + ] += f" This is JavaScript {value['type']} type parameter in string representation." + if value["type"] == "array": + value[ + "description" + ] += f" The list elements are of type {value['items']['type']}; they are not in string representation." + del value["items"] + + if value["type"] == "dict": + if "properties" in value: # not every dict has properties + value[ + "description" + ] += f" The dictionary entries have the following schema; they are not in string representation. {json.dumps(value['properties'])}" + del value["properties"] + + value["type"] = "string" + + return function diff --git a/sieval/community/bfcl_v3/prompts.py b/sieval/community/bfcl_v3/prompts.py new file mode 100644 index 00000000..6053cb23 --- /dev/null +++ b/sieval/community/bfcl_v3/prompts.py @@ -0,0 +1,32 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/constants/default_prompts.py +Upstream blob: 37eb080480daff689748601c2c77179578c187e3 +License: Apache-2.0 + +Whole file, copied entire. This file has no imports, so no rewrite was +needed here. +""" + +MAXIMUM_STEP_LIMIT = 20 + +DEFAULT_SYSTEM_PROMPT_WITHOUT_FUNC_DOC = """You are an expert in composing functions. You are given a question and a set of possible functions. Based on the question, you will need to make one or more function/tool calls to achieve the purpose. +If none of the functions can be used, point it out. If the given question lacks the parameters required by the function, also point it out. +You should only return the function calls in your response. + +If you decide to invoke any of the function(s), you MUST put it in the format of [func_name1(params_name1=params_value1, params_name2=params_value2...), func_name2(params)] +You SHOULD NOT include any other text in the response. + +At each turn, you should try your best to complete the tasks requested by the user within the current turn. Continue to output functions to call until you have fulfilled the user's request to the best of your ability. Once you have no more functions to call, the system will consider the current turn complete and proceed to the next turn or task. +""" + +DEFAULT_SYSTEM_PROMPT = ( + DEFAULT_SYSTEM_PROMPT_WITHOUT_FUNC_DOC + + """ +Here is a list of functions in JSON format that you can invoke.\n{functions}\n +""" +) + +DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC = "I have updated some more functions you can choose from. What about now?" + +DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_PROMPTING = "{functions}\n" + DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC diff --git a/sieval/community/bfcl_v3/source_parser/__init__.py b/sieval/community/bfcl_v3/source_parser/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sieval/community/bfcl_v3/source_parser/java_parser.py b/sieval/community/bfcl_v3/source_parser/java_parser.py new file mode 100644 index 00000000..4601211b --- /dev/null +++ b/sieval/community/bfcl_v3/source_parser/java_parser.py @@ -0,0 +1,148 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/model_handler/parser/java_parser.py +Upstream blob: f9081a958f1d173427064c3505d25ecbebf8aa70 +License: Apache-2.0 + +Only deviation: none beyond this docstring. The module imports only +`tree_sitter` and `tree_sitter_java`, neither of which is `bfcl_eval.*`. + +Held in `source_parser/` rather than upstream's `parser/`, whose name this +package's own `parser.py` already occupies. Nothing inside the file changed. +""" + +from tree_sitter import Language, Parser +import tree_sitter_java + +JAVA_LANGUAGE = Language(tree_sitter_java.language(), "java") + +parser = Parser() +parser.set_language(JAVA_LANGUAGE) + + +def parse_java_function_call(source_code): + tree = parser.parse(bytes(source_code, "utf8")) + root_node = tree.root_node + sexp_result = root_node.sexp() + + if "ERROR" in sexp_result: + raise SyntaxError("Error parsing java the source code.") + + def get_text(node): + """Returns the text represented by the node.""" + return source_code[node.start_byte : node.end_byte] + + def traverse_node(node, nested=False): + if node.type == "string_literal": + if nested: + return get_text(node) + # Strip surrounding quotes from string literals + return get_text(node)[1:-1] + elif node.type == "character_literal": + if nested: + return get_text(node) + # Strip surrounding single quotes from character literals + return get_text(node)[1:-1] + """Traverse the node to collect texts for complex structures.""" + if node.type in [ + "identifier", + "class_literal", + "type_identifier", + "method_invocation", + ]: + return get_text(node) + elif node.type == "array_creation_expression": + # Handle array creation expression specifically + type_node = node.child_by_field_name("type") + value_node = node.child_by_field_name("value") + type_text = traverse_node(type_node, True) + value_text = traverse_node(value_node, True) + return f"new {type_text}[]{value_text}" + elif node.type == "object_creation_expression": + # Handle object creation expression specifically + type_node = node.child_by_field_name("type") + arguments_node = node.child_by_field_name("arguments") + type_text = traverse_node(type_node, True) + if arguments_node: + # Process each argument carefully, avoiding unnecessary punctuation + argument_texts = [] + for child in arguments_node.children: + if child.type not in [ + ",", + "(", + ")", + ]: # Exclude commas and parentheses + argument_text = traverse_node(child, True) + argument_texts.append(argument_text) + arguments_text = ", ".join(argument_texts) + return f"new {type_text}({arguments_text})" + else: + return f"new {type_text}()" + elif node.type == "set": + # Handling sets specifically + items = [ + traverse_node(n, True) + for n in node.children + if n.type not in [",", "set"] + ] + return "{" + ", ".join(items) + "}" + + elif node.child_count > 0: + return "".join(traverse_node(child, True) for child in node.children) + else: + return get_text(node) + + def extract_arguments(args_node): + arguments = {} + for child in args_node.children: + if child.type == "assignment_expression": + # For named parameters + name_node, value_node = child.children[0], child.children[2] + name = get_text(name_node) + value = traverse_node(value_node) + if name in arguments: + if not isinstance(arguments[name], list): + arguments[name] = [arguments[name]] + arguments[name].append(value) + else: + arguments[name] = value + # arguments.append({'name': name, 'value': value}) + elif child.type in ["identifier", "class_literal", "set"]: + # For unnamed parameters and handling sets + value = traverse_node(child) + if None in arguments: + if not isinstance(arguments[None], list): + arguments[None] = [arguments[None]] + arguments[None].append(value) + else: + arguments[None] = value + return arguments + + def traverse(node): + if node.type == "method_invocation": + # Extract the function name and its arguments + method_name = get_text(node.child_by_field_name("name")) + class_name_node = node.child_by_field_name("object") + if class_name_node: + class_name = get_text(class_name_node) + function_name = f"{class_name}.{method_name}" + else: + function_name = method_name + arguments_node = node.child_by_field_name("arguments") + if arguments_node: + arguments = extract_arguments(arguments_node) + for key, value in arguments.items(): + if isinstance(value, list): + raise Exception( + "Error: Multiple arguments with the same name are not supported." + ) + return [{function_name: arguments}] + + else: + for child in node.children: + result = traverse(child) + if result: + return result + + result = traverse(root_node) + return result if result else {} diff --git a/sieval/community/bfcl_v3/source_parser/js_parser.py b/sieval/community/bfcl_v3/source_parser/js_parser.py new file mode 100644 index 00000000..9421a45c --- /dev/null +++ b/sieval/community/bfcl_v3/source_parser/js_parser.py @@ -0,0 +1,76 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/model_handler/parser/js_parser.py +Upstream blob: 162510cf681283c95458904177776a41271d014a +License: Apache-2.0 + +Only deviation: none beyond this docstring. The module imports only +`tree_sitter` and `tree_sitter_javascript`, neither of which is `bfcl_eval.*`. + +Held in `source_parser/` rather than upstream's `parser/`, whose name this +package's own `parser.py` already occupies. Nothing inside the file changed. +""" + +from tree_sitter import Language, Parser +import tree_sitter_javascript + +JS_LANGUAGE = Language(tree_sitter_javascript.language(), "javascript") + +parser = Parser() +parser.set_language(JS_LANGUAGE) + + +def parse_javascript_function_call(source_code): + # Parse the source code + tree = parser.parse(bytes(source_code, "utf8")) + root_node = tree.root_node + sexp_result = root_node.sexp() + if "ERROR" in sexp_result: + raise SyntaxError("Error js parsing the source code.") + + # Function to recursively extract argument details + def extract_arguments(node): + args = {} + for child in node.children: + if child.type == "assignment_expression": + # Extract left (name) and right (value) parts of the assignment + name = child.children[0].text.decode("utf-8") + value = child.children[2].text.decode("utf-8") + if (value.startswith('"') and value.endswith('"')) or ( + value.startswith("'") and value.endswith("'") + ): + value = value[1:-1] # Trim the quotation marks + if name in args: + if not isinstance(args[name], list): + args[name] = [args[name]] + args[name].append(value) + else: + args[name] = value + + elif child.type == "identifier" or child.type == "true": + # Handle non-named arguments and boolean values + value = child.text.decode("utf-8") + if None in args: + if not isinstance(args[None], list): + args[None] = [args[None]] + args[None].append(value) + else: + args[None] = value + return args + + # Find the function call and extract its name and arguments + if root_node.type == "program": + for child in root_node.children: + if child.type == "expression_statement": + for sub_child in child.children: + if sub_child.type == "call_expression": + function_name = sub_child.children[0].text.decode("utf8") + arguments_node = sub_child.children[1] + parameters = extract_arguments(arguments_node) + for key, value in parameters.items(): + if isinstance(value, list): + raise Exception( + "Error: Multiple arguments with the same name are not supported." + ) + result = [{function_name: parameters}] + return result diff --git a/sieval/community/bfcl_v3/tool_convert.py b/sieval/community/bfcl_v3/tool_convert.py new file mode 100644 index 00000000..d82d3fe7 --- /dev/null +++ b/sieval/community/bfcl_v3/tool_convert.py @@ -0,0 +1,217 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/model_handler/utils.py +Upstream blob: f78acc240bee3c43b18de3d4195fbf6ee1a207f1 +License: Apache-2.0 + +Symbols taken (bodies unchanged): `convert_to_tool`, `_cast_to_openai_type`. +A partial extraction -- the rest of `model_handler/utils.py` is out of scope. + +Deviations, beyond the usual import rewrite: + +- `_cast_to_openai_type` compares its `mapping` argument against the module + global `GORILLA_TO_OPENAPI`, imported here from the vendored `.type_mappings`. +- `convert_to_tool` branches on upstream's `ModelStyle` + (`bfcl_eval.model_handler.model_style`). Rather than vendor that file, the + enum below copies the 11 members these two functions reference, with + upstream's values. `Gorilla` and `NEXUS` are omitted: present upstream, read + by neither copied function. +""" + +import copy +import json +import re +from enum import Enum + +from .type_mappings import GORILLA_TO_OPENAPI + + +class ModelStyle(Enum): + OpenAI_Completions = "openai-completions" + OpenAI_Responses = "openai-responses" + Anthropic = "claude" + Mistral = "mistral" + GOOGLE = "google" + AMAZON = "amazon" + FIREWORK_AI = "firework_ai" + OSSMODEL = "ossmodel" + COHERE = "cohere" + WRITER = "writer" + NOVITA_AI = "novita_ai" + + +def _cast_to_openai_type(properties, mapping): + for key, value in properties.items(): + if "type" not in value: + properties[key]["type"] = "string" + else: + var_type = value["type"] + if mapping == GORILLA_TO_OPENAPI and var_type == "float": + properties[key]["format"] = "float" + properties[key]["description"] += " This is a float type value." + if var_type in mapping: + properties[key]["type"] = mapping[var_type] + else: + properties[key]["type"] = "string" + + # Currently support: + # - list of any + # - list of list of any + # - list of dict + # - list of list of dict + # - dict of any + + if properties[key]["type"] == "array" or properties[key]["type"] == "object": + if "properties" in properties[key]: + properties[key]["properties"] = _cast_to_openai_type( + properties[key]["properties"], mapping + ) + elif "items" in properties[key]: + properties[key]["items"]["type"] = mapping[properties[key]["items"]["type"]] + if ( + properties[key]["items"]["type"] == "array" + and "items" in properties[key]["items"] + ): + properties[key]["items"]["items"]["type"] = mapping[ + properties[key]["items"]["items"]["type"] + ] + elif ( + properties[key]["items"]["type"] == "object" + and "properties" in properties[key]["items"] + ): + properties[key]["items"]["properties"] = _cast_to_openai_type( + properties[key]["items"]["properties"], mapping + ) + return properties + + +def convert_to_tool(functions, mapping, model_style): + functions = copy.deepcopy(functions) + oai_tool = [] + for item in functions: + if "." in item["name"] and model_style in [ + ModelStyle.OpenAI_Completions, + ModelStyle.OpenAI_Responses, + ModelStyle.Mistral, + ModelStyle.GOOGLE, + ModelStyle.OSSMODEL, + ModelStyle.Anthropic, + ModelStyle.COHERE, + ModelStyle.AMAZON, + ModelStyle.NOVITA_AI, + ]: + # OAI does not support "." in the function name so we replace it with "_". ^[a-zA-Z0-9_-]{1,64}$ is the regex for the name. + item["name"] = re.sub(r"\.", "_", item["name"]) + + item["parameters"]["type"] = "object" + item["parameters"]["properties"] = _cast_to_openai_type( + item["parameters"]["properties"], mapping + ) + + if model_style == ModelStyle.Anthropic: + item["input_schema"] = item["parameters"] + del item["parameters"] + + if model_style == ModelStyle.AMAZON: + item["inputSchema"] = {"json": item["parameters"]} + del item["parameters"] + + if model_style in [ + ModelStyle.GOOGLE, + ModelStyle.WRITER, + ]: + # Remove fields that are not supported by Gemini or Palmyra. + # No `optional` field in function schema. + if "optional" in item["parameters"]: + del item["parameters"]["optional"] + for params in item["parameters"]["properties"].values(): + if "description" not in params: + params["description"] = "" + # No `default` field in GOOGLE or Palmyra's schema. + if "default" in params: + params["description"] += f" Default is: {str(params['default'])}." + del params["default"] + # No `optional` field in parameter schema as well. + if "optional" in params: + params["description"] += f" Optional: {str(params['optional'])}." + del params["optional"] + # No `required` field in parameter schema as well. + if "required" in params: + params["description"] += f" Required: {str(params['required'])}." + del params["required"] + # No `maximum` field. + if "maximum" in params: + params["description"] += f" Maximum value: {str(params['maximum'])}." + del params["maximum"] + # No `minItems` field. + if "minItems" in params: + params[ + "description" + ] += f" Minimum number of items: {str(params['minItems'])}." + del params["minItems"] + # No `maxItems` field. + if "maxItems" in params: + params[ + "description" + ] += f" Maximum number of items: {str(params['maxItems'])}." + del params["maxItems"] + # No `additionalProperties` field. + if "additionalProperties" in params: + params[ + "description" + ] += f" Additional properties: {str(params['additionalProperties'])}." + del params["additionalProperties"] + # For Gemini, only `enum` field when the type is `string`. + # For Palmyra, `enum` field is not supported. + if "enum" in params and ( + model_style == ModelStyle.WRITER + or (model_style == ModelStyle.GOOGLE and params["type"] != "string") + ): + params["description"] += f" Enum values: {str(params['enum'])}." + del params["enum"] + # No `format` when type is `string` + if "format" in params and params["type"] == "string": + params["description"] += f" Format: {str(params['format'])}." + del params["format"] + + # Process the return field + if "response" in item: + if model_style in [ + ModelStyle.Anthropic, + ModelStyle.GOOGLE, + ModelStyle.FIREWORK_AI, + ModelStyle.WRITER, + ModelStyle.AMAZON, + ModelStyle.NOVITA_AI, + ]: + item[ + "description" + ] += f" The response field has the following schema: {json.dumps(item['response'])}" + del item["response"] + + if model_style in [ + ModelStyle.Anthropic, + ModelStyle.GOOGLE, + ModelStyle.OSSMODEL, + ]: + oai_tool.append(item) + elif model_style in [ + ModelStyle.OpenAI_Responses + ]: + oai_tool.append({"type": "function", + "name": item["name"], + "description": item["description"], + "parameters": item["parameters"]}) + elif model_style in [ + ModelStyle.COHERE, + ModelStyle.OpenAI_Completions, + ModelStyle.Mistral, + ModelStyle.FIREWORK_AI, + ModelStyle.WRITER, + ModelStyle.NOVITA_AI, + ]: + oai_tool.append({"type": "function", "function": item}) + elif model_style == ModelStyle.AMAZON: + oai_tool.append({"toolSpec": item}) + + return oai_tool diff --git a/sieval/community/bfcl_v3/type_convertor/__init__.py b/sieval/community/bfcl_v3/type_convertor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sieval/community/bfcl_v3/type_convertor/java_type_converter.py b/sieval/community/bfcl_v3/type_convertor/java_type_converter.py new file mode 100644 index 00000000..a558a8af --- /dev/null +++ b/sieval/community/bfcl_v3/type_convertor/java_type_converter.py @@ -0,0 +1,420 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/eval_checker/ast_eval/type_convertor/java_type_converter.py +Upstream blob: 7d2b141dfcb8e527df6424893e60a833eb69f3e1 +License: Apache-2.0 + +Only deviation: `from bfcl_eval.constants.type_mappings import JAVA_TYPE_CONVERSION` +rewritten to the absolute `sieval.community.bfcl_v3.type_mappings` path. Absolute +rather than `..type_mappings` because the import policy reserves relative imports +for the same package, and upstream's line was absolute anyway -- one rewritten +import either way. No other edit. +""" + +import re +from typing import List, Dict, Union +from sieval.community.bfcl_v3.type_mappings import JAVA_TYPE_CONVERSION + + +def java_type_converter(value, expected_type, nested_type=None): + if expected_type not in JAVA_TYPE_CONVERSION: + raise ValueError(f"Unsupported type: {expected_type}") + if ( + expected_type == "byte" + or expected_type == "short" + or expected_type == "integer" + ): + if not re.match(r"^-?\d+$", value): + return str(value) # default to string + return int(value) + elif expected_type == "float": + if not re.match(r"^-?\d+(\.\d+)?([eE][+-]?\d+)?[fF]$", value): + return str(value) # default to string + return float(re.sub(r"[fF]$", "", value)) + elif expected_type == "double": + if not re.match(r"^-?\d+(\.\d+)?([eE][+-]?\d+)?$", value): + return str(value) # default to string + return float(value) + elif expected_type == "long": + if not re.match(r"^-?\d+[lL]$", value): + return str(value) # default to string + return int(re.sub(r"[lL]$", "", value)) + elif expected_type == "boolean": + if value not in ["true", "false"]: + return str(value) # default to string + return parse_java_boolean(value) + elif expected_type == "char": + if not re.match(r"^\'.$\'", value): + return str(value) # default to string + return value # Remove the single quotes + elif expected_type == "Array" or expected_type == "ArrayList": + return parse_java_collection(value, expected_type, nested_type) + elif expected_type == "Set": + raise NotImplementedError("Set conversion is not implemented") + elif expected_type == "HashMap": + return parse_java_collection(value, expected_type, nested_type) + elif expected_type == "Hashtable": + raise NotImplementedError("Set conversion is not implemented") + elif expected_type == "Queue" or expected_type == "Stack": + raise NotImplementedError(f"{expected_type} conversion is not implemented") + elif expected_type == "String" or expected_type == "any": + return str(value) # we output as string for `any` type + else: + raise ValueError(f"Unsupported type: {expected_type}") + + +def parse_java_boolean(value): + return value == "true" + + +def parse_java_collection( + input_str: str, type_str: str, nested_type=None +) -> Union[List, Dict]: + if type_str == "ArrayList": + return parse_arraylist(input_str, nested_type) + elif type_str == "Array": + return parse_array(input_str, nested_type) + elif type_str == "HashMap": + return parse_hashmap(input_str) + else: + raise ValueError(f"Unsupported type: {type_str}") + + +def parse_arraylist(input_str: str, nested_type=None) -> List: + match_asList = re.search( + r"new\s+ArrayList<\w*>\(Arrays\.asList\((.+?)\)\)", input_str + ) + if match_asList: + elements_str = match_asList.group(1) + elements = [] + for element_str in elements_str.split(","): + element_str = element_str.strip() + if nested_type == "char": + element = element_str[1:-1] # Remove the single quotes + elif nested_type == "String": + element = element_str[1:-1] # Remove the double quotes + else: + element = ( + java_type_converter(element_str, nested_type) + if nested_type + else parse_java_value(element_str) + ) + elements.append(element) + return elements + + match_add = re.search( + r"new\s+ArrayList<\w*>\(\)\s*\{\{\s*(.+?)\s*\}\}", input_str, re.DOTALL + ) + if match_add: + adds_str = match_add.group(1) + elements = [] + matches = re.findall(r"add\((.+?)\)", adds_str) + for match in matches: + value_str = match.strip() + if nested_type == "char": + value = value_str[1:-1] # Remove the single quotes + elif nested_type == "String": + value = value_str[1:-1] # Remove the double quotes + else: + value = ( + java_type_converter(value_str, nested_type) + if nested_type + else parse_java_value(value_str) + ) + elements.append(value) + return elements + + match_empty = re.search(r"new\s+ArrayList<\w*>\(\)", input_str) + if match_empty: + return [] # Return an empty list for an empty ArrayList + + return input_str # default to string + + +def parse_array(input_str: str, nested_type=None) -> List: + match = re.search(r"new\s+\w+\[\]\s*\{(.*?)\}", input_str) + if match: + elements_str = match.group(1) + if nested_type: + elements = [ + java_type_converter(x.strip(), nested_type) + for x in elements_str.split(",") + if x.strip() + ] + else: + elements = [ + parse_java_value(x.strip()) + for x in elements_str.split(",") + if x.strip() + ] + + return elements + else: + return input_str # default to string + + +def parse_hashmap(input_str: str) -> Dict: + elements = {} + match = re.search( + r"new\s+HashMap<.*?>\s*\(\)\s*\{\s*\{?\s*(.*?)\s*\}?\s*\}", input_str, re.DOTALL + ) + if match: + puts_str = match.group(1) + if puts_str.strip(): + matches = re.findall(r"put\(\"(.*?)\",\s*(.*?)\)", puts_str) + for match in matches: + key = match[0] + value = parse_java_value(match[1].strip()) + elements[key] = value + return elements + + match_empty = re.search(r"new\s+HashMap<.*?>\s*\(\)", input_str) + if match_empty: + return {} # Return an empty dictionary for an empty HashMap + + return input_str # default to string + + +# This method parses without the information of what each element type is, contrary of the previous +def parse_java_value(value_str: str): + # check if it's boolean + if value_str == "true": + return True + elif value_str == "false": + return False + # check if it's a string + elif value_str.startswith('"') and value_str.endswith('"'): + return value_str[1:-1] + # check if it's a long + elif re.match(r"^-?\d+[lL]$", value_str): + return int(value_str[:-1]) + # check if it's a float + elif re.match(r"^-?\d+(\.\d+)?([eE][+-]?\d+)?[fF]$", value_str): + return float(re.sub(r"[fF]$", "", value_str)) + # check if it's a integer-like and float-like types (including byte, short, integer, double, etc) + else: + try: + return int(value_str) + except ValueError: + try: + return float(value_str) + except ValueError: + # this assuming all other types are converted to string + return value_str + + +# Write tests for the `java_type_converter` function +def test_java_type_converter(): + # Test valid conversions + assert java_type_converter("true", "boolean") == True + assert java_type_converter("false", "boolean") == False + assert java_type_converter("123", "integer") == 123 + assert java_type_converter("-123", "integer") == -123 + assert java_type_converter("3.14f", "float") == 3.14 + assert java_type_converter("-3.14f", "float") == -3.14 + assert java_type_converter("3.14", "double") == 3.14 + assert java_type_converter("-3.14", "double") == -3.14 + assert java_type_converter("123L", "long") == 123 + assert java_type_converter("-123L", "long") == -123 + assert java_type_converter("a", "char") == "a" + assert java_type_converter("abc", "String") == "abc" + assert java_type_converter("new int[]{1, 2, 3}", "Array") == [1, 2, 3] + assert java_type_converter( + 'new ArrayList<>(Arrays.asList("a", "b"))', "ArrayList" + ) == ["a", "b"] + assert java_type_converter( + 'new HashMap() {{ put("key", "value"); }}', "HashMap" + ) == {"key": "value"} + assert java_type_converter("3f", "float") == 3.0 + assert java_type_converter("3e3F", "float") == 3e3 + assert java_type_converter("3e-3F", "float") == 3e-3 + assert java_type_converter("3.14e2", "double") == 3.14e2 + assert java_type_converter("3.14e-2", "double") == 3.14e-2 + assert java_type_converter("127", "byte") == 127 + assert java_type_converter("-128", "byte") == -128 + assert java_type_converter("32767", "short") == 32767 + assert java_type_converter("-32768", "short") == -32768 + assert java_type_converter("9223372036854775807L", "long") == 9223372036854775807 + assert java_type_converter("-9223372036854775808L", "long") == -9223372036854775808 + assert java_type_converter("123", "any") == "123" + assert java_type_converter("abc", "any") == "abc" + + # Test empty collections + assert java_type_converter("new int[]{}", "Array") == [] + assert java_type_converter("new ArrayList<>()", "ArrayList") == [] + assert java_type_converter("new HashMap<>()", "HashMap") == {} + + # Test collections with mixed types + assert java_type_converter('new Object[]{1, "abc", true}', "Array") == [ + 1, + "abc", + True, + ] + assert java_type_converter( + 'new ArrayList<>(Arrays.asList(1, "abc", true))', "ArrayList" + ) == [1, "abc", True] + assert java_type_converter( + 'new HashMap() {{ put("key1", 1); put("key2", "value"); put("key3", true); }}', + "HashMap", + ) == {"key1": 1, "key2": "value", "key3": True} + + # Test invalid values + try: + java_type_converter("true", "integer") + except ValueError as e: + assert str(e) == "Invalid integer value: true" + + try: + java_type_converter("abc", "integer") + except ValueError as e: + assert str(e) == "Invalid integer value: abc" + + try: + java_type_converter("abc", "long") + except ValueError as e: + assert str(e) == "Invalid long value: abc" + + try: + java_type_converter("3.14", "float") + except ValueError as e: + assert str(e) == "Invalid float value: 3.14" + + try: + java_type_converter("3.14f", "double") + except ValueError as e: + assert str(e) == "Invalid double value: 3.14f" + + try: + java_type_converter("128", "byte") + except ValueError as e: + assert str(e) == "Invalid byte value: 128" + + try: + java_type_converter("32768", "short") + except ValueError as e: + assert str(e) == "Invalid short value: 32768" + + try: + java_type_converter("invalid", "boolean") + except ValueError as e: + assert str(e) == "Invalid boolean value: invalid" + + try: + java_type_converter("abc", "char") + except ValueError as e: + assert str(e) == "Invalid char value: abc" + + # Test unsupported types + try: + java_type_converter("abc", "Set") + except NotImplementedError as e: + assert str(e) == "Set conversion is not implemented" + + try: + java_type_converter("abc", "Hashtable") + except NotImplementedError as e: + assert str(e) == "Set conversion is not implemented" + + try: + java_type_converter("abc", "Queue") + except NotImplementedError as e: + assert str(e) == "Queue conversion is not implemented" + + try: + java_type_converter("abc", "Stack") + except NotImplementedError as e: + assert str(e) == "Stack conversion is not implemented" + + # extra array testing + assert java_type_converter("new int[]{}", "Array") == [] + assert java_type_converter("new int[] {}", "Array") == [] + assert java_type_converter("new int[] { }", "Array") == [] + assert java_type_converter("new int[]{1,2,3}", "Array") == [1, 2, 3] + assert java_type_converter("new int[]{1, 2, 3}", "Array") == [1, 2, 3] + assert java_type_converter("new int[] {1, 2, 3}", "Array") == [1, 2, 3] + assert java_type_converter("new int[] { 1, 2, 3 }", "Array") == [1, 2, 3] + + # extra hashmap testing + assert java_type_converter("new HashMap<>()", "HashMap") == {} + assert java_type_converter("new HashMap<>() {}", "HashMap") == {} + assert java_type_converter("new HashMap<>() {{}}", "HashMap") == {} + assert java_type_converter("new HashMap<>() {{ }}", "HashMap") == {} + assert java_type_converter( + 'new HashMap() {{ put("key", "value"); }}', "HashMap" + ) == {"key": "value"} + assert java_type_converter( + 'new HashMap() {{put("key", "value");}}', "HashMap" + ) == {"key": "value"} + assert java_type_converter( + 'new HashMap() { { put("key", "value"); } }', "HashMap" + ) == {"key": "value"} + assert java_type_converter( + 'new HashMap() {{ put("key1", 123); put("key2", true); }}', + "HashMap", + ) == {"key1": 123, "key2": True} + assert java_type_converter( + 'new HashMap() {{ put("key1", "value 1"); put("key2", "value 2"); }}', + "HashMap", + ) == {"key1": "value 1", "key2": "value 2"} + + def test_parse_array_long(): + input_str = "new long[]{1L, 2L, 3L}" + expected_output = [1, 2, 3] + assert parse_array(input_str, nested_type="long") == expected_output + + def test_parse_array_mixed_long(): + input_str = "new long[]{1L, 2, 3L}" + expected_output = [1, "2", 3] + assert parse_array(input_str, nested_type="long") == expected_output + + def test_parse_array_invalid_long(): + input_str = "new long[]{1L, 2.0, 3L}" + expected_output = [1, "2.0", 3] + assert parse_array(input_str, nested_type="long") == expected_output + + def test_parse_arraylist_int(): + input_str = "new ArrayList(Arrays.asList(1, 2, 3))" + expected_output = [1, 2, 3] + assert parse_arraylist(input_str, nested_type="integer") == expected_output + + def test_parse_arraylist_float(): + input_str = "new ArrayList() {{ add(1.0f); add(2.0f); add(3.0f); }}" + expected_output = [1.0, 2.0, 3.0] + assert parse_arraylist(input_str, nested_type="float") == expected_output + + def test_parse_arraylist_double(): + input_str = "new ArrayList() {{ add(1.0); add(2.0); add(3.0); }}" + expected_output = [1.0, 2.0, 3.0] + assert parse_arraylist(input_str, nested_type="double") == expected_output + + def test_parse_arraylist_boolean(): + input_str = "new ArrayList(Arrays.asList(true, false, true))" + expected_output = [True, False, True] + assert parse_arraylist(input_str, nested_type="boolean") == expected_output + + def test_parse_arraylist_char(): + input_str = "new ArrayList() {{ add('a'); add('b'); add('c'); }}" + expected_output = ["a", "b", "c"] + print(parse_arraylist(input_str, nested_type="char")) + assert parse_arraylist(input_str, nested_type="char") == expected_output + + def test_parse_arraylist_string(): + input_str = 'new ArrayList() {{ add("aasdasd"); add("basdasd"); add("casdasd"); }}' + expected_output = ["aasdasd", "basdasd", "casdasd"] + print(parse_arraylist(input_str)) + assert parse_arraylist(input_str) == expected_output + + test_parse_array_long() + test_parse_array_mixed_long() + test_parse_array_invalid_long() + test_parse_arraylist_int() + test_parse_arraylist_float() + test_parse_arraylist_double() + test_parse_arraylist_boolean() + test_parse_arraylist_char() + test_parse_arraylist_string() + print("All tests passed successfully!") + + +if __name__ == "__main__": + test_java_type_converter() diff --git a/sieval/community/bfcl_v3/type_convertor/js_type_converter.py b/sieval/community/bfcl_v3/type_convertor/js_type_converter.py new file mode 100644 index 00000000..f5f36a92 --- /dev/null +++ b/sieval/community/bfcl_v3/type_convertor/js_type_converter.py @@ -0,0 +1,324 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/eval_checker/ast_eval/type_convertor/js_type_converter.py +Upstream blob: 4205b92e05af83d2e61a7bf4bd2530718b57acee +License: Apache-2.0 + +Only deviation: `from bfcl_eval.constants.type_mappings import JS_TYPE_CONVERSION` +rewritten to the absolute `sieval.community.bfcl_v3.type_mappings` path. Absolute +rather than `..type_mappings` because the import policy reserves relative imports +for the same package, and upstream's line was absolute anyway -- one rewritten +import either way. No other edit. +""" + +import re +from sieval.community.bfcl_v3.type_mappings import JS_TYPE_CONVERSION + + +def js_type_converter(value, expected_type, nested_type=None): + if expected_type not in JS_TYPE_CONVERSION: + raise ValueError(f"Unsupported type: {expected_type}") + + if expected_type == "String": + if not (value.startswith('"') and value.endswith('"')) and not ( + value.startswith("'") and value.endswith("'") + ): + return str(value) + return value[1:-1] + + elif expected_type == "integer": + if not re.match(r"^-?\d+$", value): + return str(value) # default to string + return int(value) + elif expected_type == "float": + if not re.match(r"^-?\d+(\.\d+)?$", value): + return str(value) # default to string + return float(value) + elif expected_type == "Bigint": + if not re.match(r"^-?\d+n$", value): + return str(value) # default to string + return int(value[:-1]) + elif expected_type == "Boolean": + if value not in ["true", "false"]: + return str(value) # default to string + return value == "true" + elif expected_type == "dict": + return parse_js_collection(value, "dict", nested_type) + elif expected_type == "array": + return parse_js_collection(value, "array", nested_type) + elif expected_type == "any": + return str(value) + else: + raise ValueError(f"Unsupported type: {expected_type}") + + +def parse_js_collection(code, type_str, nested_type=None): + code = code.strip() + if type_str == "array": + # Regular expression patterns + array_2d_pattern = r"\[\s*\[.*?\]\s*(,\s*\[.*?\]\s*)*\]|\bnew\s+Array\(\s*\[.*?\]\s*(,\s*\[.*?\]\s*)*\)" + array_pattern = r"\[(.*?)\]|\bnew\s+Array\((.*?)\)" + + # Check if the code is a 2D array + array_2d_match = re.match(array_2d_pattern, code) + try: + if array_2d_match: + elements_str = array_2d_match.group(0) + inner_arrays = re.findall(r"\[(.*?)\]", elements_str) + elements = [] + for idx, inner_array_str in enumerate(inner_arrays): + inner_array_str = inner_array_str.strip() + if idx == 0 and inner_array_str.startswith("["): + inner_array_str = inner_array_str[1:] + inner_array_elements = [ + e.strip() for e in inner_array_str.split(",") + ] + if nested_type: + inner_array = [parse_js_value(e) for e in inner_array_elements] + else: + inner_array = [parse_js_value(e) for e in inner_array_elements] + elements.append(inner_array) + return elements + + # Check if the code is a 1D array + array_match = re.match(array_pattern, code) + if array_match: + if array_match.group(1) is not None: + elements_str = array_match.group(1).strip() + if elements_str: + elements = elements_str.split(",") + else: + elements = [] + elif array_match.group(2) is not None: + elements_str = array_match.group(2).strip() + if elements_str: + elements = elements_str.split(",") + else: + elements = [] + else: + elements = [] + if nested_type: + elements = [ + ( + js_type_converter(e.strip(), nested_type, "String") + if (e.strip().startswith("'") or e.strip().startswith('"')) + else js_type_converter(e.strip(), nested_type) + ) + for e in elements + ] + else: + elements = [parse_js_value(e.strip()) for e in elements] + return elements + else: + return code + except: + return code + + elif type_str == "dict": + if code == "{}": + return {} # Return an empty dictionary for an empty object + dict_pattern = r"\{(.*?)\}" + # Check if the code is a dictionary + dict_match = re.match(dict_pattern, code) + if dict_match: + try: + content = dict_match.group(1) + pairs = re.findall(r"([^:]+):\s*(.*?)(?:,\s*(?=[^,]+:)|$)", content) + dictionary = {} + for key, value in pairs: + key = key.strip().strip("'\"") + value = value.strip() + if value.startswith("[") and value.endswith("]"): + # Handle array values + dictionary[key] = parse_js_collection(value, "array") + elif value.startswith("{") and value.endswith("}"): + # Handle nested dictionary values + dictionary[key] = parse_js_collection(value, "dict") + else: + dictionary[key] = parse_js_value(value.strip("'\"")) + return dictionary + except Exception as e: + print(f"Error parsing dictionary: {e}") + return code + else: + return code # default to string + else: + raise ValueError(f"Unsupported type: {type_str}") + + +def parse_js_value(value_str: str): + value_str = value_str.strip() + if value_str == "true": + return True + elif value_str == "false": + return False + elif (value_str.startswith('"') and value_str.endswith('"')) or ( + value_str.startswith("'") and value_str.endswith("'") + ): + return value_str[1:-1] + else: + try: + return int(value_str) + except ValueError: + try: + return float(value_str) + except ValueError: + return value_str + + +# Write tests for the `js_type_converter` function +def test_js_type_converter(): + assert js_type_converter("true", "Boolean") == True + assert js_type_converter("false", "Boolean") == False + assert js_type_converter("123", "integer") == 123 + assert js_type_converter("3.14", "float") == 3.14 + assert js_type_converter("123n", "Bigint") == 123 + assert js_type_converter("abc", "String") == "abc" + assert js_type_converter("[1, 2, 3]", "array") == [1, 2, 3] + assert js_type_converter("new Array(1, 2, 3)", "array") == [1, 2, 3] + assert js_type_converter("{'key': 'value'}", "dict") == {"key": "value"} + assert js_type_converter("{'key': 123}", "dict") == {"key": 123} + assert js_type_converter("{'key': true}", "dict") == {"key": True} + + # Additional test cases + # Test empty array and dictionary + assert js_type_converter("[]", "array") == [] + assert js_type_converter("{}", "dict") == {} + + # Test array with mixed types + assert js_type_converter("[1, 'two', true]", "array") == [1, "two", True] + + # Test dictionary with mixed types + assert js_type_converter( + "{'key1': 123, 'key2': 'value', 'key3': false}", "dict" + ) == {"key1": 123, "key2": "value", "key3": False} + + # Test string with special characters + + # Test negative integer and float values + assert js_type_converter("-123", "integer") == -123 + assert js_type_converter("-3.14", "float") == -3.14 + + # Test invalid type + try: + js_type_converter("123", "InvalidType") + except ValueError as e: + assert str(e) == "Unsupported type: InvalidType" + + # Test invalid integer value + try: + js_type_converter("123.45", "integer") + except ValueError as e: + assert str(e) == "Invalid integer value: 123.45" + + # Test invalid float value + try: + js_type_converter("3.14abc", "float") + except ValueError as e: + assert str(e) == "Invalid float value: 3.14abc" + + # Test invalid Bigint value + try: + js_type_converter("123", "Bigint") + except ValueError as e: + assert str(e) == "Invalid Bigint value: 123" + + # Test invalid boolean value + try: + js_type_converter("not_a_boolean", "Boolean") + except ValueError as e: + assert str(e) == "Invalid boolean value: not_a_boolean" + + print("All tests passed successfully!") + + +def test_js_type_converter_nested_array(): + # Test array with nested integers + assert js_type_converter("[1, 2, 3]", "array", "integer") == [1, 2, 3] + assert js_type_converter("new Array(4, 5, 6)", "array", "integer") == [4, 5, 6] + + # Test array with nested floats + assert js_type_converter("[1.1, 2.2, 3.3]", "array", "float") == [1.1, 2.2, 3.3] + assert js_type_converter("new Array(4.4, 5.5, 6.6)", "array", "float") == [ + 4.4, + 5.5, + 6.6, + ] + + # Test array with nested Bigints + assert js_type_converter("[1n, 2n, 3n]", "array", "Bigint") == [1, 2, 3] + assert js_type_converter("new Array(4n, 5n, 6n)", "array", "Bigint") == [4, 5, 6] + + # Test array with nested booleans + assert js_type_converter("[true, false, true]", "array", "Boolean") == [ + True, + False, + True, + ] + assert js_type_converter("new Array(false, true, false)", "array", "Boolean") == [ + False, + True, + False, + ] + + # Test array with nested strings + print(js_type_converter('["hello", "world", "!"]', "array", "String")) + assert js_type_converter('["hello", "world", "!"]', "array", "String") == [ + "hello", + "world", + "!", + ] + assert js_type_converter('new Array("foo", "bar", "baz")', "array", "String") == [ + "foo", + "bar", + "baz", + ] + + # Test array with mixed nested types + assert js_type_converter('[1, "two", true]', "array") == [1, "two", True] + assert js_type_converter('new Array(3.14, "pi", false)', "array") == [ + 3.14, + "pi", + False, + ] + + # Test array with nested arrays + print(js_type_converter(" [ [1, 2], [3, 4], [5, 6]]", "array", "array")) + assert js_type_converter(" [ [ 1, 2 ], [ 3, 4], [5, 6]]", "array", "array") == [ + [1, 2], + [3, 4], + [5, 6], + ] # this example has many weird spacings + assert js_type_converter("new Array([1, 2], [3, 4], [5, 6])", "array", "array") == [ + [1, 2], + [3, 4], + [5, 6], + ] + + # Test array with nested dictionaries + assert js_type_converter( + '[{"key1": 1}, {"key2": 2}, {"key3": 3}]', "array", "dict" + ) == [{"key1": 1}, {"key2": 2}, {"key3": 3}] + assert js_type_converter( + 'new Array({"key1": 1}, {"key2": 2}, {"key3": 3})', "array", "dict" + ) == [{"key1": 1}, {"key2": 2}, {"key3": 3}] + + print("All nested array tests passed successfully!") + + +def test_js_type_converter_dictionary_with_arrays(): + complex_dict = js_type_converter( + '{"initialState": initialStateObject, "reducers": reducersMap, "middlewares": ["loggerMiddleware"], "enhancers": ["applyMiddleware", "myMiddleWare"]}', + "dict", + ) + assert isinstance(complex_dict, dict) + assert complex_dict["initialState"] == "initialStateObject" + assert complex_dict["reducers"] == "reducersMap" + assert complex_dict["middlewares"] == ["loggerMiddleware"] + assert complex_dict["enhancers"] == ["applyMiddleware", "myMiddleWare"] + print("Complex dictionary test passed successfully!") + +if __name__ == "__main__": + test_js_type_converter() + test_js_type_converter_nested_array() + test_js_type_converter_dictionary_with_arrays() diff --git a/sieval/community/bfcl_v3/type_mappings.py b/sieval/community/bfcl_v3/type_mappings.py new file mode 100644 index 00000000..51936d2b --- /dev/null +++ b/sieval/community/bfcl_v3/type_mappings.py @@ -0,0 +1,99 @@ +"""Vendored verbatim from ShishirPatil/gorilla @ v1.3 (ea13468e). + +Upstream path: bfcl_eval/constants/type_mappings.py +Upstream blob: fdbdde48402f8f2812b146f9b7341b6aae9e55b8 +License: Apache-2.0 + +This file has no imports of its own, so there was nothing to rewrite. It is +byte-identical to upstream. +""" + +GORILLA_TO_OPENAPI = { + "integer": "integer", + "number": "number", + "float": "number", + "string": "string", + "boolean": "boolean", + "bool": "boolean", + "array": "array", + "list": "array", + "dict": "object", + "object": "object", + "tuple": "array", + "any": "string", + "byte": "integer", + "short": "integer", + "long": "integer", + "double": "number", + "char": "string", + "ArrayList": "array", + "Array": "array", + "HashMap": "object", + "Hashtable": "object", + "Queue": "array", + "Stack": "array", + "Any": "string", + "String": "string", + "Bigint": "integer", +} + +GORILLA_TO_PYTHON = { + "integer": "int", + "number": "float", + "float": "float", + "string": "str", + "boolean": "bool", + "bool": "bool", + "array": "list", + "list": "list", + "dict": "dict", + "object": "dict", + "tuple": "tuple", + "any": "str", + "byte": "int", + "short": "int", + "long": "int", + "double": "float", + "char": "str", + "ArrayList": "list", + "Array": "list", + "HashMap": "dict", + "Hashtable": "dict", + "Queue": "list", + "Stack": "list", + "Any": "str", + "String": "str", + "Bigint": "int", +} + + +JAVA_TYPE_CONVERSION = { + "byte": int, + "short": int, + "integer": int, + "float": float, + "double": float, + "long": int, + "boolean": bool, + "char": str, + "Array": list, + "ArrayList": list, + "Set": set, + "HashMap": dict, + "Hashtable": dict, + "Queue": list, # this can be `queue.Queue` as well, for simplicity we check with list + "Stack": list, + "String": str, + "any": str, +} + +JS_TYPE_CONVERSION = { + "String": str, + "integer": int, + "float": float, + "Bigint": int, + "Boolean": bool, + "dict": dict, + "array": list, + "any": str, +} diff --git a/sieval/core/models/_legacy_bridge.py b/sieval/core/models/_legacy_bridge.py index d09aba9e..e0d1dd3f 100644 --- a/sieval/core/models/_legacy_bridge.py +++ b/sieval/core/models/_legacy_bridge.py @@ -24,6 +24,7 @@ from .ir import ( CompletionInput, DialectOptions, + FunctionToolCall, ModelInput, ModelProvenance, OpaqueContinuation, @@ -89,6 +90,10 @@ class ModelOutput: texts: list[str] finish_reasons: list[str] | None = None reasoning_texts: list[str] | None = None + #: Structured calls from the native tools API. ``None`` when the reply + #: carried none -- which is not the same as an empty tuple, a reply that + #: used the tools API and deliberately called nothing. + tool_calls: tuple[FunctionToolCall, ...] | None = None logprobs_tokens: list[str] | None = None logprobs: list[float | None] | None = None top_logprobs: list[dict[str, float]] | None = None @@ -471,6 +476,7 @@ def response_to_model_output(model_meta: ModelMeta, response: Response) -> Model else None ), reasoning_texts=reasoning_texts, + tool_calls=response.tool_calls, logprobs_tokens=logprobs_tokens, logprobs=logprobs, top_logprobs=top_logprobs, diff --git a/sieval/core/models/requirements.py b/sieval/core/models/requirements.py index f7f2786a..abf81c17 100644 --- a/sieval/core/models/requirements.py +++ b/sieval/core/models/requirements.py @@ -48,6 +48,7 @@ class TaskRequirements: input_scoring: bool = False sampled_logprobs: bool = False min_top_logprobs: int | None = None + function_tools: bool = False def __post_init__(self) -> None: if self.input is not None and not isinstance(self.input, InputKind): @@ -67,6 +68,8 @@ def __post_init__(self) -> None: raise TypeError("input_scoring must be a boolean") if not isinstance(self.sampled_logprobs, bool): raise TypeError("sampled_logprobs must be a boolean") + if not isinstance(self.function_tools, bool): + raise TypeError("function_tools must be a boolean") minimum = self.min_top_logprobs if minimum is not None: diff --git a/sieval/core/tasks/task.py b/sieval/core/tasks/task.py index 0121bd90..bf99e097 100644 --- a/sieval/core/tasks/task.py +++ b/sieval/core/tasks/task.py @@ -65,6 +65,8 @@ def _required_capabilities(requires: TaskRequirements) -> frozenset[str]: capabilities.add("sampled_logprobs") if requires.min_top_logprobs is not None: capabilities.add("top_logprobs") + if requires.function_tools: + capabilities.add("function_tools") return frozenset(capabilities) diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index b5c5fb8c..a55ab7b0 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -45,6 +45,14 @@ from .arc_easy import ( ARCEasyDataset, ARCEasyDatasetSample, ) +from .bfcl_v3_live import ( + BfclV3LiveDataset, + BfclV3LiveDatasetSample, +) +from .bfcl_v3_non_live import ( + BfclV3NonLiveDataset, + BfclV3NonLiveDatasetSample, +) from .browsecomp import ( BrowseCompDataset, BrowseCompDatasetSample, @@ -275,6 +283,10 @@ __all__ = [ "ApexShortlist2025DatasetSample", "BRUMO2025Dataset", "BRUMO2025DatasetSample", + "BfclV3LiveDataset", + "BfclV3LiveDatasetSample", + "BfclV3NonLiveDataset", + "BfclV3NonLiveDatasetSample", "BrowseCompDataset", "BrowseCompDatasetSample", "CEvalDataset", diff --git a/sieval/datasets/_bfcl_v3.py b/sieval/datasets/_bfcl_v3.py new file mode 100644 index 00000000..b99a2689 --- /dev/null +++ b/sieval/datasets/_bfcl_v3.py @@ -0,0 +1,164 @@ +"""Shared loader for the BFCL v3 single-turn category files. + +One HF repo holds every category as its own JSONL file, with ground truth in a +parallel `possible_answer/` file. Three categories ship no gold -- `irrelevance`, +`live_irrelevance` and `live_relevance` are scored on whether a call was +produced, not on which call it was. + +The join key is the **universal index** -- the `` of +`_[--]` -- not the whole `id`. That is upstream's key: +its runner sorts both files on `(category, universal_index)` and pairs them +positionally, discarding the sub-indices. It is load-bearing on exactly one row +at the pinned revision, where prompt `live_multiple_1052-79-0` is graded against +gold `live_multiple_1052-279-0`: one sub-index is an upstream typo, invisible to +upstream because it never enters the key. Joining on the whole `id` would score +1052 of `live_multiple`'s 1053 rows and publish a column upstream never did. + +An index join equals upstream's positional one only while the key is unique per +joined file, so the loader asserts that instead of assuming it -- stricter than +upstream, which only checks the two files are the same length. Not asserted for +the goldless three, which are never joined: `live_relevance` ships one row twice, +18 rows over 17 distinct ids. + +Row counts are asserted per category on the gold file as well as the prompt file, +which together cover upstream's own `len(prompt) == len(possible_answer)`. The +revision pin stops a silent re-upload but not a pin bump, so a count that moves +fails here rather than quietly rescoring a leaderboard column. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json +from collections.abc import Mapping +from pathlib import Path + +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.community.bfcl_v3 import GOLDLESS_CATEGORIES, LANGUAGE_BY_CATEGORY + +#: Pinned snapshot. Verified row-for-row against the harness's own +#: `bfcl_eval/data/` at gorilla v1.3. +BFCL_V3_REVISION = "61fc0608cfd831fcfbbaa676ebdfef0ed963eeda" + + +def _read_jsonl(path: Path) -> list[dict]: + with path.open(encoding="utf-8") as handle: + return [json.loads(line) for line in handle if line.strip()] + + +def _universal_index(row_id: str, category: str, what: str) -> int: + """The join key of one row: the index in `_[--]`. + + Mirrors the index that upstream's own sort key extracts -- it splits on the + last underscore and keeps only what precedes the first dash, so the two + sub-indices the live categories carry are discarded here exactly as they + are there. + """ + index = row_id.rsplit("_", 1)[-1].split("-")[0] + try: + return int(index) + except ValueError: + raise ValueError( + f"BFCL v3 {what} row {row_id!r} in category {category!r} has " + f"{index!r} where its universal index should be. The index is the " + "join key upstream sorts both files on, so an id whose shape moved " + "cannot be joined at all; re-verify the category against upstream." + ) from None + + +def _index_rows(rows: list[dict], category: str, what: str) -> dict[int, dict]: + by_index: dict[int, dict] = {} + collisions: dict[int, list[str]] = {} + for row in rows: + index = _universal_index(row["id"], category, what) + if index in by_index: + collisions.setdefault(index, [by_index[index]["id"]]).append(row["id"]) + else: + by_index[index] = row + + if collisions: + # The ids, not just a count: on a pin bump this is the difference + # between a one-minute diagnosis and a manual scan of 1053 rows. + examples = {index: collisions[index] for index in sorted(collisions)[:3]} + plural = "index" if len(collisions) == 1 else "indices" + raise ValueError( + f"BFCL v3 category {category!r} has {len(collisions)} universal " + f"{plural} shared by more than one {what} row (e.g. {examples}). " + "The index is the join key and upstream's runner pairs the two " + "files positionally under it, so a repeat makes the pairing " + "ambiguous rather than merely odd." + ) + return by_index + + +def load_categories(name_or_path: str, counts: Mapping[str, int]) -> HFDatasetDict: + root = Path(name_or_path) + rows: list[dict] = [] + for category, expected in counts.items(): + prompts = _read_jsonl(root / f"BFCL_v3_{category}.json") + if len(prompts) != expected: + raise ValueError( + f"BFCL v3 category {category!r} has {len(prompts)} rows at the " + f"pinned revision, expected {expected}. A count that moved means " + "the pin was bumped; re-verify the category against upstream and " + "update the table deliberately, do not relax this check." + ) + + gold_by_index: dict[int, str] = {} + if category not in GOLDLESS_CATEGORIES: + # Only a category that is actually joined needs a unique key. The + # goldless three are not joined, and one of them must not be held to + # this: `live_relevance` ships `live_relevance_3-3-0` twice, so it + # has 18 rows and 17 distinct ids. Upstream scores all 18 and so do + # we -- deduplicating here would publish a denominator upstream + # never used. + prompt_by_index = _index_rows(prompts, category, "prompt") + gold = _read_jsonl(root / "possible_answer" / f"BFCL_v3_{category}.json") + if len(gold) != expected: + raise ValueError( + f"BFCL v3 category {category!r} has {len(gold)} " + f"possible_answer rows at the pinned revision, expected " + f"{expected}. Upstream asserts the prompt and gold files are " + "the same length, so a gold file that moved on its own would " + "rescore the column without a single prompt changing." + ) + answers = _index_rows(gold, category, "possible_answer") + gold_by_index = { + index: json.dumps(entry["ground_truth"], ensure_ascii=False) + for index, entry in answers.items() + } + missing = set(prompt_by_index) - set(gold_by_index) + if missing: + examples = [prompt_by_index[i]["id"] for i in sorted(missing)[:3]] + raise ValueError( + f"BFCL v3 category {category!r} has {len(missing)} prompt rows " + f"with no possible_answer entry (e.g. {examples}). The join key " + "is the universal index; a miss means the two files disagree." + ) + + for row in prompts: + turns = row["question"] + # Every phase-1 category is single-turn. Asserting beats taking [0]: + # a multi-turn file loaded here would silently score only its first + # turn and look like a very poor model. + if len(turns) != 1: + raise ValueError( + f"BFCL v3 row {row['id']!r} in {category!r} has {len(turns)} " + "turns; the single-turn datasets accept exactly one. " + "Multi-turn categories are phase 2 and must not be loaded here." + ) + rows.append( + { + "id": row["id"], + "category": category, + "language": LANGUAGE_BY_CATEGORY.get(category, "Python"), + "question": turns[0], + "function": json.dumps(row["function"], ensure_ascii=False), + "ground_truth": gold_by_index.get( + _universal_index(row["id"], category, "prompt") + ), + } + ) + + return HFDatasetDict({"test": HFDataset.from_list(rows)}) diff --git a/sieval/datasets/bfcl_v3_live.py b/sieval/datasets/bfcl_v3_live.py new file mode 100644 index 00000000..368d75f5 --- /dev/null +++ b/sieval/datasets/bfcl_v3_live.py @@ -0,0 +1,66 @@ +"""BFCL v3 live categories -- the user-contributed half of the benchmark. + +Six categories, 2251 rows: four AST categories built from real user queries, +plus irrelevance and relevance detection sets with no gold. + +`function` and `ground_truth` are json strings because pyarrow cannot type +their heterogeneous nesting -- the same reason `t_eval.py` stores its +`ground_truth` that way. + +References: + +* Blog: +* Harness: +* Dataset: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict + +from sieval.community.bfcl_v3 import LIVE_COUNTS +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) + +from ._bfcl_v3 import BFCL_V3_REVISION, load_categories + + +class BfclV3LiveDatasetSample(TypedDict): + """One BFCL v3 live row. + + `ground_truth` is None for `live_irrelevance` and `live_relevance`, which + have no gold: the correct outcome there is about whether a call is + produced, not about which call it is. + """ + + id: str + category: str + language: str + question: list[dict] + function: str + ground_truth: str | None + + +@sieval_dataset( + name="bfcl_v3_live", + display_name="BFCL v3 (live)", + description=( + "Berkeley Function Calling Leaderboard v3 live: " + "user-contributed AST, irrelevance, relevance." + ), + source=f"hf:gorilla-llm/Berkeley-Function-Calling-Leaderboard@{BFCL_V3_REVISION}", + categories=(Category(Level1Category.AGENT, "ToolUseSimple"),), + tags=("english", "function-calling"), + license="Apache-2.0", +) +class BfclV3LiveDataset(Dataset[BfclV3LiveDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + _ = kwargs + return load_categories(name_or_path, LIVE_COUNTS) diff --git a/sieval/datasets/bfcl_v3_non_live.py b/sieval/datasets/bfcl_v3_non_live.py new file mode 100644 index 00000000..92dbeb6d --- /dev/null +++ b/sieval/datasets/bfcl_v3_non_live.py @@ -0,0 +1,65 @@ +"""BFCL v3 non-live categories -- the hand-written half of the benchmark. + +Seven categories, 1390 rows: four Python AST categories, Java and JavaScript +simple-call categories, and an irrelevance-detection set with no gold. + +`function` and `ground_truth` are json strings because pyarrow cannot type +their heterogeneous nesting -- the same reason `t_eval.py` stores its +`ground_truth` that way. + +References: + +* Blog: +* Harness: +* Dataset: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict + +from sieval.community.bfcl_v3 import NON_LIVE_COUNTS +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) + +from ._bfcl_v3 import BFCL_V3_REVISION, load_categories + + +class BfclV3NonLiveDatasetSample(TypedDict): + """One BFCL v3 non-live row. + + `ground_truth` is None for `irrelevance`, which has no gold: the correct + outcome there is that no call is produced at all. + """ + + id: str + category: str + language: str + question: list[dict] + function: str + ground_truth: str | None + + +@sieval_dataset( + name="bfcl_v3_non_live", + display_name="BFCL v3 (non-live)", + description=( + "Berkeley Function Calling Leaderboard v3 non-live: " + "Python/Java/JS AST plus irrelevance." + ), + source=f"hf:gorilla-llm/Berkeley-Function-Calling-Leaderboard@{BFCL_V3_REVISION}", + categories=(Category(Level1Category.AGENT, "ToolUseSimple"),), + tags=("english", "function-calling"), + license="Apache-2.0", +) +class BfclV3NonLiveDataset(Dataset[BfclV3NonLiveDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + _ = kwargs + return load_categories(name_or_path, NON_LIVE_COUNTS) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 63ea41e8..99133737 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -284,6 +284,48 @@ "license": "cc-by-sa-4.0", "checksums": {} }, + { + "name": "bfcl_v3_live", + "display_name": "BFCL v3 (live)", + "description": "Berkeley Function Calling Leaderboard v3 live: user-contributed AST, irrelevance, relevance.", + "source": [ + "hf:gorilla-llm/Berkeley-Function-Calling-Leaderboard@61fc0608cfd831fcfbbaa676ebdfef0ed963eeda" + ], + "categories": [ + { + "level1": "Agent", + "level2": "ToolUseSimple" + } + ], + "tags": [ + "english", + "function-calling" + ], + "deps_group": null, + "license": "Apache-2.0", + "checksums": {} + }, + { + "name": "bfcl_v3_non_live", + "display_name": "BFCL v3 (non-live)", + "description": "Berkeley Function Calling Leaderboard v3 non-live: Python/Java/JS AST plus irrelevance.", + "source": [ + "hf:gorilla-llm/Berkeley-Function-Calling-Leaderboard@61fc0608cfd831fcfbbaa676ebdfef0ed963eeda" + ], + "categories": [ + { + "level1": "Agent", + "level2": "ToolUseSimple" + } + ], + "tags": [ + "english", + "function-calling" + ], + "deps_group": null, + "license": "Apache-2.0", + "checksums": {} + }, { "name": "browsecomp", "display_name": "BrowseComp", @@ -1736,6 +1778,90 @@ "status": "experimental", "reference_kind": "value" }, + { + "name": "bfcl_v3_live_0shot_gen", + "display_name": "BFCL v3 Live (0-shot, prompting)", + "description": "Berkeley Function Calling Leaderboard v3, live categories, upstream's prompting protocol.", + "dataset": "bfcl_v3_live", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "function-calling" + ], + "deps_group": "bfcl-v3", + "model_type": "chat", + "reference_impl": { + "source": "gorilla/berkeley-function-call-leaderboard", + "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. Replay of upstream's gpt-4.1-2025-04-14 (Prompt) rollouts agrees on 2251/2251 rows and reproduces every published cell: live_simple 85.66, live_multiple 76.54, live_parallel 93.75, live_parallel_multiple 75.00, live_irrelevance 77.89, live_relevance 88.89; ast_summary 78.46, Live Overall Acc 78.32. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every verdict compared against upstream's own per-row score file, which pins the grader against real output -- wrong answers, decode failures and the FC name rewrite included. It does not cover prompt construction: the replies are upstream's, so that rests on the two model-facing helpers being byte-identical to upstream rather than on a measurement." + }, + "status": "stable", + "reference_kind": "value" + }, + { + "name": "bfcl_v3_live_0shot_gen_fc", + "display_name": "BFCL v3 Live (0-shot, FC)", + "description": "Berkeley Function Calling Leaderboard v3, live categories, upstream's FC (tools API) protocol.", + "dataset": "bfcl_v3_live", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "function-calling" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "gorilla/berkeley-function-call-leaderboard", + "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Live Overall Acc`: the sample-count-weighted mean of six categories, which is algebraically the pooled rate over all 2251 live rows, so it carries a problem-clustered interval; `ast_summary` (weighted over the four AST categories) carries one on `n_ast` for the same reason. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected at task construction, which prelaunch reconciliation reaches before the first request and before a result directory exists. Replay of upstream's gpt-4.1-2025-04-14-FC rollouts agrees on 2251/2251 rows and reproduces every published cell: live_simple 80.23, live_multiple 78.35, live_parallel 68.75, live_parallel_multiple 66.67, live_irrelevance 82.31, live_relevance 77.78; ast_summary 78.39, Live Overall Acc 79.92. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every verdict compared against upstream's own per-row score file, which pins the grader against real output -- wrong answers, decode failures and the FC name rewrite included. It does not cover prompt construction: the replies are upstream's, so that rests on the two model-facing helpers being byte-identical to upstream rather than on a measurement." + }, + "status": "stable", + "reference_kind": "value" + }, + { + "name": "bfcl_v3_non_live_0shot_gen", + "display_name": "BFCL v3 Non-Live (0-shot, prompting)", + "description": "Berkeley Function Calling Leaderboard v3, non-live categories, upstream's prompting protocol.", + "dataset": "bfcl_v3_non_live", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "function-calling" + ], + "deps_group": "bfcl-v3", + "model_type": "chat", + "reference_impl": { + "source": "gorilla/berkeley-function-call-leaderboard", + "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", + "notes": "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. No interval on the headline, simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate (javascript's 50 rows weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=False: schemas reach the model verbatim, so dotted function names survive and gold needs no rewrite. Replay of upstream's gpt-4.1-2025-04-14 (Prompt) rollouts agrees on 1390/1390 rows and reproduces every published cell: simple 95.50, multiple 94.00, parallel 93.00, parallel_multiple 87.50, java 64.00, javascript 82.00, irrelevance 88.75; simple_ast 80.50, Non_Live Overall Acc 88.75. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every verdict compared against upstream's own per-row score file, which pins the grader against real output -- wrong answers, decode failures and the FC name rewrite included. It does not cover prompt construction: the replies are upstream's, so that rests on the two model-facing helpers being byte-identical to upstream rather than on a measurement." + }, + "status": "stable", + "reference_kind": "value" + }, + { + "name": "bfcl_v3_non_live_0shot_gen_fc", + "display_name": "BFCL v3 Non-Live (0-shot, FC)", + "description": "Berkeley Function Calling Leaderboard v3, non-live categories, upstream's FC (tools API) protocol.", + "dataset": "bfcl_v3_non_live", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "function-calling" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "gorilla/berkeley-function-call-leaderboard", + "url": "https://github.com/ShishirPatil/gorilla/tree/ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard", + "notes": "FC protocol -- upstream's `(FC)` leaderboard column. Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast (itself the unweighted mean of simple/java/javascript), multiple, parallel, parallel_multiple and irrelevance. No interval on the headline, simple_ast or ast_summary: an unweighted mean of category rates is not the pooled rate (javascript's 50 rows weigh as much as simple's 400), so a sample-clustered CI would bracket a different statistic. Every per-category rate carries its own. underscore_to_dot=True: convert_to_tool rewrote `.` to `_` in every function name on the way out, because a native tool name cannot carry a dot, so gold is rewritten to match before comparison. Requires the `function_tools` capability; a binding that does not offer it is rejected at task construction, which prelaunch reconciliation reaches before the first request and before a result directory exists. Replay of upstream's gpt-4.1-2025-04-14-FC rollouts agrees on 1390/1390 rows and reproduces every published cell: simple 93.50, multiple 90.50, parallel 91.00, parallel_multiple 86.00, java 61.00, javascript 68.00, irrelevance 89.58; simple_ast 74.17, Non_Live Overall Acc 86.25. AST checker, source parsers, type converters, prompt and aggregation vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for `bfcl_eval.*` import rewrites and one execution-safety deviation -- the Prompt decoder resolves an argument's arithmetic by walking the AST rather than by `eval`-ing model output, which computes the same value for every expression that does not execute something (the FC protocol parses no source text, so it never reaches that code). exec_*, rest, sql and chatable are excluded -- upstream has them commented out of its own TEST_FILE_MAPPING and they need live API keys. Anchored on upstream's released evaluation archive (HuanzhiMao/BFCL-Result, 2025-06-14): its recorded gpt-4.1-2025-04-14 replies are replayed through this grader and every verdict compared against upstream's own per-row score file, which pins the grader against real output -- wrong answers, decode failures and the FC name rewrite included. It does not cover prompt construction: the replies are upstream's, so that rests on the two model-facing helpers being byte-identical to upstream rather than on a measurement." + }, + "status": "stable", + "reference_kind": "value" + }, { "name": "browsecomp_0shot_gen", "display_name": "BrowseComp (0-shot, generative)", diff --git a/sieval/tasks/CLAUDE.md b/sieval/tasks/CLAUDE.md index b2fc40d5..2b58d6e7 100644 --- a/sieval/tasks/CLAUDE.md +++ b/sieval/tasks/CLAUDE.md @@ -25,6 +25,7 @@ name, so it is the only place the distinction can live. | *(none)* | Tracks upstream — its protocol, its grader, its defects | | `_fixed` | Ours, diverging to repair a defect in upstream's grader or data | | `_parse` | Upstream's *other* published protocol over the same benchmark and grader: same data, same scoring, a different prompt and an output extractor, each with its own published column | +| `_fc` | Upstream's native-function-calling protocol over the same benchmark and grader: same data, same scoring, the function schemas delivered through the provider's tools API instead of the prompt, and calls read back from `tool_calls` instead of parsed out of the reply | - **The unqualified name always tracks upstream, bugs included**, and is never repurposed by a local change. It stays free even if nothing will occupy it — @@ -43,6 +44,15 @@ name, so it is the only place the distinction can live. 32%, so one number for "NL2SH accuracy" would hide whether a model was scored on translation or on formatting compliance. Not for a prompt *we* prefer, and not for an extractor we added: the second column has to be upstream's. +- **`_fc` is licensed by a capability, not a prompt.** It exists where upstream + publishes a native-function-calling column beside its prompting one, and it is + a different axis from `_parse`: that one layers a second extractor over the + same model interface, while `_fc` changes *which capability is exercised* — it + gates on `function_tools`, so it changes which models can run the task at all. + The unqualified name goes to the prompting reading, since every model on such a + leaderboard has that row while only some have an FC row, and a default name + half the fleet cannot run is the narrow one. + `bfcl_v3_non_live_0shot_gen` / `_fc` earned the row. - The mode is read positionally, so a variant may not spell one: `foo_0shot_clp_gen.py` has two readings and is rejected. - The table is the current vocabulary, not the limit — a new variant earns a row diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index dc8656f4..115f9da7 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -40,6 +40,18 @@ from .arc.arc_easy_kshot_clp import ( from .arc.arc_easy_kshot_ppl import ( ARCEasyFewShotPplTask, ) +from .bfcl_v3.bfcl_v3_live_0shot_gen import ( + BfclV3LiveZeroShotGenTask, +) +from .bfcl_v3.bfcl_v3_live_0shot_gen_fc import ( + BfclV3LiveZeroShotGenFCTask, +) +from .bfcl_v3.bfcl_v3_non_live_0shot_gen import ( + BfclV3NonLiveZeroShotGenTask, +) +from .bfcl_v3.bfcl_v3_non_live_0shot_gen_fc import ( + BfclV3NonLiveZeroShotGenFCTask, +) from .browsecomp_0shot_gen import ( BrowseCompZeroShotGenTask, ) @@ -257,6 +269,10 @@ __all__ = [ "Apex2025ZeroShotGenTask", "ApexShortlist2025ZeroShotGenTask", "BRUMO2025ZeroShotGenTask", + "BfclV3LiveZeroShotGenFCTask", + "BfclV3LiveZeroShotGenTask", + "BfclV3NonLiveZeroShotGenFCTask", + "BfclV3NonLiveZeroShotGenTask", "BrowseCompZeroShotGenTask", "CEvalFewShotCLPTask", "CMIMC2025ZeroShotGenTask", diff --git a/sieval/tasks/bfcl_v3/__init__.py b/sieval/tasks/bfcl_v3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sieval/tasks/bfcl_v3/_base.py b/sieval/tasks/bfcl_v3/_base.py new file mode 100644 index 00000000..54d72a01 --- /dev/null +++ b/sieval/tasks/bfcl_v3/_base.py @@ -0,0 +1,549 @@ +"""Shared stages for the BFCL v3 single-turn task family. + +Four leaves come out of two independent choices, and this module holds what they +share. + +**Group** (``non_live`` / ``live``) selects the dataset and the rollup, so it is +the generic base. + +**Protocol** (``Prompt`` / ``FC``) is upstream's own pair of published columns. +The two mixins differ in exactly two places -- how the schemas reach the model +(a rendered system turn, or native tools) and how calls are read back (parsed +out of the reply, or off structured tool calls) -- which is why the protocol is +a stateless mixin rather than a second class hierarchy. + +**One divergence follows from the protocol, and it is upstream's.** A native +tool name cannot carry a dot in the OpenAI dialects, so upstream rewrites +``geometry.triangle_area`` to ``geometry_triangle_area`` on the way out and +reverses the rewrite before comparing against gold. That reversal is the +``underscore_to_dot`` flag -- true for FC, false for Prompt -- and +:func:`grade_single_turn` says why it travels as an argument. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json +from collections import defaultdict +from collections.abc import Callable, Mapping +from typing import ClassVar + +from sieval.community.bfcl_v3 import ( + CALL_EXPECTED, + GOLDLESS_CATEGORIES, + LIVE_COUNTS, + NON_LIVE_COUNTS, + ast_checker, + calculate_unweighted_accuracy, + calculate_weighted_accuracy, + convert_to_tool, + func_doc_language_specific_pre_processing, + is_empty_output, + is_function_calling_format_output, + set_underscore_to_dot, + system_prompt_pre_processing_chat_model, +) +from sieval.community.bfcl_v3.tool_convert import ModelStyle +from sieval.community.bfcl_v3.type_mappings import GORILLA_TO_OPENAPI +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + InputKind, + JudgementRecord, + PredictionRecord, + PromptRecord, + Task, + TaskRequirements, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, +) +from sieval.core.tasks.metrics import ( + DENOMINATOR_FIELD, + DENOMINATOR_REQUESTED, + SCORE_KEY_FIELD, + interval_metrics, + merge_metrics, + metric_interval, +) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound + +# `CALL_EXPECTED` is upstream's opposite polarity: `live_relevance` is correct +# when a call IS produced, the two irrelevance sets when one is NOT. + + +def grade_single_turn( + functions_json: str, + decoded: list | None, + gold_json: str | None, + language: str, + category: str, + underscore_to_dot: bool, +) -> bool: + """Grade one decoded reply. Module-level so `run_cpu_bound` can pickle it. + + `underscore_to_dot` is an ARGUMENT, not a module global set by the caller, + so that the write below and the read inside `ast_checker` cannot be + separated: this function has no `await` in it, and an async caller setting + the global before awaiting would. Cross-wiring the two protocols is silent + -- all 734 gradeable rows whose gold name carries a dot grade wrong, with + nothing reported -- so the atomicity is the whole reason grading goes + through one module-level function rather than a method. + + `decoded` is None when the reply could not be decoded into calls at all. + """ + set_underscore_to_dot(underscore_to_dot) + + produced_call = ( + decoded is not None + and is_function_calling_format_output(decoded) + and not is_empty_output(decoded) + ) + + if category in GOLDLESS_CATEGORIES: + return produced_call if category in CALL_EXPECTED else not produced_call + + if not produced_call: + # `ast_decoder:decoder_failed` / `:decoder_wrong_output_format` -- a + # scored outcome, not an error. + return False + + if gold_json is None: + raise ValueError( + f"BFCL v3 category {category!r} is not goldless but row gold is None; " + "a value-reference task must not grade without its reference." + ) + + result = ast_checker( + json.loads(functions_json), + decoded, + json.loads(gold_json), + language, + category, + "sieval", + ) + return bool(result["valid"]) + + +#: Upstream's harness at the pinned revision, cited by every leaf's +#: `reference_impl`. +BFCL_V3_HARNESS_URL = ( + "https://github.com/ShishirPatil/gorilla/tree/" + "ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard" +) + +#: What every `reference_impl.notes` for this benchmark has to say, whichever +#: group and protocol it is. The specific half is prepended by each leaf. +BFCL_V3_SHARED_NOTES = ( + "AST checker, source parsers, type converters, prompt and aggregation " + "vendored from gorilla v1.3 (Apache-2.0); bodies unchanged except for " + "`bfcl_eval.*` import rewrites and one execution-safety deviation -- the " + "Prompt decoder resolves an argument's arithmetic by walking the AST " + "rather than by `eval`-ing model output, which computes the same value " + "for every expression that does not execute something (the FC protocol " + "parses no source text, so it never reaches that code). " + "exec_*, rest, sql and chatable are excluded -- upstream has them commented " + "out of its own TEST_FILE_MAPPING and they need live API keys. " + "Anchored on upstream's released evaluation archive (HuanzhiMao/BFCL-Result, " + "2025-06-14): its recorded gpt-4.1-2025-04-14 replies are replayed through " + "this grader and every verdict compared against upstream's own per-row score " + "file, which pins the grader against real output -- wrong answers, decode " + "failures and the FC name rewrite included. It does not cover prompt " + "construction: the replies are upstream's, so that rests on the two " + "model-facing helpers being byte-identical to upstream rather than on a " + "measurement." +) + + +class BfclV3Task[TSample]( + Task[ + TSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + # `str`: the report carries `score_key` and `denominator_policy`, which + # name things rather than measure them. `list[float]` is an interval and + # `dict[str, str]` the map saying which population each is clustered on. + dict[str, float | int | str | list[float] | dict[str, str]], + ] +): + """Every stage but the protocol hooks and the group-level rollup.""" + + #: Set by the protocol mixin. + UNDERSCORE_TO_DOT: ClassVar[bool] + #: Set by the group base. Maps every category of this group to its declared + #: row count, which is also its scoring DENOMINATOR. + CATEGORY_COUNTS: ClassVar[dict[str, int]] + #: Set by the group base. `"non_live"` / `"live"` -- names the headline key + #: (`f"{GROUP_KEY}_overall_acc"`) and so also `score_key`. + GROUP_KEY: ClassVar[str] + + def _build_messages(self, raw) -> tuple[list, list[dict] | None]: + """Return (messages, tools). Implemented by the protocol mixin. + + The messages list is bare: a record's `prompt` is `JSONValue`, and a + `list[dict]` is not assignable to it. + """ + raise NotImplementedError + + def _decode(self, output: ModelOutput, language: str) -> list | None: + """Return decoded calls, or None when the reply yields none. + + `language` is the row's language -- "Python", "Java" or "JavaScript". + The Prompt protocol passes it straight to `ast_parse`, which dispatches + on it. The FC protocol ignores it: it reads structured tool calls and + never parses source text. + """ + raise NotImplementedError + + def _aggregate(self, cell: Callable[[str], dict]) -> dict: + """Group-level rollup. Implemented by the group base.""" + raise NotImplementedError + + async def preprocess(self, raw, ctx) -> PromptRecord: + messages, tools = self._build_messages(raw) + return build_prompt_record( + messages, + reference=raw["ground_truth"], + extra={ + "id": raw["id"], + "category": raw["category"], + "language": raw["language"], + "function": raw["function"], + "tools": tools, + }, + ) + + async def infer(self, pre, ctx) -> ModelOutput: + tools = pre["extra"]["tools"] + kwargs = {"tools": tools} if tools else {} + return await self.model.agenerate(pre["prompt"], **kwargs) + + async def postprocess(self, inf, ctx) -> PredictionRecord: + # The row's language selects the PARSER, so it has to reach `_decode`. + # It is carried on the preprocess record rather than re-derived here. + language = ctx.preprocess_result["extra"]["language"] + try: + decoded = self._decode(inf, language) + except ImportError: + # NOT a decode failure: `_decode` defers its import of the vendored + # parser, which lives behind the optional `bfcl-v3` extra. Scoring a + # missing tree-sitter as an undecodable reply would grade every row + # of the run wrong with `fails` still 0; propagating costs the one + # sample and names the cause. + raise + except Exception as exc: # noqa: BLE001 -- a decode failure is a SCORE + # Upstream's `ast_decoder:decoder_failed`: the model produced + # something undecodable, which is the model's outcome, not a fault. + return build_prediction_record([None], extras=[{"decode_error": str(exc)}]) + return build_prediction_record([decoded]) + + async def feedback(self, post, ctx) -> tuple[bool, JudgementRecord]: + pre_extra = ctx.preprocess_result["extra"] + category = pre_extra["category"] + # `.get()`, not `[]`: a None prediction is dropped by serialization, so + # on a resume the key is ABSENT for exactly the rows that failed to + # decode -- the ones this task most needs to score. + decoded = post["rollouts"][0].get("prediction") + gold = ctx.preprocess_result.get("reference") + + try: + correct = await run_cpu_bound( + grade_single_turn, + pre_extra["function"], + decoded, + gold, + pre_extra["language"], + category, + self.UNDERSCORE_TO_DOT, + timeout=GRADE_TIMEOUT, + ) + except TimeoutError: + # A grade that could not be computed in time is a wrong answer -- + # the prediction is a shape the grader cannot bound. Every OTHER + # exception propagates: a broken grader must not read as a wrong + # model. + correct = False + + # A goldless category's reference is the expected OUTCOME, not None: + # `reference_kind="value"` is inferred from whether the call sites pass + # a literal None, and these rows do have something to compare against. + reference = ( + gold + if gold is not None + else ("call" if category in CALL_EXPECTED else "no_call") + ) + return True, build_judgement_record( + reference, + [build_rollout_judgement(0, correct)], + extra={"category": category, "id": pre_extra["id"]}, + ) + + async def report(self, finals: list, fails: list) -> dict: + # One 0/1 per sample that came back, NOT one percentage point: the same + # list feeds the cells and the interval estimators, and those read + # `sum(values) / denominator` as a probability -- hand them percent and + # any rate above 1 point reads as saturated. Scaling happens once, in + # `cell`, so the rate and its bounds agree. + correct_by_category: dict[str, list[float]] = defaultdict(list) + for ctx in finals: + fb = ctx.feedback_result + correct_by_category[fb["extra"]["category"]].append( + 1.0 if fb["rollouts"][0]["correct"] else 0.0 + ) + + def cell(category: str) -> dict: + """Upstream's accuracy-dict shape, over the DECLARED denominator. + + `total_count` is the category's row count, not the number that came + back: `DENOMINATOR_REQUESTED` charges a missing sample as wrong, and + a sample that failed before `preprocess` has no category anyway. + + All three of upstream's keys: both aggregation helpers read + `display_accuracy` unconditionally, and they return the same three, + which is what lets `simple_ast` nest back in as a cell. Never "N/A" + -- that marks a category upstream did not evaluate. + """ + values = correct_by_category.get(category, []) + denominator = self.CATEGORY_COUNTS[category] + accuracy = 100.0 * sum(values) / denominator # percentage points + return { + "accuracy": accuracy, + "total_count": denominator, + "display_accuracy": accuracy, + } + + rollup = self._aggregate(cell) + + result: dict = { + "score": rollup[f"{self.GROUP_KEY}_overall_acc"], + "fails": len(fails), + SCORE_KEY_FIELD: f"{self.GROUP_KEY}_overall_acc", + DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, + } + result |= rollup + for category in self.CATEGORY_COUNTS: + result[f"n_{category}"] = self.CATEGORY_COUNTS[category] + + # Per-category intervals: each IS `sum(values) / denominator`, so each + # is a candidate. No `group_keys`: one row is one problem is one + # rollout, so there are no repeated copies to collapse. + fragments = [ + metric_interval( + category, + correct_by_category.get(category, []), + denominator=self.CATEGORY_COUNTS[category], + unit=f"n_{category}", + ) + for category in self.CATEGORY_COUNTS + ] + fragments.extend(self._group_intervals(correct_by_category)) + return result | merge_metrics(*fragments) + + def _group_intervals( + self, correct_by_category: Mapping[str, list[float]] + ) -> list[dict]: + """Intervals for the group-level rollups, where any are defensible.""" + del correct_by_category + return [] + + +class BfclV3NonLiveTask[TSample](BfclV3Task[TSample]): + """Non-live: seven categories, rolled up by UNWEIGHTED means. + + `simple_ast` nests three categories inside the five-way overall, so the + headline is a mean of means -- not the pooled rate over 1390 rows, and not + convergent to it: `javascript` (50 rows) enters with the same weight as + `simple` (400). That is why it publishes no interval. + """ + + GROUP_KEY: ClassVar[str] = "non_live" + CATEGORY_COUNTS: ClassVar[dict[str, int]] = dict(NON_LIVE_COUNTS) + + def _aggregate(self, cell: Callable[[str], dict]) -> dict: + simple_ast = calculate_unweighted_accuracy( + [cell("simple"), cell("java"), cell("javascript")] + ) + ast_cells = [ + simple_ast, + cell("multiple"), + cell("parallel"), + cell("parallel_multiple"), + ] + ast_summary = calculate_unweighted_accuracy(ast_cells) + overall = calculate_unweighted_accuracy([*ast_cells, cell("irrelevance")]) + return { + "non_live_overall_acc": overall["accuracy"], + "ast_summary": ast_summary["accuracy"], + "simple_ast": simple_ast["accuracy"], + **{ + category: cell(category)["accuracy"] + for category in self.CATEGORY_COUNTS + }, + } + + +class BfclV3LiveTask[TSample](BfclV3Task[TSample]): + """Live: six categories, rolled up by SAMPLE-COUNT-WEIGHTED means. + + A weighted mean over these cells is algebraically the pooled rate over the + union of their rows, so both rollups are genuine per-sample rates and both + carry intervals. + """ + + GROUP_KEY: ClassVar[str] = "live" + CATEGORY_COUNTS: ClassVar[dict[str, int]] = dict(LIVE_COUNTS) + + #: The four AST categories, in upstream's column order. `ast_summary` is + #: over these only -- irrelevance and relevance join at the overall. + AST_CATEGORIES: ClassVar[tuple[str, ...]] = ( + "live_simple", + "live_multiple", + "live_parallel", + "live_parallel_multiple", + ) + + def _aggregate(self, cell: Callable[[str], dict]) -> dict: + ast_summary = calculate_weighted_accuracy( + [cell(category) for category in self.AST_CATEGORIES] + ) + overall = calculate_weighted_accuracy( + [cell(category) for category in self.CATEGORY_COUNTS] + ) + return { + "live_overall_acc": overall["accuracy"], + "ast_summary": ast_summary["accuracy"], + **{ + category: cell(category)["accuracy"] + for category in self.CATEGORY_COUNTS + }, + } + + def _group_intervals( + self, correct_by_category: Mapping[str, list[float]] + ) -> list[dict]: + ast_values = [ + value + for category in self.AST_CATEGORIES + for value in correct_by_category.get(category, []) + ] + all_values = [ + value + for category in self.CATEGORY_COUNTS + for value in correct_by_category.get(category, []) + ] + return [ + # `score` and `live_overall_acc` are ONE number, so the alias rides + # along on this call. Emits `n_problems` itself. + interval_metrics( + all_values, + denominator=sum(self.CATEGORY_COUNTS.values()), + aliases=("live_overall_acc",), + ), + # Emits `n_ast` itself, alongside the interval it is the unit for. + metric_interval( + "ast_summary", + ast_values, + denominator=sum( + self.CATEGORY_COUNTS[category] for category in self.AST_CATEGORIES + ), + unit="n_ast", + ), + ] + + +def _call_arguments(call) -> dict: + """Read one tool call's arguments as a dict. + + `FunctionToolCall.arguments` is typed `JSONValue`, not `str`. The OpenAI + dialects put the provider's wire JSON string there and substitute `""` when + the provider omits it entirely, so a bare `json.loads` raises on a + no-argument call -- a real shape for a zero-parameter tool. + """ + arguments = call.arguments + if isinstance(arguments, str): + return json.loads(arguments) if arguments.strip() else {} + return dict(arguments) if arguments else {} + + +def _model_facing_functions(raw) -> list: + """The function schemas as upstream shows them to the model. + + Both protocols run this, and they must run the *same* one: it is what makes + the Prompt and FC columns two readings of one benchmark rather than two + benchmarks. It rewrites Java and JavaScript parameter types to ``string`` + and appends a language hint to every description, so a model asked for a + `java` row is told it is reading Java 8 -- without it, the 150 Java and + JavaScript rows are posed in a language the schema never names. + + It does not touch grading. Upstream's `ast_file_runner` reads `function` + back out of the dataset and hands it to `ast_checker` unprocessed, so the + checker still compares against the real declared types -- which is why the + preprocessed list is built here, per protocol, instead of replacing what + `preprocess` stores for `feedback`. + + `json.loads` is not incidental: the vendored helper mutates the list it is + given, so it needs a private copy and gets one for free by re-parsing the + row's JSON. + """ + return func_doc_language_specific_pre_processing( + json.loads(raw["function"]), raw["category"] + ) + + +class BfclV3PromptMixin: + """Upstream's prompting protocol: schemas in the system turn, calls in text.""" + + UNDERSCORE_TO_DOT: ClassVar[bool] = False + + def _build_messages(self, raw) -> tuple[list, list[dict] | None]: + # Upstream merges rather than prepends: a row that already opens with a + # system turn keeps its own text, with the schema block in front of it. + # 92 live rows do, and prepending a second system turn instead is a + # shape no upstream run ever sent. Copy each message -- the helper + # rewrites `content` in place, and `raw` is the stored sample. + messages = system_prompt_pre_processing_chat_model( + [dict(message) for message in raw["question"]], + _model_facing_functions(raw), + raw["category"], + ) + return messages, None + + def _decode(self, output: ModelOutput, language: str) -> list | None: + # Deferred on purpose: `ast_parse` is the one symbol in the vendored + # package that reaches tree-sitter, which lives in the optional + # `bfcl-v3` extra. Importing a task module registers it, and + # registration is paid by every task listing and by every eval that + # fails before grading -- so at module scope this would make the extra + # a hard requirement and break importing the task registry on a base + # install. It is genuinely needed only here -- but here it is needed + # for EVERY language, not just Java and JavaScript: `ast_parse` imports + # both source parsers at module scope, so the first call pulls + # tree-sitter in whichever grammar it ends up dispatching to. + from sieval.community.bfcl_v3 import ast_parse + + text = output.texts[0] if output.texts else "" + return ast_parse(text, language) + + +class BfclV3FCMixin: + """Upstream's native-function-calling protocol: schemas as tools.""" + + UNDERSCORE_TO_DOT: ClassVar[bool] = True + + requires = TaskRequirements(input=InputKind.CHAT, function_tools=True) + + def _build_messages(self, raw) -> tuple[list, list[dict] | None]: + tools = convert_to_tool( + _model_facing_functions(raw), + GORILLA_TO_OPENAPI, + ModelStyle.OpenAI_Completions, + ) + return list(raw["question"]), tools + + def _decode(self, output: ModelOutput, language: str) -> list | None: + del language # Structured calls arrive typed; no source text is parsed. + calls = output.tool_calls + if not calls: + return [] + return [{call.name: _call_arguments(call)} for call in calls] diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py new file mode 100644 index 00000000..8bb021c0 --- /dev/null +++ b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen.py @@ -0,0 +1,76 @@ +"""BFCL v3 live categories, 0-shot, upstream's prompting protocol. + +Six user-contributed categories, 2251 rows: four Python AST categories plus +irrelevance and relevance detection. Unlike the non-live group this one carries +intervals, because its weighted rollup IS the pooled rate over the union of its +rows -- see `BfclV3LiveTask`. + +Upstream's `(Prompt)` column: schemas arrive in the system turn and the reply is +parsed with `ast_parse`. The `_fc` sibling is the `(FC)` column, where the same +schemas arrive as tools. Headline, interval policy and the published numbers are +in `reference_impl.notes`; why the unqualified name is this one is in the +variants table. + +References: + +* Blog: +* Harness: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import BfclV3LiveDatasetSample + +from ._base import ( + BFCL_V3_HARNESS_URL, + BFCL_V3_SHARED_NOTES, + BfclV3LiveTask, + BfclV3PromptMixin, +) + + +@sieval_task( + name="bfcl_v3_live_0shot_gen", + display_name="BFCL v3 Live (0-shot, prompting)", + description=( + "Berkeley Function Calling Leaderboard v3, live categories, " + "upstream's prompting protocol." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "function-calling"), + # Needed even though every live category is Python, which is the part that + # reads wrong: `_decode` calls `ast_parse`, and its module imports both + # tree-sitter source parsers at module scope, so the import is paid before + # the language dispatch ever runs. Not on the `_fc` sibling -- FC parses no + # source text. Without this, readiness reports `yes`, the run bills for + # inference, and then every sample dies at postprocess. + deps_group="bfcl-v3", + model_type="chat", + status="stable", + reference_kind="value", + reference_impl=ReferenceImpl( + source="gorilla/berkeley-function-call-leaderboard", + url=BFCL_V3_HARNESS_URL, + notes=( + "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. " + "Headline is `Live Overall Acc`: the sample-count-weighted mean of " + "six categories, which is algebraically the pooled rate over all " + "2251 live rows, so it carries a problem-clustered interval; " + "`ast_summary` (weighted over the four AST categories) carries one " + "on `n_ast` for the same reason. underscore_to_dot=False: schemas " + "reach the model verbatim, so dotted function names survive and " + "gold needs no rewrite. Replay of upstream's gpt-4.1-2025-04-14 " + "(Prompt) rollouts agrees on 2251/2251 rows and reproduces every " + "published cell: live_simple 85.66, live_multiple 76.54, " + "live_parallel 93.75, live_parallel_multiple 75.00, " + "live_irrelevance 77.89, live_relevance 88.89; ast_summary 78.46, " + f"Live Overall Acc 78.32. {BFCL_V3_SHARED_NOTES}" + ), + ), +) +class BfclV3LiveZeroShotGenTask( + BfclV3PromptMixin, BfclV3LiveTask[BfclV3LiveDatasetSample] +): + pass diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py new file mode 100644 index 00000000..23ab97d4 --- /dev/null +++ b/sieval/tasks/bfcl_v3/bfcl_v3_live_0shot_gen_fc.py @@ -0,0 +1,75 @@ +"""BFCL v3 live categories, 0-shot, upstream's function-calling protocol. + +Six user-contributed categories, 2251 rows: four Python AST categories plus +irrelevance and relevance detection. Unlike the non-live group this one carries +intervals, because its weighted rollup IS the pooled rate over the union of its +rows -- see `BfclV3LiveTask`. + +Upstream's `(FC)` column: the same schemas reach the model through the provider's +tools API and the calls are read off structured `tool_calls` rather than parsed +out of the reply. Beyond that transport, the only behavioural difference from +the Prompt sibling is `underscore_to_dot=True` -- upstream's own reversal of the +`.`->`_` rewrite a native tool name forces, explained in `_base`. Headline, +interval policy and the published numbers are in `reference_impl.notes`. + +References: + +* Blog: +* Harness: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import BfclV3LiveDatasetSample + +from ._base import ( + BFCL_V3_HARNESS_URL, + BFCL_V3_SHARED_NOTES, + BfclV3FCMixin, + BfclV3LiveTask, +) + + +@sieval_task( + name="bfcl_v3_live_0shot_gen_fc", + display_name="BFCL v3 Live (0-shot, FC)", + description=( + "Berkeley Function Calling Leaderboard v3, live categories, " + "upstream's FC (tools API) protocol." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "function-calling"), + model_type="chat", + status="stable", + reference_kind="value", + reference_impl=ReferenceImpl( + source="gorilla/berkeley-function-call-leaderboard", + url=BFCL_V3_HARNESS_URL, + notes=( + "FC protocol -- upstream's `(FC)` leaderboard column. Headline is " + "`Live Overall Acc`: the sample-count-weighted mean of six " + "categories, which is algebraically the pooled rate over all 2251 " + "live rows, so it carries a problem-clustered interval; " + "`ast_summary` (weighted over the four AST categories) carries one " + "on `n_ast` for the same reason. underscore_to_dot=True: " + "convert_to_tool rewrote `.` to `_` in every function name on the " + "way out, because a native tool name cannot carry a dot, so gold is " + "rewritten to match before comparison. Requires the " + "`function_tools` capability; a binding that does not offer it is " + "rejected at task construction, which prelaunch reconciliation " + "reaches before the first request and before a result directory " + "exists. Replay of upstream's gpt-4.1-2025-04-14-FC rollouts agrees " + "on 2251/2251 rows and reproduces every published cell: live_simple " + "80.23, live_multiple 78.35, live_parallel 68.75, " + "live_parallel_multiple 66.67, live_irrelevance 82.31, " + "live_relevance 77.78; ast_summary 78.39, Live Overall Acc " + f"79.92. {BFCL_V3_SHARED_NOTES}" + ), + ), +) +class BfclV3LiveZeroShotGenFCTask( + BfclV3FCMixin, BfclV3LiveTask[BfclV3LiveDatasetSample] +): + pass diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py new file mode 100644 index 00000000..38993e7a --- /dev/null +++ b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen.py @@ -0,0 +1,76 @@ +"""BFCL v3 non-live categories, 0-shot, upstream's prompting protocol. + +Seven hand-written categories, 1390 rows: four Python AST categories, Java and +JavaScript simple-call categories, and irrelevance detection. + +Upstream's `(Prompt)` column: schemas arrive in the system turn and the reply is +parsed with `ast_parse`. The `_fc` sibling is the `(FC)` column, where the same +schemas arrive as tools. Headline, interval policy and the published numbers are +in `reference_impl.notes`; why the unqualified name is this one is in the +variants table. + +References: + +* Blog: +* Harness: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import BfclV3NonLiveDatasetSample + +from ._base import ( + BFCL_V3_HARNESS_URL, + BFCL_V3_SHARED_NOTES, + BfclV3NonLiveTask, + BfclV3PromptMixin, +) + + +@sieval_task( + name="bfcl_v3_non_live_0shot_gen", + display_name="BFCL v3 Non-Live (0-shot, prompting)", + description=( + "Berkeley Function Calling Leaderboard v3, non-live categories, " + "upstream's prompting protocol." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "function-calling"), + # Declared here and NOT on the `_fc` sibling: `_decode` calls `ast_parse`, + # whose module imports both tree-sitter source parsers at module scope, so + # the Prompt protocol needs the extra for every category rather than just + # `java` and `javascript`. FC never parses source text. Without this the + # import is deferred far enough that readiness reports `yes`, the run bills + # for inference, and then every sample dies at postprocess. + deps_group="bfcl-v3", + model_type="chat", + status="stable", + reference_kind="value", + reference_impl=ReferenceImpl( + source="gorilla/berkeley-function-call-leaderboard", + url=BFCL_V3_HARNESS_URL, + notes=( + "PROMPT protocol -- upstream's `(Prompt)` leaderboard column. " + "Headline is `Non_Live Overall Acc`: unweighted mean of simple_ast " + "(itself the unweighted mean of simple/java/javascript), multiple, " + "parallel, parallel_multiple and irrelevance. No interval on the " + "headline, simple_ast or ast_summary: an unweighted mean of category " + "rates is not the pooled rate (javascript's 50 rows weigh as much as " + "simple's 400), so a sample-clustered CI would bracket a different " + "statistic. Every per-category rate carries its own. " + "underscore_to_dot=False: schemas reach the model verbatim, so " + "dotted function names survive and gold needs no rewrite. Replay of " + "upstream's gpt-4.1-2025-04-14 (Prompt) rollouts agrees on 1390/1390 " + "rows and reproduces every published cell: simple 95.50, multiple " + "94.00, parallel 93.00, parallel_multiple 87.50, java 64.00, " + "javascript 82.00, irrelevance 88.75; simple_ast 80.50, Non_Live " + f"Overall Acc 88.75. {BFCL_V3_SHARED_NOTES}" + ), + ), +) +class BfclV3NonLiveZeroShotGenTask( + BfclV3PromptMixin, BfclV3NonLiveTask[BfclV3NonLiveDatasetSample] +): + pass diff --git a/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py new file mode 100644 index 00000000..8f964f05 --- /dev/null +++ b/sieval/tasks/bfcl_v3/bfcl_v3_non_live_0shot_gen_fc.py @@ -0,0 +1,74 @@ +"""BFCL v3 non-live categories, 0-shot, upstream's function-calling protocol. + +Seven hand-written categories, 1390 rows: four Python AST categories, Java and +JavaScript simple-call categories, and irrelevance detection. + +Upstream's `(FC)` column: the same schemas reach the model through the provider's +tools API and the calls are read off structured `tool_calls` rather than parsed +out of the reply. Beyond that transport, the only behavioural difference from +the Prompt sibling is `underscore_to_dot=True` -- upstream's own reversal of the +`.`->`_` rewrite a native tool name forces, explained in `_base`. Headline, +interval policy and the published numbers are in `reference_impl.notes`. + +References: + +* Blog: +* Harness: + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import BfclV3NonLiveDatasetSample + +from ._base import ( + BFCL_V3_HARNESS_URL, + BFCL_V3_SHARED_NOTES, + BfclV3FCMixin, + BfclV3NonLiveTask, +) + + +@sieval_task( + name="bfcl_v3_non_live_0shot_gen_fc", + display_name="BFCL v3 Non-Live (0-shot, FC)", + description=( + "Berkeley Function Calling Leaderboard v3, non-live categories, " + "upstream's FC (tools API) protocol." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "function-calling"), + model_type="chat", + status="stable", + reference_kind="value", + reference_impl=ReferenceImpl( + source="gorilla/berkeley-function-call-leaderboard", + url=BFCL_V3_HARNESS_URL, + notes=( + "FC protocol -- upstream's `(FC)` leaderboard column. Headline is " + "`Non_Live Overall Acc`: unweighted mean of simple_ast (itself the " + "unweighted mean of simple/java/javascript), multiple, parallel, " + "parallel_multiple and irrelevance. No interval on the headline, " + "simple_ast or ast_summary: an unweighted mean of category rates is " + "not the pooled rate (javascript's 50 rows weigh as much as simple's " + "400), so a sample-clustered CI would bracket a different statistic. " + "Every per-category rate carries its own. underscore_to_dot=True: " + "convert_to_tool rewrote `.` to `_` in every function name on the " + "way out, because a native tool name cannot carry a dot, so gold is " + "rewritten to match before comparison. Requires the `function_tools` " + "capability; a binding that does not offer it is rejected at task " + "construction, which prelaunch reconciliation reaches before the " + "first request and before a result directory exists. Replay of " + "upstream's gpt-4.1-2025-04-14-FC rollouts agrees on 1390/1390 rows " + "and reproduces every published cell: simple 93.50, multiple 90.50, " + "parallel 91.00, parallel_multiple 86.00, java 61.00, javascript " + "68.00, irrelevance 89.58; simple_ast 74.17, Non_Live Overall Acc " + f"86.25. {BFCL_V3_SHARED_NOTES}" + ), + ), +) +class BfclV3NonLiveZeroShotGenFCTask( + BfclV3FCMixin, BfclV3NonLiveTask[BfclV3NonLiveDatasetSample] +): + pass diff --git a/tests/acceptance/bfcl_v3/test_gold_replay.py b/tests/acceptance/bfcl_v3/test_gold_replay.py new file mode 100644 index 00000000..ecf666b9 --- /dev/null +++ b/tests/acceptance/bfcl_v3/test_gold_replay.py @@ -0,0 +1,151 @@ +"""Replay BFCL v3 gold as the reply, and require the grader to accept it. + +Each row's `possible_answer` IS an accepted answer, so grading it must return +correct. That checks the dataset join, the category routing and the vendored +checker together, against real data, with no model spend -- and it is the only +check here that can distinguish "the port is wired up right" from "the model is +good". + +Restricted to `language == "Python"`. Java and JavaScript gold is not in the +decoded form: `ast_checker` requires every parameter there to arrive as a raw +source-text string for the type converters to parse, while gold holds native +values. Replaying it would mean writing a Java/JS serializer inside a test, and +its bugs would be indistinguishable from grader bugs. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json +import pathlib + +import pytest + +from sieval.community.bfcl_v3 import GOLDLESS_CATEGORIES +from sieval.core.utils.hf import maybe_resolve_hf_path +from sieval.datasets import BfclV3LiveDataset, BfclV3NonLiveDataset +from sieval.tasks.bfcl_v3._base import grade_single_turn + +#: The snapshot both datasets are pinned to; `maybe_resolve_hf_path` rewrites it +#: to `$SIEVAL_DATA_DIR/gorilla-llm/...`. +SNAPSHOT = "gorilla-llm/Berkeley-Function-Calling-Leaderboard" + +#: Rows where upstream's gold contradicts upstream's own schema, so no reply can +#: be both the gold answer and schema-valid. Each was read individually; see the +#: table in this module's commit message. Asserted by EQUALITY: a pin bump that +#: fixes one must fail here and be updated in the same commit. +UPSTREAM_GOLD_DEFECTS = frozenset( + { + "simple_363", + "live_simple_106-63-0", + "live_simple_112-68-0", + "live_multiple_507-149-4", + "live_multiple_862-181-3", + "live_multiple_964-207-0", + "live_multiple_1038-265-0", + } +) + + +def _resolve(node): + """Gold nests alternatives inside dicts AND lists; unwrap every level.""" + if isinstance(node, dict): + return { + key: _first(accepted) + for key, accepted in node.items() + if _first(accepted) is not None + } + if isinstance(node, list): + return [_resolve(item) for item in node] + return node + + +def _first(accepted): + """One concrete value out of a gold alternatives list. + + A leaf is *usually* a list of acceptable values, but three leaves in 3641 + rows are a bare scalar, so a non-list is passed through rather than indexed. + `""` means "omitting is acceptable" and is skipped here; whether omission is + actually permitted is the schema's call, made in `_replay`. + """ + if not isinstance(accepted, list): + return accepted + for candidate in accepted: + if candidate != "": + return _resolve(candidate) + return None + + +def _replay(gold_entry: list, functions: list) -> list[dict]: + """Turn one `ground_truth` entry into the decoded form a model would yield. + + Reads the schema because gold alone is ambiguous: `""` among a parameter's + alternatives licenses omitting it, but only where the schema does not mark it + required -- and some parameters gold offers are not in the schema at all, so + naming them is an `unexpected_param`. + """ + schemas = {fn["name"]: fn for fn in functions} + replayed = [] + for call in gold_entry: + for name, params in call.items(): + parameters = schemas.get(name, {}).get("parameters", {}) + required = set(parameters.get("required", ())) + args = {} + for param, accepted in params.items(): + if not accepted: + continue + # `""` licenses omitting the parameter, but only where the + # schema does not require it -- dropping a required one is + # `missing_required`, and emitting one the schema never declares + # is `unexpected_param`, so both sides need the schema. + omittable = isinstance(accepted, list) and "" in accepted + if omittable and param not in required: + continue + value = _first(accepted) + if value is not None: + args[param] = value + replayed.append({name: args}) + return replayed + + +@pytest.mark.skipif( + not pathlib.Path(maybe_resolve_hf_path(SNAPSHOT)).is_dir(), + reason=f"needs the downloaded BFCL v3 snapshot ({SNAPSHOT})", +) +def test_replaying_gold_as_the_reply_grades_correct(): + missed = set() + graded = 0 + for dataset in (BfclV3NonLiveDataset(SNAPSHOT), BfclV3LiveDataset(SNAPSHOT)): + test_set = dataset.test_set + # Bound rather than iterated inline: the property is `HFDataset | None`, + # and a missing split would otherwise walk zero rows on the way to the + # count assertion below, reporting a moved pin for an absent split. + assert test_set is not None, f"{type(dataset).__name__} has no test split" + for row in test_set: + if row["category"] in GOLDLESS_CATEGORIES or row["language"] != "Python": + continue + graded += 1 + decoded = _replay( + json.loads(row["ground_truth"]), json.loads(row["function"]) + ) + if not grade_single_turn( + row["function"], + decoded, + row["ground_truth"], + row["language"], + row["category"], + False, + ): + missed.add(row["id"]) + + assert graded == 2351, ( + f"Expected 2351 gradeable Python rows, walked {graded}. The snapshot pin " + "moved, or the goldless/language filter changed -- either way the defect " + "list below was measured against a different corpus." + ) + assert missed == UPSTREAM_GOLD_DEFECTS, ( + "Replaying gold as the reply must grade correct.\n" + f" newly failing (OUR defect, fix it): " + f"{sorted(missed - UPSTREAM_GOLD_DEFECTS)}\n" + f" newly passing (upstream fixed it, update the set): " + f"{sorted(UPSTREAM_GOLD_DEFECTS - missed)}" + ) diff --git a/tests/acceptance/bfcl_v3/test_upstream_rollout_replay.py b/tests/acceptance/bfcl_v3/test_upstream_rollout_replay.py new file mode 100644 index 00000000..6929c459 --- /dev/null +++ b/tests/acceptance/bfcl_v3/test_upstream_rollout_replay.py @@ -0,0 +1,337 @@ +"""Anchor BFCL v3 grading on upstream's own released rollouts and verdicts. + +BFCL publishes an evaluation archive beside the leaderboard: per model, per +protocol, per category, both the model's recorded replies and its own per-row +verdicts. That is per-prediction ground truth, so the port pins exactly with no +model spend. + +Stronger than `test_gold_replay.py`, which feeds gold back and so only ever +exercises the *correct* path -- never a wrong answer, a decode failure, or +either protocol's decoder. This replays real output, 9% of it wrong, through the +production `_decode` of both mixins. `java` at 64% is the point: a third of +those rows are wrong and we have to agree about which third. + +It does NOT cover prompt construction -- the replies are upstream's. That rests +on the two model-facing helpers being byte-identical to upstream (pinned by +`test_identity.py`) and running on a row-for-row verified snapshot: a +construction argument, not a measurement. + +No file hashes are pinned, and they would be redundant: a swapped archive fails +the :data:`PUBLISHED` header assertion, and a doctored `result` file fails the +comparison it feeds. + +Staging is opt-in, like the gold replay -- ~6MB, so not vendored: + + R=https://raw.githubusercontent.com/HuanzhiMao/BFCL-Result/main/2025-06-14 + D="$SIEVAL_DATA_DIR/HuanzhiMao/BFCL-Result/2025-06-14" + for m in gpt-4.1-2025-04-14 gpt-4.1-2025-04-14-FC; do + for k in result score; do + mkdir -p "$D/$k/$m" + for c in simple multiple parallel parallel_multiple java javascript \ + irrelevance live_simple live_multiple live_parallel \ + live_parallel_multiple live_irrelevance live_relevance; do + curl -sSf -o "$D/$k/$m/BFCL_v3_${c}_${k}.json" \ + "$R/$k/$m/BFCL_v3_${c}_${k}.json" + done + done + done + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import collections +import json +import pathlib + +import pytest + +from sieval.core.models import FunctionToolCall, GenModel, ModelOutput +from sieval.core.utils.hf import maybe_resolve_hf_path +from sieval.datasets import BfclV3LiveDataset, BfclV3NonLiveDataset +from sieval.tasks.bfcl_v3._base import ( + BfclV3FCMixin, + BfclV3LiveTask, + BfclV3NonLiveTask, + BfclV3PromptMixin, + grade_single_turn, +) + +#: The archive snapshot these counts were read from. It predates the pinned +#: harness revision (v1.3, 2025-07-17) by a month, and agrees anyway -- which is +#: itself evidence the checker did not move for these categories in between. +ARCHIVE = "HuanzhiMao/BFCL-Result" +SNAPSHOT = "2025-06-14" + +DATA_SNAPSHOT = "gorilla-llm/Berkeley-Function-Calling-Leaderboard" + +NON_LIVE = ( + "simple", + "multiple", + "parallel", + "parallel_multiple", + "java", + "javascript", + "irrelevance", +) +LIVE = ( + "live_simple", + "live_multiple", + "live_parallel", + "live_parallel_multiple", + "live_irrelevance", + "live_relevance", +) + +#: `(correct_count, total_count)` off each score file's header line -- upstream's +#: published cell, as integers rather than the rounded percentage the +#: leaderboard renders. Asserted both against the staged file (archive +#: integrity) and against our own replay (the actual anchor). +PUBLISHED = { + "gpt-4.1-2025-04-14": { + "simple": (382, 400), + "multiple": (188, 200), + "parallel": (186, 200), + "parallel_multiple": (175, 200), + "java": (64, 100), + "javascript": (41, 50), + "irrelevance": (213, 240), + "live_simple": (221, 258), + "live_multiple": (806, 1053), + "live_parallel": (15, 16), + "live_parallel_multiple": (18, 24), + "live_irrelevance": (687, 882), + "live_relevance": (16, 18), + }, + "gpt-4.1-2025-04-14-FC": { + "simple": (374, 400), + "multiple": (181, 200), + "parallel": (182, 200), + "parallel_multiple": (172, 200), + "java": (61, 100), + "javascript": (34, 50), + "irrelevance": (215, 240), + "live_simple": (207, 258), + "live_multiple": (825, 1053), + "live_parallel": (11, 16), + "live_parallel_multiple": (16, 24), + "live_irrelevance": (726, 882), + "live_relevance": (14, 18), + }, +} + +#: Protocol under test: archive model dir, and the mixin whose production +#: `_decode` and `UNDERSCORE_TO_DOT` are exercised. +PROTOCOLS = { + "prompt": ("gpt-4.1-2025-04-14", BfclV3PromptMixin), + "fc": ("gpt-4.1-2025-04-14-FC", BfclV3FCMixin), +} + +_ROOT = pathlib.Path(maybe_resolve_hf_path(ARCHIVE)) / SNAPSHOT + +requires_archive = pytest.mark.skipif( + not _ROOT.is_dir(), + reason=f"needs the staged BFCL result archive ({ARCHIVE} @ {SNAPSHOT})", +) + + +def _jsonl(path: pathlib.Path) -> list[dict]: + return [json.loads(x) for x in path.open(encoding="utf-8") if x.strip()] + + +def _model_output(protocol: str, recorded) -> ModelOutput: + """Upstream's recorded reply, in the shape the mixin's `_decode` reads. + + A real `ModelOutput`, not a stand-in with the two attributes `_decode` + touches. The FC archive stores each call as `{name: arguments-JSON-string}`, + which is what the OpenAI dialects put on `FunctionToolCall.arguments`, so + the calls are rebuilt rather than pre-parsed and `_call_arguments` is part + of what this anchors. + """ + meta = GenModel(model="archive-replay", api_key="none").meta() + if protocol == "prompt": + text = recorded if isinstance(recorded, str) else "" + return ModelOutput(model=meta, texts=[text]) + calls = [] + for item in recorded if isinstance(recorded, list) else []: + if not isinstance(item, dict): + continue + for index, (name, arguments) in enumerate(item.items()): + calls.append( + FunctionToolCall( + f"call_{len(calls)}_{index}", + name, + arguments if isinstance(arguments, str) else json.dumps(arguments), + ) + ) + return ModelOutput(model=meta, texts=[""], tool_calls=tuple(calls)) + + +def _decode_or_fail(decoder, protocol: str, recorded, language): + """The mixin's production `_decode`, under `postprocess`'s failure semantic. + + `postprocess` wraps the call in `except Exception -> [None]`: an undecodable + reply is upstream's `ast_decoder:decoder_failed`, a SCORE rather than a + fault. Mirrored here or a real reply takes the test out -- a model that + declines in prose raises `SyntaxError` straight out of `ast.parse`. Only + those two lines are mirrored; the decoder is the shipped one, and + `postprocess`'s narrower `ImportError` re-raise is pinned in the unit tests. + """ + try: + return decoder._decode(_model_output(protocol, recorded), language) + except Exception: # noqa: BLE001 -- a decode failure is a score + return None + + +def _replay(protocol: str, categories, dataset) -> dict[str, tuple[int, int]]: + """Grade every recorded reply; return `{category: (ours, total)}`. + + Asserts row-for-row agreement with upstream's verdicts on the way, so a + disagreement names the row rather than only moving a total. + """ + model_dir, mixin = PROTOCOLS[protocol] + decoder = mixin() + + # `live_relevance` ships one id twice (18 rows, 17 distinct ids), so rows and + # recorded replies are both consumed from per-id queues. Keying either by id + # would drop that row and quietly score 17. + rows = collections.defaultdict(collections.deque) + for row in dataset.test_set: + rows[row["id"]].append(row) + + out: dict[str, tuple[int, int]] = {} + for category in categories: + base = _ROOT / "{kind}" / model_dir / f"BFCL_v3_{category}_{{kind}}.json" + results = _jsonl(pathlib.Path(str(base).format(kind="result"))) + scored = _jsonl(pathlib.Path(str(base).format(kind="score"))) + + header = scored[0] + expected = PUBLISHED[model_dir][category] + assert (header["correct_count"], header["total_count"]) == expected, ( + f"{model_dir}/{category}: staged archive publishes " + f"{header['correct_count']}/{header['total_count']}, pinned " + f"{expected[0]}/{expected[1]} -- the archive moved, so every number " + "below was measured against a different corpus." + ) + wrong = collections.Counter(r["id"] for r in scored[1:] if "id" in r) + + pool = {k: collections.deque(v) for k, v in rows.items()} + correct = walked = 0 + disagreed = [] + for record in results: + queue = pool.get(record["id"]) + if not queue: + continue + row = queue.popleft() + walked += 1 + ours = grade_single_turn( + row["function"], + _decode_or_fail(decoder, protocol, record["result"], row["language"]), + row["ground_truth"], + row["language"], + category, + mixin.UNDERSCORE_TO_DOT, + ) + correct += int(ours) + if ours != (wrong[record["id"]] == 0): + disagreed.append(record["id"]) + + assert walked == expected[1], ( + f"{model_dir}/{category}: replayed {walked} of {expected[1]} rows. " + "A short walk scores the missing rows as absent, not as wrong." + ) + assert not disagreed, ( + f"{model_dir}/{category}: our verdict differs from upstream's on " + f"{len(disagreed)} row(s), e.g. {disagreed[:5]}" + ) + out[category] = (correct, expected[1]) + return out + + +@requires_archive +@pytest.mark.parametrize("protocol", sorted(PROTOCOLS)) +@pytest.mark.parametrize( + ("group", "categories"), [("non_live", NON_LIVE), ("live", LIVE)] +) +def test_replaying_upstream_rollouts_reproduces_every_published_cell( + protocol: str, group: str, categories +): + dataset = ( + BfclV3NonLiveDataset(DATA_SNAPSHOT) + if group == "non_live" + else BfclV3LiveDataset(DATA_SNAPSHOT) + ) + model_dir, _ = PROTOCOLS[protocol] + replayed = _replay(protocol, categories, dataset) + assert replayed == {c: PUBLISHED[model_dir][c] for c in categories} + + +@requires_archive +@pytest.mark.parametrize("protocol", sorted(PROTOCOLS)) +def test_the_live_rollup_is_the_pooled_rate_over_every_live_row(protocol: str): + """The weighted rollup's own claim, checked rather than asserted in prose. + + `BfclV3LiveTask` documents its weighted mean as algebraically the pooled + rate over the union of the six categories' rows -- which is what makes the + headline a genuine per-sample rate, and so interval-bearing. + """ + model_dir, _ = PROTOCOLS[protocol] + cells = {c: PUBLISHED[model_dir][c] for c in LIVE} + + def cell(category: str) -> dict: + correct, total = cells[category] + accuracy = 100.0 * correct / total + return { + "accuracy": accuracy, + "total_count": total, + "display_accuracy": accuracy, + } + + # Built without `__init__` (which would demand a live model binding); the + # rollup reads only ClassVars. Runs the PRODUCTION aggregation. + task = object.__new__(BfclV3LiveTask) + rollup = task._aggregate(cell) # noqa: SLF001 -- tests are the carve-out + pooled = ( + 100.0 * sum(c for c, _ in cells.values()) / sum(t for _, t in cells.values()) + ) + assert rollup["live_overall_acc"] == pytest.approx(pooled, abs=1e-9) + + +@requires_archive +@pytest.mark.parametrize("protocol", sorted(PROTOCOLS)) +def test_the_non_live_rollup_is_the_documented_mean_of_means(protocol: str): + """Non-live nests `simple_ast` inside a five-way unweighted mean. + + Deliberately NOT the pooled rate: javascript's 50 rows weigh as much as + simple's 400, which is why the task publishes no interval on this headline. + Pinned here so a "fix" toward pooling fails loudly. + """ + model_dir, _ = PROTOCOLS[protocol] + cells = {c: PUBLISHED[model_dir][c] for c in NON_LIVE} + + def rate(category: str) -> float: + correct, total = cells[category] + return 100.0 * correct / total + + def cell(category: str) -> dict: + return { + "accuracy": rate(category), + "total_count": cells[category][1], + "display_accuracy": rate(category), + } + + task = object.__new__(BfclV3NonLiveTask) + rollup = task._aggregate(cell) # noqa: SLF001 -- tests are the carve-out + simple_ast = sum(rate(c) for c in ("simple", "java", "javascript")) / 3 + expected = ( + simple_ast + + sum( + rate(c) + for c in ("multiple", "parallel", "parallel_multiple", "irrelevance") + ) + ) / 5 + assert rollup["simple_ast"] == pytest.approx(simple_ast, abs=1e-9) + assert rollup["non_live_overall_acc"] == pytest.approx(expected, abs=1e-9) + pooled = ( + 100.0 * sum(c for c, _ in cells.values()) / sum(t for _, t in cells.values()) + ) + assert rollup["non_live_overall_acc"] != pytest.approx(pooled, abs=1e-6) diff --git a/tests/unit/community/bfcl_v3/test__safe_eval.py b/tests/unit/community/bfcl_v3/test__safe_eval.py new file mode 100644 index 00000000..a3f413f5 --- /dev/null +++ b/tests/unit/community/bfcl_v3/test__safe_eval.py @@ -0,0 +1,180 @@ +"""Cover the guard that replaces `eval` in the BFCL v3 argument resolver. + +Two properties, and neither is worth much alone: the guard must refuse what +executes (or it is not a guard) *and* agree with upstream on everything that +does not (or it is a scoring change wearing a safety label). + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import ast + +import pytest + +from sieval.community.bfcl_v3._safe_eval import MAX_RESULT_SIZE, safe_eval +from sieval.community.bfcl_v3.parser import ast_parse, resolve_ast_by_type + + +def _argument(source: str) -> ast.expr: + """The `x=` argument node of a one-call snippet, as the resolver sees it.""" + return ast.parse(source, mode="eval").body.keywords[0].value # ty: ignore + + +def test_a_payload_in_an_argument_does_not_run(tmp_path): + """The whole point, observed as a side effect rather than as a refusal. + + Asserting only that a `ValueError` comes back would pass against a guard + that refuses *after* evaluating. The marker file is what distinguishes + "refused" from "ran, then complained": under the `eval` this replaces, the + file exists and the decoder returns an ordinary-looking number. + """ + marker = tmp_path / "payload-ran" + source = f"f(x=__import__('pathlib').Path({str(marker)!r}).write_text('x') + 0)" + + with pytest.raises(ValueError, match="refusing to evaluate Call"): + resolve_ast_by_type(_argument(source)) + + assert not marker.exists(), "the payload executed before being refused" + + +def test_the_decoder_refuses_rather_than_executing_a_hostile_reply(tmp_path): + """Through the real entry point, because that is what `postprocess` calls. + + The refusal has to surface as a decode failure -- an exception out of + `ast_parse` -- so the sample scores wrong. A reply that puts executable code + in an argument slot did not produce the function call it was asked for. + """ + marker = tmp_path / "payload-ran" + reply = f"[f(x=__import__('pathlib').Path({str(marker)!r}).write_text('x') + 0)]" + + with pytest.raises(ValueError, match="refusing"): + ast_parse(reply, "Python") + + assert not marker.exists() + + +#: Every shape upstream's `eval(ast.unparse(node))` computes without executing +#: anything. The expected values are upstream's, so a divergence shows up as a +#: wrong value rather than as a refusal. +@pytest.mark.parametrize( + ("source", "expected"), + [ + ("f(x=1 + 2)", 3), + ("f(x=60 * 60)", 3600), + ("f(x=10 / 4)", 2.5), + ("f(x=7 // 2)", 3), + ("f(x=7 % 3)", 1), + ("f(x=2 ** 10)", 1024), + ("f(x=-3 + 1)", -2), + ("f(x='a' + 'b')", "ab"), + ("f(x=[1] + [2])", [1, 2]), + ("f(x=(1,) + (2,))", (1, 2)), + ("f(x=1 << 4)", 16), + ("f(x=6 & 3)", 2), + ("f(x=6 | 3)", 7), + ("f(x=6 ^ 3)", 5), + ("f(x=1.5 * 2)", 3.0), + ("f(x=(1 + 2) * (3 - 1))", 6), + ("f(x=[1 + 1, 2 * 2])", [2, 4]), + ("f(x={'a': 1 + 1})", {"a": 2}), + ], +) +def test_a_non_executing_expression_keeps_upstreams_value(source, expected): + node = _argument(source) + result = resolve_ast_by_type(node) + assert result == expected + # `1 == 1.0` and `True == 1`, so equality alone would let a type change + # through -- and the checker compares types. + assert type(result) is type(expected) + + +@pytest.mark.parametrize( + "source", + [ + "f(x=__import__('os').getpid() + 0)", + "f(x=open('/etc/passwd').read() + '')", + "f(x=[].__class__ + 0)", + "f(x=undefined_name + 1)", + "f(x=(lambda: 1)() + 1)", + "f(x=[i for i in range(3)] + [])", + # The spread must sit inside a BinOp to reach this guard at all: a bare + # `{**a}` argument is handled by upstream's own `ast.Dict` branch, which + # refuses it a different way (`Exception: Unsupported AST type`). + "f(x={**{'a': 1}} | {'b': 2})", + "f(x=f'a' + 'b')", + ], +) +def test_an_expression_that_could_name_anything_is_refused(source): + with pytest.raises(ValueError, match="refusing"): + resolve_ast_by_type(_argument(source)) + + +@pytest.mark.parametrize( + "source", + [ + "f(x=9 ** 9 ** 9)", # 369 million digits + "f(x='a' * 10 ** 9)", # a gigabyte of string + "f(x=1 << 10 ** 9)", + "f(x=[0] * 10 ** 9)", + ], +) +def test_an_unbounded_expression_is_refused_before_it_is_computed(source): + """A hang here is the failure, not an error. + + Decoding runs inline on the session's event loop with no timeout around it, + so computing any of these would stall every other sample in the run. The + test can only assert that it returns at all -- which it does only because + the cost is screened before the operator runs. + """ + refused = "refusing an expression|refusing an intermediate" + with pytest.raises(ValueError, match=refused): + resolve_ast_by_type(_argument(source)) + + +@pytest.mark.parametrize( + "source", + [ + "f(x='%.400000000f' % 1.0)", # ~400MB of string from two tiny operands + "f(x='%.20f' % 1.0)", # 22 chars -- admissible by SIZE, refused anyway + "f(x=b'%.400000000f' % 1.0)", + ], +) +def test_percent_formatting_on_a_string_is_refused_outright(source): + """Refused by SHAPE, not by size -- which is the whole point. + + A precision field sets the result size independently of both operands, so + there is nothing for `_refuse_if_unbounded` to measure and `_bounded` would + see the value only after the allocation it exists to prevent. The 22-char + case is the discriminating one: it is far under `MAX_RESULT_SIZE`, so a + post-hoc size check admits it. Only a categorical refusal rejects it, and + only a categorical refusal bounds the 400MB sibling before it is built. + """ + with pytest.raises(ValueError, match="refusing `%` formatting"): + resolve_ast_by_type(_argument(source)) + + +def test_integer_modulo_is_still_admissible(): + """`%` on integers is bounded by its right operand, so it keeps working.""" + assert safe_eval(_argument("f(x=10 % 3)")) == 1 + assert safe_eval(_argument("f(x=10 ** 100 % 7)")) == 10**100 % 7 + + +def test_the_bound_admits_far_more_than_any_real_argument(): + """Evidence the cap does not bind: the pinned largest literal is 13 digits.""" + assert safe_eval(_argument("f(x=2 ** 4096)")) == 2**4096 + assert safe_eval(_argument("f(x=1617262800000 * 2)")) == 3234525600000 + assert MAX_RESULT_SIZE >= 1 << 20 + + +def test_the_lambda_branch_still_cannot_reach_its_eval(): + """Upstream's second `eval` is dead, and this fails if it stops being dead. + + `ast.Lambda.body` is a single expression node, so upstream's + `value.body[0].value` raises while the argument to `eval` is still being + built. That is why the branch was left byte-identical: hardening a line that + cannot execute buys nothing. If someone later "repairs" the subscript, this + test fails -- which is the point, because the repair would install a live + `eval` over model output. + """ + with pytest.raises(TypeError, match="not subscriptable"): + resolve_ast_by_type(_argument("f(x=lambda a: a + 1)")) diff --git a/tests/unit/community/bfcl_v3/test_identity.py b/tests/unit/community/bfcl_v3/test_identity.py new file mode 100644 index 00000000..6365eaed --- /dev/null +++ b/tests/unit/community/bfcl_v3/test_identity.py @@ -0,0 +1,118 @@ +"""Pin the vendored BFCL v3 files, and keep their provenance resolvable. + +A sha256 over our own copy pins whatever was copied: it detects drift after +vendoring and proves nothing about what was vendored. The non-circular anchor is +each module's `Upstream blob:` line -- a git blob id anyone can resolve in +upstream's repository. All twelve were resolved against `ShishirPatil/gorilla` +at `ea13468e` and matched. + +Hence two tests: one detects drift (its digests can only be read off the copy), +the other keeps the real anchor *present*. The second cannot resolve the blob -- +no test may reach for upstream's repository -- so it checks the claim is +recorded in a resolvable shape; without it, a re-vendoring that drops the +docstring leaves the first passing over bytes with no stated origin. + +`Upstream path:` is relative to `berkeley-function-call-leaderboard/`, not to +the repository root, where none of these paths exist. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import hashlib +import re +from pathlib import Path + +import pytest + +import sieval.community.bfcl_v3 as pkg + +#: sha256 of each vendored file AS COMMITTED -- upstream's bytes plus the +#: documented import rewrites, provenance docstring, and for `parser.py` the +#: annotated safety deviation. Updated only alongside a change to that file's +#: docstring deviation list, in the same commit, so the reason a digest moved is +#: always readable next to the new digest. +VENDORED_SHA256 = { + "ast_checker.py": ( + "66ba103f5dd17d9e2b4ee597ebfdb013486add739412366e65c4cb6691acb936" + ), + "type_mappings.py": ( + "cebc6f9a1b28e277483dd13b2a3fd3ceb95ad8a4631daf50933ad1af3015c698" + ), + "type_convertor/java_type_converter.py": ( + "41082275fda22106130db71d65902786884ca58379814124291a1274ae40451b" + ), + "type_convertor/js_type_converter.py": ( + "32ec360c519e3813c2159993ce78e290152b1a79abdfa3e35f03c6ed3af24202" + ), + "source_parser/java_parser.py": ( + "e31322b491601ab0e5ca3a3ae5a227f9478ed93783f740813d51228ef20a999b" + ), + "source_parser/js_parser.py": ( + "abf44b3f6d38b1cfb14d57b005e0ea7d133543a393bfd54c0056c7c74482e23a" + ), + "parser.py": ("e5655d4e60f40388f7b061cd551d29769c9f1cab1e67d7c1699a556531d8d17e"), + "tool_convert.py": ( + "f6474e2a5348e521b16e51fc5a3f669aef85343a1307a723715209696c407804" + ), + "output_checks.py": ( + "683a43d71e3e55d52b5edc7afd94def3ce449261642c3dbde543ae82ceab5268" + ), + "aggregate.py": ( + "b7abb648a15492db0101d7a0628b8cc1fdfa2d1542992e0819a44911729c9472" + ), + "prompts.py": ("fae6ed78b354c3e1a355795569a78ba18f2b0826f52bfa1dca35a2454655b78f"), + "preprocess.py": ( + "ce638e4bee853d12644528e728475025810b719287631f0ea68e9197dfde763f" + ), +} + +#: Upstream revision every `Upstream blob:` line above was resolved against. +UPSTREAM_REVISION = "ea13468e" + + +@pytest.mark.parametrize("relative_path", sorted(VENDORED_SHA256)) +def test_vendored_file_has_not_drifted(relative_path: str): + root = Path(pkg.__file__).parent + digest = hashlib.sha256((root / relative_path).read_bytes()).hexdigest() + assert digest == VENDORED_SHA256[relative_path], ( + f"{relative_path} changed since vendoring. A vendored file tracks " + "upstream; if the change is deliberate, say what it is in the module " + "docstring's deviation list and update this hash in the same commit." + ) + + +@pytest.mark.parametrize("relative_path", sorted(VENDORED_SHA256)) +def test_vendored_file_records_resolvable_provenance(relative_path: str): + """Every vendored file states where it came from, resolvably. + + Three files legitimately share one blob (`parser.py`, `tool_convert.py` and + `preprocess.py` are all partial extractions from upstream's + `model_handler/utils.py`), so the blob ids are not asserted unique. + """ + text = (Path(pkg.__file__).parent / relative_path).read_text() + assert re.search(r"^Upstream path: \S+\.py$", text, re.M), ( + f"{relative_path} records no `Upstream path:`. The sha256 above pins " + "these bytes but says nothing about their origin; this line is what a " + "reader follows to check the copy against upstream." + ) + assert re.search(r"^Upstream blob: [0-9a-f]{40}$", text, re.M), ( + f"{relative_path} records no `Upstream blob:` git object id. That id is " + f"the only anchor outside this checkout -- resolvable at " + f"{UPSTREAM_REVISION} as " + f"`berkeley-function-call-leaderboard/`." + ) + + +def test_category_tables_total_the_pinned_row_counts(): + """The per-category tables are constants; these are the totals they must sum to. + + Named for what it does: it reads no snapshot. The check that the tables + match the downloaded data is the acceptance replay, which is skipped + wherever the snapshot is absent -- so this one has to stand on its own as a + guard against an edited table, not be mistaken for the one that walks rows. + """ + from sieval.community.bfcl_v3 import LIVE_COUNTS, NON_LIVE_COUNTS + + assert sum(NON_LIVE_COUNTS.values()) == 1390 + assert sum(LIVE_COUNTS.values()) == 2251 + assert set(NON_LIVE_COUNTS) & set(LIVE_COUNTS) == set() diff --git a/tests/unit/community/bfcl_v3/test_source_parsers.py b/tests/unit/community/bfcl_v3/test_source_parsers.py new file mode 100644 index 00000000..677dcc6a --- /dev/null +++ b/tests/unit/community/bfcl_v3/test_source_parsers.py @@ -0,0 +1,40 @@ +"""`ast_parse` must actually dispatch to the vendored source-text parsers. + +The identity test proves the vendored bytes have not drifted; it says nothing +about whether the Java/JavaScript branches run. These assert real behaviour +through `ast_parse`'s public surface, covering all three branches (Python, +Java, JavaScript) plus the unsupported-language error path. + +Expected values were derived by running the parsers, not assumed: both +tree-sitter parsers return every argument value as a string (`"true"`, not +`True`), which is upstream's behaviour, not a porting artifact. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import pytest + +from sieval.community.bfcl_v3 import ast_parse + + +def test_java_source_is_parsed_into_a_call_dict(): + assert ast_parse( + '[TaskManager.removeTask(task="Meeting", urgent=true)]', "Java" + ) == [{"TaskManager.removeTask": {"task": "Meeting", "urgent": "true"}}] + + +def test_javascript_source_is_parsed_into_a_call_dict(): + assert ast_parse('[doTask(name="Meeting", urgent=true)]', "JavaScript") == [ + {"doTask": {"name": "Meeting", "urgent": "true"}} + ] + + +def test_python_source_is_still_parsed_by_ast(): + assert ast_parse('[doTask(name="Meeting", urgent=True)]', "Python") == [ + {"doTask": {"name": "Meeting", "urgent": True}} + ] + + +def test_unknown_language_still_raises_not_implemented(): + with pytest.raises(NotImplementedError): + ast_parse('[doTask(name="Meeting")]', "Ruby") diff --git a/tests/unit/core/models/test_model.py b/tests/unit/core/models/test_model.py index 0c70a961..9c6404e7 100644 --- a/tests/unit/core/models/test_model.py +++ b/tests/unit/core/models/test_model.py @@ -23,6 +23,7 @@ ConnectionIdentity, ConnectionPool, DialectOptions, + FunctionToolCall, GenModel, InputScoringResult, Model, @@ -1587,6 +1588,40 @@ def test_empty_reasoning_stays_none(self): ) assert out.reasoning_texts is None + def test_tool_calls_reach_the_task_in_order(self): + """A task that declares ``function_tools`` reads the calls back here. + + The bridge rebuilds the record field by field, so a channel it does not + name is dropped without a failure anywhere. + """ + out = self._bridge( + Response( + texts=("",), + tool_calls=( + FunctionToolCall("call_1", "get_weather", '{"city":"Paris"}'), + FunctionToolCall("call_2", "get_time", '{"tz":"UTC"}'), + ), + ) + ) + assert out.tool_calls == ( + FunctionToolCall("call_1", "get_weather", '{"city":"Paris"}'), + FunctionToolCall("call_2", "get_time", '{"tz":"UTC"}'), + ) + + def test_tool_calls_absent_stays_none(self): + out = self._bridge(Response(texts=("t",), tool_calls=None)) + assert out.tool_calls is None + + def test_tool_calls_present_but_empty_survives(self): + """Calling nothing is a reply about the tools; absence is not.""" + out = self._bridge(Response(texts=("t",), tool_calls=())) + assert out.tool_calls == () + + def test_tool_calls_default_to_none_when_omitted(self): + """Every existing construction site omits the field.""" + model = GenModel(model="m", api_key="k") + assert ModelOutput(model=model.meta(), texts=["t"]).tool_calls is None + def test_caller_mapping_is_not_mutated_and_not_shared(self): """Provenance is attached to a copy. diff --git a/tests/unit/core/models/test_requirements.py b/tests/unit/core/models/test_requirements.py index 1c3abc8e..9c621f9c 100644 --- a/tests/unit/core/models/test_requirements.py +++ b/tests/unit/core/models/test_requirements.py @@ -101,6 +101,22 @@ def test_invalid_requirement_values_are_rejected( with pytest.raises((TypeError, ValueError), match=message): TaskRequirements(**cast(Any, kwargs)) + def test_function_tools_defaults_off_and_requires_no_capability(self) -> None: + from sieval.core.tasks.task import _required_capabilities + + assert TaskRequirements().function_tools is False + assert "function_tools" not in _required_capabilities(TaskRequirements()) + + def test_function_tools_flag_demands_the_capability(self) -> None: + from sieval.core.tasks.task import _required_capabilities + + requires = TaskRequirements(function_tools=True) + assert _required_capabilities(requires) == frozenset({"function_tools"}) + + def test_function_tools_rejects_a_non_bool(self) -> None: + with pytest.raises(TypeError, match="function_tools must be a boolean"): + TaskRequirements(function_tools=cast(Any, "yes")) + class TestNormalizedBindings: def test_all_bindings_are_frozen_and_hold_only_setup_values(self) -> None: diff --git a/tests/unit/datasets/test_bfcl_v3.py b/tests/unit/datasets/test_bfcl_v3.py new file mode 100644 index 00000000..8ca070eb --- /dev/null +++ b/tests/unit/datasets/test_bfcl_v3.py @@ -0,0 +1,376 @@ +"""The shared BFCL v3 single-turn loader: the index join and its guards. + +What is covered here is the join key (the universal index, not the whole id), +the per-category row counts, the guards that refuse an ambiguous or incomplete +join, and the one thing neither can check -- that a goldless category stays +goldless when it is loaded next to a category that has gold. Also pinned: the +package holding the loader's category tables must import without the optional +`bfcl-v3` extra, since every dataset in the repo is registered through the same +loop. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from sieval.community.bfcl_v3 import GOLDLESS_CATEGORIES +from sieval.datasets._bfcl_v3 import load_categories + +_ROOT = Path(__file__).parents[3] + + +def _write(root, category, rows, gold=None): + (root / f"BFCL_v3_{category}.json").write_text( + "\n".join(json.dumps(r) for r in rows), encoding="utf-8" + ) + if gold is not None: + answers = root / "possible_answer" + answers.mkdir(exist_ok=True) + (answers / f"BFCL_v3_{category}.json").write_text( + "\n".join(json.dumps(g) for g in gold), encoding="utf-8" + ) + + +def test_gold_is_joined_onto_its_prompt_row(tmp_path): + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [[{"role": "user", "content": "hi"}]], + "function": [{"name": "f"}], + } + ], + gold=[{"id": "simple_0", "ground_truth": [{"f": {"a": [1]}}]}], + ) + row = load_categories(str(tmp_path), {"simple": 1})["test"][0] + assert row["id"] == "simple_0" + assert row["language"] == "Python" + assert json.loads(row["ground_truth"]) == [{"f": {"a": [1]}}] + assert row["question"] == [{"role": "user", "content": "hi"}] + + +def test_a_goldless_category_needs_no_answer_file(tmp_path): + assert "irrelevance" in GOLDLESS_CATEGORIES + _write( + tmp_path, + "irrelevance", + [ + { + "id": "irrelevance_0", + "question": [[{"role": "user", "content": "hi"}]], + "function": [{"name": "f"}], + } + ], + ) + row = load_categories(str(tmp_path), {"irrelevance": 1})["test"][0] + assert row["ground_truth"] is None + + +def test_a_goldless_category_stays_goldless_beside_one_that_has_gold(tmp_path): + """The gold table is per-category, and the universal indices of two + categories overlap -- `irrelevance_0` and `simple_0` both key on 0. A gold + table built once for the whole call would hand every goldless row its + neighbour's answer, which reads downstream as a real gold rather than as an + error: the sample TypedDict documents `None` as "this row has no gold". + """ + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [[{"role": "user", "content": "set it to 50"}]], + "function": [{"name": "set_volume", "parameters": {}}], + } + ], + gold=[{"id": "simple_0", "ground_truth": [{"set_volume": {"volume": [50]}}]}], + ) + _write( + tmp_path, + "irrelevance", + [ + { + "id": "irrelevance_0", + "question": [[{"role": "user", "content": "how are you"}]], + "function": [{"name": "set_volume", "parameters": {}}], + } + ], + ) + + rows = list(load_categories(str(tmp_path), {"simple": 1, "irrelevance": 1})["test"]) + + # Every category's rows, not just the last one's. + assert [row["id"] for row in rows] == ["simple_0", "irrelevance_0"] + by_id = {row["id"]: row for row in rows} + assert json.loads(by_id["simple_0"]["ground_truth"]) == [ + {"set_volume": {"volume": [50]}} + ] + assert by_id["irrelevance_0"]["ground_truth"] is None + # `function` is the tool schema the whole grading path reads, and is declared + # `str` because pyarrow cannot type its nesting. + for row in rows: + assert isinstance(row["function"], str) + assert json.loads(row["function"]) == [{"name": "set_volume", "parameters": {}}] + + +def test_java_and_javascript_carry_their_language(tmp_path): + for category, language in (("java", "Java"), ("javascript", "JavaScript")): + _write( + tmp_path, + category, + [ + { + "id": f"{category}_0", + "question": [[{"role": "user", "content": "x"}]], + "function": [{"name": "f"}], + } + ], + gold=[{"id": f"{category}_0", "ground_truth": [{"f": {}}]}], + ) + row = load_categories(str(tmp_path), {category: 1})["test"][0] + assert row["language"] == language + + +def test_a_moved_row_count_raises(tmp_path): + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [[{"role": "user", "content": "hi"}]], + "function": [], + } + ], + gold=[{"id": "simple_0", "ground_truth": []}], + ) + with pytest.raises(ValueError, match="expected 400"): + load_categories(str(tmp_path), {"simple": 400}) + + +def test_a_moved_gold_row_count_raises(tmp_path): + """Upstream asserts prompt and gold are the same length. Guarding only the + prompt side would let a re-upload that adds gold rows through: the loader + joins on the index, so the extra rows are simply never looked up and the + column is rescored with nothing to show for it. + """ + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [[{"role": "user", "content": "hi"}]], + "function": [], + } + ], + gold=[{"id": f"simple_{i}", "ground_truth": []} for i in range(3)], + ) + with pytest.raises(ValueError, match="3 possible_answer rows"): + load_categories(str(tmp_path), {"simple": 1}) + + +def test_a_non_numeric_universal_index_raises(tmp_path): + """The id shape moving while the counts hold is exactly what a pin bump can + do, and it must not surface as a bare `int()` failure with no category, row + or file in it. + """ + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0a", + "question": [[{"role": "user", "content": "hi"}]], + "function": [], + } + ], + gold=[{"id": "simple_0a", "ground_truth": []}], + ) + with pytest.raises(ValueError, match="where its universal index should be") as e: + load_categories(str(tmp_path), {"simple": 1}) + assert "'simple_0a'" in str(e.value) + assert "'simple'" in str(e.value) + assert "'0a'" in str(e.value) + + +def test_a_multi_turn_row_is_refused(tmp_path): + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [ + [{"role": "user", "content": "a"}], + [{"role": "user", "content": "b"}], + ], + "function": [], + } + ], + gold=[{"id": "simple_0", "ground_truth": []}], + ) + with pytest.raises(ValueError, match="turns"): + load_categories(str(tmp_path), {"simple": 1}) + + +def test_gold_joins_on_the_universal_index_not_the_whole_id(tmp_path): + """Upstream sorts both files on the universal index and pairs them + positionally, so a row whose sub-indices disagree between the two files is + still graded. `live_multiple_1052-79-0` / `live_multiple_1052-279-0` is that + row at the pinned revision; joining on the whole id would drop it. + """ + _write( + tmp_path, + "live_multiple", + [ + { + "id": "live_multiple_1052-79-0", + "question": [[{"role": "user", "content": "volume 50"}]], + "function": [{"name": "set_volume"}], + } + ], + gold=[ + { + "id": "live_multiple_1052-279-0", + "ground_truth": [{"set_volume": {"volume": [50]}}], + } + ], + ) + row = load_categories(str(tmp_path), {"live_multiple": 1})["test"][0] + assert row["id"] == "live_multiple_1052-79-0" + assert json.loads(row["ground_truth"]) == [{"set_volume": {"volume": [50]}}] + + +def _two_rows(ids): + return [ + { + "id": row_id, + "question": [[{"role": "user", "content": row_id}]], + "function": [], + } + for row_id in ids + ] + + +def test_a_repeated_universal_index_in_the_prompt_file_raises(tmp_path): + """Two rows sharing the join key make the pairing ambiguous. Upstream cannot + see this -- it only checks that the two files are the same length. + + The gold file here is well-formed, so only the prompt-side guard can fire. + """ + _write( + tmp_path, + "live_multiple", + _two_rows(["live_multiple_0-1-0", "live_multiple_0-2-0"]), + gold=[ + {"id": "live_multiple_0-1-0", "ground_truth": []}, + {"id": "live_multiple_1-1-0", "ground_truth": []}, + ], + ) + with pytest.raises( + ValueError, match="shared by more than one prompt row" + ) as raised: + load_categories(str(tmp_path), {"live_multiple": 2}) + # The colliding index and its ids, not a bare count of excess rows. + assert "{0: ['live_multiple_0-1-0', 'live_multiple_0-2-0']}" in str(raised.value) + assert "1 universal index " in str(raised.value) + + +def test_a_repeated_universal_index_in_the_gold_file_raises(tmp_path): + """The mirror of the test above: the prompt file is well-formed and the + repeat is on the gold side, so only the possible_answer guard can fire. + """ + _write( + tmp_path, + "live_multiple", + _two_rows(["live_multiple_0-1-0", "live_multiple_1-1-0"]), + gold=[ + {"id": "live_multiple_0-1-0", "ground_truth": []}, + {"id": "live_multiple_0-2-0", "ground_truth": []}, + ], + ) + with pytest.raises(ValueError, match="shared by more than one possible_answer row"): + load_categories(str(tmp_path), {"live_multiple": 2}) + + +def test_the_dataset_registry_loads_without_the_bfcl_v3_extra(): + """Both loaders import `sieval.community.bfcl_v3` for two pure-data tables, + and `import_all_datasets()` imports every dataset module in a bare loop with + no per-module `except`. So anything that package pulls in at import time is a + hard requirement for *every* dataset in the repo -- `sieval dataset list` + included -- and tree-sitter, which only the vendored `parser.py` needs, is in + the optional `bfcl-v3` extra. + + Out of process with the extra blocked at `sys.meta_path`: this venv has + tree-sitter installed (CI's `INSTALL_GROUPS` carries `-G bfcl-v3` so the + Java/JavaScript decode path is exercised), so an in-process check would pass + with the eager import restored. + """ + child = f""" +import sys + +class _NoTreeSitter: + def find_spec(self, name, path=None, target=None): + if name == "tree_sitter" or name.startswith("tree_sitter."): + raise ModuleNotFoundError("No module named " + name) + return None + +sys.meta_path.insert(0, _NoTreeSitter()) + +import sieval + +# The editable install may name a different checkout than the tree this test +# was loaded from; without this the child would probe that one instead. +assert sieval.__file__.startswith({str(_ROOT)!r}), sieval.__file__ + +from sieval.core.datasets.meta import import_all_datasets + +import_all_datasets() +assert "sieval.datasets.bfcl_v3_non_live" in sys.modules +assert "sieval.datasets.bfcl_v3_live" in sys.modules +""" + inherited = os.environ.get("PYTHONPATH") + env = { + **os.environ, + "PYTHONPATH": os.pathsep.join( + [str(_ROOT), *([inherited] if inherited else [])] + ), + } + completed = subprocess.run( + [sys.executable, "-c", child], + capture_output=True, + text=True, + timeout=300, + check=False, + env=env, + ) + assert completed.returncode == 0, ( + "the dataset registry must import without the optional `bfcl-v3` " + f"extra:\n{completed.stderr}" + ) + + +def test_a_gold_miss_raises(tmp_path): + _write( + tmp_path, + "simple", + [ + { + "id": "simple_0", + "question": [[{"role": "user", "content": "hi"}]], + "function": [], + } + ], + gold=[{"id": "simple_99", "ground_truth": []}], + ) + with pytest.raises(ValueError, match="no possible_answer"): + load_categories(str(tmp_path), {"simple": 1}) diff --git a/tests/unit/scripts/test_check_layer_imports.py b/tests/unit/scripts/test_check_layer_imports.py index 3a171031..075b40f1 100644 --- a/tests/unit/scripts/test_check_layer_imports.py +++ b/tests/unit/scripts/test_check_layer_imports.py @@ -800,6 +800,29 @@ def test_scripts_tree_is_out_of_scope(self, tmp_path: Path): f = self._write(tmp_path, "scripts/x.py", "from ..y import z\n") assert _check_relative_scope(f, ast.parse(f.read_text())) == [] + def test_community_tree_is_not_exempt(self, tmp_path: Path): + # Vendored code gets no carve-out from this rule. A copy cannot keep + # upstream's absolute `bfcl_eval.*` import, so the line is rewritten + # either way and can be spelled absolutely for free; exempting the tree + # to allow `..x` would drop the rule over every future drop. + f = self._write( + tmp_path, + "sieval/community/bfcl_v3/type_convertor/java_type_converter.py", + "from ..type_mappings import JAVA_TYPE_CONVERSION\n", + ) + errors = _check_relative_scope(f, ast.parse(f.read_text())) + assert len(errors) == 1 + assert "cross-package relative import '..type_mappings'" in errors[0] + + def test_community_absolute_import_passes(self, tmp_path: Path): + """The spelling the vendored converters actually ship.""" + f = self._write( + tmp_path, + "sieval/community/bfcl_v3/type_convertor/java_type_converter.py", + "from sieval.community.bfcl_v3.type_mappings import JAVA_TYPE_CONVERSION\n", + ) + assert _check_relative_scope(f, ast.parse(f.read_text())) == [] + class TestCheckFileRelativeScopeIntegration: """Rule 3 wired into _check_file, and its interaction with rule 2.""" @@ -942,6 +965,22 @@ def test_level_1_undotted_subpackage_private_name_stays_exempt( ) assert _check_file(f) == [] + def test_community_gets_every_rule(self, tmp_path: Path): + # `community/` has no carve-out from any of the four rules. Preflight is + # the only surface that checks this tree at all — pre-commit's global + # `exclude` skips it for every hook — so a carve-out added here, in + # `_check_relative_scope`, or in the preflight wrapper's file list would + # take private-module protection with it. This pins both halves. + f = self._write( + tmp_path, + "sieval/community/bfcl_v3/type_convertor/java_type_converter.py", + "from ..type_mappings import JAVA_TYPE_CONVERSION\n" + "from sieval.core.utils import _private_helper\n", + ) + errors = _check_file(f) + assert any("cross-package relative import" in e for e in errors) + assert any("import of private name '_private_helper'" in e for e in errors) + class TestCheckSubpackageImports: """Sub-package boundary check — check 1 at dotted granularity. diff --git a/tests/unit/scripts/test_check_preflight.py b/tests/unit/scripts/test_check_preflight.py index 295d1b80..f3836c32 100644 --- a/tests/unit/scripts/test_check_preflight.py +++ b/tests/unit/scripts/test_check_preflight.py @@ -384,6 +384,68 @@ def test_fail_when_script_exits_nonzero(self, tmp_path: Path): assert "2 import-policy violation(s)" in results[0].message assert len(results[0].details) == 2 + def test_warning_noise_is_shown_but_not_counted(self, tmp_path: Path): + # `check_layer_imports.py` parses every file it is handed, and several + # vendored modules raise SyntaxWarning on parse — on every run, passing + # or failing. Counting raw stderr lines reported two real violations as + # twenty-six, which reads as an unfixable pile rather than a two-line + # regression. + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + (scripts_dir / "check_layer_imports.py").write_text("") + sieval_dir = tmp_path / "sieval" + sieval_dir.mkdir() + (sieval_dir / "example.py").write_text("") + + runner = PreflightRunner(project_root=tmp_path) + mock_result = subprocess.CompletedProcess( + args=[], + returncode=1, + stdout="", + stderr=( + "sieval/community/deepseek_math.py:192: SyntaxWarning: " + "invalid escape sequence '\\%'\n" + ' string = string.replace("\\%", "%")\n' + "sieval/core/x.py:7: core/ must not import tasks/ (sieval.tasks)\n" + ), + ) + side_effect = self._make_layers_side_effect(mock_result) + with patch("check_preflight.subprocess.run", side_effect=side_effect): + results = runner.check_imports() + + assert results[0].status == "FAIL" + assert "1 import-policy violation(s)" in results[0].message + assert results[0].details == [ + "sieval/core/x.py:7: core/ must not import tasks/ (sieval.tasks)" + ] + + def test_unrecognised_stderr_still_reported_on_failure(self, tmp_path: Path): + # If the checker's output shape ever moves, a "0 violation(s)" FAIL + # would read as reassuring noise. Fall back to the raw lines. + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + (scripts_dir / "check_layer_imports.py").write_text("") + sieval_dir = tmp_path / "sieval" + sieval_dir.mkdir() + (sieval_dir / "example.py").write_text("") + + runner = PreflightRunner(project_root=tmp_path) + mock_result = subprocess.CompletedProcess( + args=[], + returncode=1, + stdout="", + stderr="sieval/community/x.py:1: SyntaxWarning: invalid escape\n", + ) + side_effect = self._make_layers_side_effect(mock_result) + with patch("check_preflight.subprocess.run", side_effect=side_effect): + results = runner.check_imports() + + assert results[0].status == "FAIL" + assert "1 import-policy violation(s)" in results[0].message + assert results[0].details == [ + "sieval/community/x.py:1: SyntaxWarning: invalid escape" + ] + def test_fail_when_script_not_found(self, tmp_path: Path): # tmp_path has no scripts/ directory at all runner = PreflightRunner(project_root=tmp_path) diff --git a/tests/unit/tasks/bfcl_v3/__init__.py b/tests/unit/tasks/bfcl_v3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/tasks/bfcl_v3/conftest.py b/tests/unit/tasks/bfcl_v3/conftest.py new file mode 100644 index 00000000..1eed9320 --- /dev/null +++ b/tests/unit/tasks/bfcl_v3/conftest.py @@ -0,0 +1,193 @@ +"""Shared rows for the BFCL v3 task family. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json +import types + +import pytest + +from sieval.tasks.bfcl_v3._base import BfclV3LiveTask, BfclV3NonLiveTask +from tests.conftest import MockChatModel, MockDataset + + +@pytest.fixture +def simple_row(): + return { + "id": "simple_0", + "category": "simple", + "language": "Python", + "question": [{"role": "user", "content": "Area of a 10x5 triangle?"}], + "function": json.dumps( + [ + { + "name": "calculate_triangle_area", + "description": "Area of a triangle.", + "parameters": { + "type": "dict", + "properties": { + "base": {"type": "integer", "description": "base"}, + "height": {"type": "integer", "description": "height"}, + }, + "required": ["base", "height"], + }, + } + ] + ), + "ground_truth": json.dumps( + [{"calculate_triangle_area": {"base": [10], "height": [5]}}] + ), + } + + +@pytest.fixture +def dotted_row(simple_row): + """A function name containing a dot -- the FC/Prompt divergence probe.""" + row = dict(simple_row) + functions = json.loads(row["function"]) + functions[0]["name"] = "geometry.triangle_area" + row["function"] = json.dumps(functions) + row["ground_truth"] = json.dumps( + [{"geometry.triangle_area": {"base": [10], "height": [5]}}] + ) + return row + + +@pytest.fixture +def java_row(simple_row): + """A `java` row -- the cohort upstream's schema preprocessing rewrites. + + 100 non-live rows are Java and 50 more are JavaScript; on those, upstream + restates every parameter as a `string` before the schema reaches the model, + because the model is being asked for Java source rather than for JSON. + """ + row = dict(simple_row) + row["id"] = "java_0" + row["category"] = "java" + row["language"] = "Java" + functions = json.loads(row["function"]) + functions[0]["parameters"]["properties"]["tags"] = { + "type": "ArrayList", + "description": "tags", + "items": {"type": "String"}, + } + row["function"] = json.dumps(functions) + return row + + +@pytest.fixture +def system_turn_row(simple_row): + """A row that already opens with its own system turn. + + 92 live rows do. Upstream merges the schema block into that turn; a port + that prepends a second one instead sends a two-system-turn conversation no + upstream run ever produced. + """ + row = dict(simple_row) + row["id"] = "live_simple_0" + row["category"] = "live_simple" + row["question"] = [ + {"role": "system", "content": "You are a geometry tutor."}, + {"role": "user", "content": "Area of a 10x5 triangle?"}, + ] + return row + + +@pytest.fixture +def irrelevance_row(simple_row): + row = dict(simple_row) + row["id"] = "irrelevance_0" + row["category"] = "irrelevance" + row["ground_truth"] = None + return row + + +# -------------------------------------------------------------------------- +# The report. `report` reads one thing off a finalized context -- its judgement +# record -- so these stand in for one rather than driving a runner. +# -------------------------------------------------------------------------- + + +def _ctx(category: str, correct: bool, index: int): + return types.SimpleNamespace( + feedback_result={ + "reference": None, + "rollouts": [{"index": 0, "correct": correct}], + "n_rollouts": 1, + "n_correct": int(correct), + "extra": {"category": category, "id": f"{category}_{index}"}, + } + ) + + +def _finals(spec: dict[str, tuple[int, int]]) -> list: + """One context per sample that came back; the first *n_correct* are right. + + *spec* maps a category to `(n_returned, n_correct)`. A category returning + fewer rows than it declares is how a pipeline failure reaches the report: + the failed samples are simply absent, with nothing naming their category. + """ + return [ + _ctx(category, index < correct, index) + for category, (returned, correct) in spec.items() + for index in range(returned) + ] + + +def _group_task(cls): + """A group base with no protocol mixin -- `report` reads neither hook.""" + return cls(MockDataset(), MockChatModel()) + + +#: Every non-live category complete, and the three `simple_ast` members at +#: DIFFERENT rates: a pooled rate over their 550 rows is 72.7, an unweighted +#: mean of the three is 33.3, so the two rollup rules cannot be confused. +_NON_LIVE_FULL = { + "simple": (400, 400), + "multiple": (200, 200), + "parallel": (200, 200), + "parallel_multiple": (200, 200), + "java": (100, 0), + "javascript": (50, 0), + "irrelevance": (240, 240), +} + +#: Every live category complete, at six rates that are deliberately unequal and +#: none of them 0 or 1. Unequal, so a weighted rollup (63.3) and an unweighted +#: one (41.1) are far apart; strictly interior, so every interval comes off the +#: Wilson branch rather than a saturated-set fallback. +_LIVE_SPREAD = { + "live_simple": (258, 129), + "live_multiple": (1053, 843), + "live_parallel": (16, 4), + "live_parallel_multiple": (24, 6), + "live_irrelevance": (882, 441), + "live_relevance": (18, 3), +} + + +@pytest.fixture +async def non_live_report(): + return await _group_task(BfclV3NonLiveTask).report(_finals(_NON_LIVE_FULL), []) + + +@pytest.fixture +async def non_live_report_short(): + """One of `simple`'s 400 rows never came back, and no other category ran.""" + return await _group_task(BfclV3NonLiveTask).report( + _finals({"simple": (399, 399)}), [] + ) + + +@pytest.fixture +async def non_live_report_with_fails(): + """The same run, with the missing row present as a FAILED sample.""" + return await _group_task(BfclV3NonLiveTask).report( + _finals({"simple": (399, 399)}), [object()] + ) + + +@pytest.fixture +async def live_report(): + return await _group_task(BfclV3LiveTask).report(_finals(_LIVE_SPREAD), []) diff --git a/tests/unit/tasks/bfcl_v3/test__base.py b/tests/unit/tasks/bfcl_v3/test__base.py new file mode 100644 index 00000000..a3eec168 --- /dev/null +++ b/tests/unit/tasks/bfcl_v3/test__base.py @@ -0,0 +1,408 @@ +"""Cover the BFCL v3 shared grading entry point and the two protocol mixins. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json + +import pytest + +from sieval.core.models import FunctionToolCall, ModelMeta, ModelOutput +from sieval.tasks.bfcl_v3._base import ( + BfclV3FCMixin, + BfclV3PromptMixin, + grade_single_turn, +) +from sieval.tasks.bfcl_v3.bfcl_v3_non_live_0shot_gen import BfclV3NonLiveZeroShotGenTask +from sieval.tasks.bfcl_v3.bfcl_v3_non_live_0shot_gen_fc import ( + BfclV3NonLiveZeroShotGenFCTask, +) +from tests.conftest import MockChatModel, MockDataset + +_META: ModelMeta = {"model": "mock-chat", "api_base": None, "default_params": {}} + + +def _output(text: str = "", tool_calls=None) -> ModelOutput: + return ModelOutput(model=_META, texts=[text], tool_calls=tool_calls) + + +def test_a_correct_call_grades_true(simple_row): + decoded = [{"calculate_triangle_area": {"base": 10, "height": 5}}] + assert ( + grade_single_turn( + simple_row["function"], + decoded, + simple_row["ground_truth"], + "Python", + "simple", + False, + ) + is True + ) + + +def test_a_wrong_argument_grades_false(simple_row): + decoded = [{"calculate_triangle_area": {"base": 10, "height": 99}}] + assert ( + grade_single_turn( + simple_row["function"], + decoded, + simple_row["ground_truth"], + "Python", + "simple", + False, + ) + is False + ) + + +def test_a_failed_decode_grades_false_for_an_ast_category(simple_row): + assert ( + grade_single_turn( + simple_row["function"], + None, + simple_row["ground_truth"], + "Python", + "simple", + False, + ) + is False + ) + + +# Both irrelevance sets, because they are separate keys in the polarity table +# and a typo in one of them is invisible while the other is tested. +@pytest.mark.parametrize("category", ["irrelevance", "live_irrelevance"]) +def test_irrelevance_is_correct_when_no_call_is_produced(irrelevance_row, category): + assert ( + grade_single_turn( + irrelevance_row["function"], None, None, "Python", category, False + ) + is True + ) + assert ( + grade_single_turn( + irrelevance_row["function"], [], None, "Python", category, False + ) + is True + ) + + +@pytest.mark.parametrize("category", ["irrelevance", "live_irrelevance"]) +def test_irrelevance_is_wrong_when_a_call_is_produced(irrelevance_row, category): + decoded = [{"calculate_triangle_area": {"base": 10, "height": 5}}] + assert ( + grade_single_turn( + irrelevance_row["function"], decoded, None, "Python", category, False + ) + is False + ) + + +def test_live_relevance_inverts_irrelevance(irrelevance_row): + decoded = [{"calculate_triangle_area": {"base": 10, "height": 5}}] + assert ( + grade_single_turn( + irrelevance_row["function"], + decoded, + None, + "Python", + "live_relevance", + False, + ) + is True + ) + assert ( + grade_single_turn( + irrelevance_row["function"], None, None, "Python", "live_relevance", False + ) + is False + ) + + +def test_underscore_to_dot_reconciles_the_fc_tool_name(dotted_row): + """FC sent `geometry_triangle_area`; gold says `geometry.triangle_area`.""" + fc_decoded = [{"geometry_triangle_area": {"base": 10, "height": 5}}] + prompt_decoded = [{"geometry.triangle_area": {"base": 10, "height": 5}}] + + assert ( + grade_single_turn( + dotted_row["function"], + fc_decoded, + dotted_row["ground_truth"], + "Python", + "simple", + True, + ) + is True + ) + assert ( + grade_single_turn( + dotted_row["function"], + prompt_decoded, + dotted_row["ground_truth"], + "Python", + "simple", + False, + ) + is True + ) + # And the flag is load-bearing in both directions, which is what makes + # passing it through the process boundary necessary rather than tidy. + assert ( + grade_single_turn( + dotted_row["function"], + fc_decoded, + dotted_row["ground_truth"], + "Python", + "simple", + False, + ) + is False + ) + + +def test_a_missing_gold_raises_rather_than_scoring_the_row(simple_row): + """A value-reference row with no gold fails; it is never graded as wrong. + + The distinction is the whole point: scoring it `False` would charge the + model for a dataset fault, and under `DENOMINATOR_REQUESTED` the two read + the same in the headline. Reached only past the produced-call guard, so the + call passed here has to be a valid one. + """ + with pytest.raises(ValueError, match="not goldless"): + grade_single_turn( + simple_row["function"], + [{"calculate_triangle_area": {"base": 10, "height": 5}}], + None, + "Python", + "simple", + False, + ) + + +# -------------------------------------------------------------------------- +# The protocol axis. Both mixins are stateless, so they are exercised directly +# rather than through a whole Task. +# -------------------------------------------------------------------------- + +#: Valid JavaScript that is ALSO valid Python, and decodes differently under +#: each. Deliberately not a snippet only one parser accepts: this one pins the +#: failure mode a hardcoded language actually has on real rows. +_JS_CALL = "[obj.method(opts={a: 1, b: 2})]" + + +def test_the_prompt_mixin_parses_under_the_row_language(): + """A JavaScript row must reach the JavaScript parser, not Python's `ast`. + + The failure this pins is the quiet one: `_JS_CALL` parses under BOTH + languages, and Python's parser resolves the object literal into a real + dict where JavaScript's keeps it as the source string. Hardcoding + `"Python"` therefore yields a plausible wrong value rather than a raise -- + a wrong score with nothing in the record to show for it. + """ + decoded = BfclV3PromptMixin()._decode(_output(_JS_CALL), "JavaScript") + assert decoded == [{"obj.method": {"opts": "{a: 1, b: 2}"}}] + # The same text under Python -- what a hardcoded language would return. + assert BfclV3PromptMixin()._decode(_output(_JS_CALL), "Python") == [ + {"obj.method": {"opts": {"a": 1, "b": 2}}} + ] + + +def test_the_prompt_mixin_puts_the_schemas_in_a_system_turn_and_sends_no_tools( + dotted_row, +): + messages, tools = BfclV3PromptMixin()._build_messages(dotted_row) + assert tools is None + assert messages[0]["role"] == "system" + # Verbatim, dot included: the Prompt protocol never renames anything, which + # is why its `UNDERSCORE_TO_DOT` is False. + assert "geometry.triangle_area" in messages[0]["content"] + assert messages[1:] == dotted_row["question"] + + +def test_the_prompt_mixin_merges_into_a_row_that_already_has_a_system_turn( + system_turn_row, +): + """Upstream merges; prepending a second system turn is a different payload. + + 92 live rows open with their own system turn. What makes this worth a test + of its own is that the wrong shape is not visibly wrong: two system turns + is a conversation every provider accepts and answers, so the run completes + and the score is merely somewhat off. + """ + messages, _ = BfclV3PromptMixin()._build_messages(system_turn_row) + assert [m["role"] for m in messages] == ["system", "user"] + content = messages[0]["content"] + assert "calculate_triangle_area" in content + # The row's own instruction survives, after the schema block rather than + # in a turn of its own. + assert content.endswith("You are a geometry tutor.") + + +def test_building_messages_leaves_the_stored_row_untouched(system_turn_row, java_row): + """Both vendored helpers rewrite their argument in place. + + Reached without a copy, they would edit the sample the runner holds, and + the damage is invisible on a first pass -- it surfaces only on the second + (a resume, a retry, `max_iterations > 1`), as a schema block appended twice + or a type already flattened. Idempotence is the discriminating check: one + call looks correct either way. + """ + for mixin in (BfclV3PromptMixin(), BfclV3FCMixin()): + for row in (system_turn_row, java_row): + before = json.dumps(row, sort_keys=True) + first = mixin._build_messages(row) + second = mixin._build_messages(row) + assert first == second + assert json.dumps(row, sort_keys=True) == before + + +def test_both_protocols_restate_java_parameters_as_strings(java_row): + """Upstream's language preprocessing, which both columns share. + + It is shared on purpose: the Prompt and FC numbers are two readings of one + benchmark only while the schemas they carry agree. So this asserts the same + property through both protocols rather than trusting one to stand for both. + """ + messages, _ = BfclV3PromptMixin()._build_messages(java_row) + prompt_text = messages[0]["content"] + _, tools = BfclV3FCMixin()._build_messages(java_row) + assert tools is not None + fc_properties = tools[0]["function"]["parameters"]["properties"] + + assert [v["type"] for v in fc_properties.values()] == ["string", "string", "string"] + assert "'type': 'string'" in prompt_text + # The Java 8 hint, and the element type carried over from the deleted + # `items` -- both protocols, because both call the same helper. + for rendered in (prompt_text, json.dumps(tools)): + assert "Java 8 SDK syntax" in rendered + assert "The list elements are of type String" in rendered + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "task_cls", [BfclV3NonLiveZeroShotGenTask, BfclV3NonLiveZeroShotGenFCTask] +) +async def test_preprocess_stores_the_unprocessed_schema_for_grading(task_cls, java_row): + """Preprocessing is model-facing only; the grader must see the real types. + + Upstream's `ast_file_runner` reads `function` back out of the dataset and + hands it to `ast_checker` unprocessed, so a port that stored the flattened + copy would compare a Java row's real `ArrayList` gold against a schema + claiming every parameter is a `string`. + """ + task = task_cls(MockDataset(), MockChatModel()) + record = await task.preprocess(java_row, None) + assert record["extra"]["function"] == java_row["function"] + assert json.loads(record["extra"]["function"])[0]["parameters"]["properties"][ + "tags" + ] == {"type": "ArrayList", "description": "tags", "items": {"type": "String"}} + + +class _Ctx: + preprocess_result = {"extra": {"language": "Python", "category": "simple"}} + + +@pytest.mark.anyio +async def test_an_undecodable_reply_is_scored_not_raised(): + """Upstream's `ast_decoder:decoder_failed` -- the model's outcome.""" + task = BfclV3NonLiveZeroShotGenTask(MockDataset(), MockChatModel()) + reply = _output("this is not a function call at all") + record = await task.postprocess(reply, _Ctx()) + (rollout,) = record["rollouts"] + assert rollout.get("prediction") is None + assert rollout["extracted"] is False + assert "decode_error" in rollout["extra"] + + +@pytest.mark.anyio +async def test_a_missing_parser_dependency_propagates_instead_of_scoring(monkeypatch): + """An absent optional extra is a broken environment, not a bad answer. + + `_decode` defers its import of the vendored parser (tree-sitter lives + behind the `bfcl-v3` extra), so the `ModuleNotFoundError` is raised from + inside the same `try` that turns a decode failure into a score. Swallowed, + it grades every row of the run wrong with `fails` at 0 -- indistinguishable + from a model that cannot call a function, and the logs go with the run. + """ + task = BfclV3NonLiveZeroShotGenTask(MockDataset(), MockChatModel()) + + def no_tree_sitter(_self, _output, _language): + raise ModuleNotFoundError("No module named 'tree_sitter'") + + monkeypatch.setattr(BfclV3PromptMixin, "_decode", no_tree_sitter) + with pytest.raises(ModuleNotFoundError, match="tree_sitter"): + await task.postprocess(_output("[f(x=1)]"), _Ctx()) + + +def test_the_fc_mixin_sends_tools_and_renames_the_dotted_function(dotted_row): + """Upstream's own rewrite -- the one `UNDERSCORE_TO_DOT` later reverses.""" + messages, tools = BfclV3FCMixin()._build_messages(dotted_row) + assert messages == dotted_row["question"] + assert tools is not None + assert [tool["function"]["name"] for tool in tools] == ["geometry_triangle_area"] + + +def test_the_fc_mixin_ignores_the_language(): + """No source text is parsed, so the row's language changes nothing.""" + output = _output( + tool_calls=(FunctionToolCall("c0", "calculate_triangle_area", '{"base": 10}'),) + ) + expected = [{"calculate_triangle_area": {"base": 10}}] + assert BfclV3FCMixin()._decode(output, "Java") == expected + assert BfclV3FCMixin()._decode(output, "Python") == expected + + +def test_the_fc_mixin_reads_no_calls_as_an_empty_list(): + """`[]` is "decoded fine, called nothing" -- correct for irrelevance. + + Distinct from a raise, which means undecodable. Both score wrong for an AST + category, but they are different upstream outcomes. + """ + assert BfclV3FCMixin()._decode(_output("no thanks"), "Python") == [] + + +@pytest.mark.parametrize("arguments", ["", " ", {}]) +def test_the_fc_mixin_reads_an_absent_argument_payload_as_no_arguments(arguments): + """A zero-parameter tool is a real shape, and `json.loads("")` raises.""" + output = _output(tool_calls=(FunctionToolCall("c0", "list_all", arguments),)) + assert BfclV3FCMixin()._decode(output, "Python") == [{"list_all": {}}] + + +def test_the_two_protocols_disagree_on_a_dotted_name_and_each_flag_reconciles_it( + dotted_row, +): + """The end-to-end reason the flag exists, across both mixins at once.""" + prompt_decoded = BfclV3PromptMixin()._decode( + _output("[geometry.triangle_area(base=10, height=5)]"), "Python" + ) + fc_decoded = BfclV3FCMixin()._decode( + _output( + tool_calls=( + FunctionToolCall( + "c0", + "geometry_triangle_area", + json.dumps({"base": 10, "height": 5}), + ), + ) + ), + "Python", + ) + assert prompt_decoded != fc_decoded + + for decoded, mixin in ( + (prompt_decoded, BfclV3PromptMixin), + (fc_decoded, BfclV3FCMixin), + ): + assert ( + grade_single_turn( + dotted_row["function"], + decoded, + dotted_row["ground_truth"], + "Python", + "simple", + mixin.UNDERSCORE_TO_DOT, + ) + is True + ) diff --git a/tests/unit/tasks/bfcl_v3/test_bfcl_v3_family.py b/tests/unit/tasks/bfcl_v3/test_bfcl_v3_family.py new file mode 100644 index 00000000..45bc435a --- /dev/null +++ b/tests/unit/tasks/bfcl_v3/test_bfcl_v3_family.py @@ -0,0 +1,158 @@ +"""One contract over the four registered BFCL v3 leaves. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from types import SimpleNamespace + +import pytest + +from sieval.core.tasks.meta import get_task_class, get_task_meta +from sieval.tasks.bfcl_v3._base import BfclV3Task +from tests.conftest import MockChatModel, MockDataset + +NAMES = ( + "bfcl_v3_non_live_0shot_gen", + "bfcl_v3_non_live_0shot_gen_fc", + "bfcl_v3_live_0shot_gen", + "bfcl_v3_live_0shot_gen_fc", +) + + +@pytest.mark.parametrize("name", NAMES) +def test_every_leaf_resolves_by_name(name): + """A nested task that imports but does not RESOLVE is the failure mode the + subpackage rule exists for -- a bare KeyError from `sieval task show`.""" + assert get_task_class(name) is not None + + +@pytest.mark.parametrize("name", NAMES) +def test_every_leaf_declares_the_metadata_the_rules_require(name): + meta = get_task_meta(get_task_class(name)) + assert meta.model_type == "chat" + assert meta.reference_kind == "value" + assert meta.n_shot == 0 + # `reference_impl` is optional on `TaskMeta`, so the narrowing is an + # assertion in its own right: a leaf that shipped without one would + # otherwise fail on an attribute error rather than on the thing under test. + assert meta.reference_impl is not None + assert meta.reference_impl.url.endswith( + "ea13468e4423454d0c213704fb87cf7cb3990433/berkeley-function-call-leaderboard" + ) + + +@pytest.mark.parametrize( + ("name", "expected"), + [(n, n.endswith("_fc")) for n in NAMES], +) +def test_only_the_fc_leaves_demand_the_tools_capability(name, expected): + # Direct attribute access, not `getattr(..., False)`: `requires` is a + # `ClassVar[TaskRequirements]` that every task inherits, so a default would + # turn Task 1's field going missing into a silently passing assertion for + # the two Prompt leaves. + cls = get_task_class(name) + assert cls.requires.function_tools is expected + + +@pytest.mark.parametrize( + ("name", "expected"), + [(n, None if n.endswith("_fc") else "bfcl-v3") for n in NAMES], +) +def test_only_the_prompt_leaves_declare_the_tree_sitter_extra(name, expected): + """The extra follows the protocol, and the split is not the obvious one. + + Both Prompt leaves need it -- including the live one, whose every category + is Python -- because `ast_parse`'s module imports both tree-sitter source + parsers at module scope, so the cost is paid before the language dispatch + runs. Neither FC leaf does: it reads structured calls and parses no source. + + Asserted in both directions, because both are silent in production. Dropping + it from a Prompt leaf makes readiness report `yes` on a base install, and + since the import is deferred into `_decode`, the run bills for inference + before every sample dies at postprocess. Adding it to an FC leaf is the + quieter fault: that leaf then reports NOT ready, and stays unrunnable, over + a dependency it never loads. + """ + assert get_task_meta(get_task_class(name)).deps_group == expected + + +class _ToollessChatModel(MockChatModel): + """A chat model whose runtime plan offers no `function_tools`. + + Not reachable through a registered dialect today -- the only one that denies + the capability is `openai_completions`, whose completion input kind trips a + different check first -- so the runtime-plan branch is how the gate is + exercised at all. + """ + + @property + def runtime_plan(self): + return SimpleNamespace( + dialect_id="openai_chat", + available_capabilities=frozenset(), + capability_minimums={}, + ) + + +def test_an_fc_leaf_is_refused_at_construction_when_tools_are_unavailable(): + """What the FC leaves' notes claim, asserted where it happens. + + The refusal is `Task.__init__`'s, so it lands at prelaunch reconciliation -- + before a request is made and before a result directory exists. Static schema + validation does NOT see it: `validate_eval_config` runs without importing or + instantiating a task, so a config naming an FC leaf passes it clean. + """ + with pytest.raises(ValueError, match="function_tools"): + get_task_class("bfcl_v3_live_0shot_gen_fc")(MockDataset(), _ToollessChatModel()) + + +def test_the_prompt_leaves_construct_on_the_same_toolless_model(): + """The other half of the pair: without it, a gate that refused EVERY model + would pass the test above just as happily.""" + get_task_class("bfcl_v3_live_0shot_gen")(MockDataset(), _ToollessChatModel()) + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("bfcl_v3_non_live_0shot_gen", "bfcl_v3_non_live"), + ("bfcl_v3_non_live_0shot_gen_fc", "bfcl_v3_non_live"), + ("bfcl_v3_live_0shot_gen", "bfcl_v3_live"), + ("bfcl_v3_live_0shot_gen_fc", "bfcl_v3_live"), + ], +) +def test_the_two_groups_bind_different_datasets(name, expected): + """The FK is not declared -- `@sieval_task` resolves it by reverse lookup + from the sample `TypedDict` the leaf binds, so this is what proves the two + groups really did bind different sample types rather than both inheriting + one.""" + assert get_task_meta(get_task_class(name)).dataset == expected + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("bfcl_v3_non_live_0shot_gen", "non_live"), + ("bfcl_v3_non_live_0shot_gen_fc", "non_live"), + ("bfcl_v3_live_0shot_gen", "live"), + ("bfcl_v3_live_0shot_gen_fc", "live"), + ], +) +def test_each_leaf_rolls_up_under_its_own_group(name, expected): + """Which group base a leaf inherits is a separate fact from which dataset it + binds, and only this one is about the arithmetic. + + The two bases differ in how the headline is computed -- the non-live one + takes an unweighted mean of five category rates and publishes no interval, + the live one pools over rows and does -- and `GROUP_KEY` is what selects the + rollup and names `score_key`. Nothing else here would notice a leaf wired to + the wrong base while still binding the right sample type: the dataset FK + comes from the generic arg, the capability from the protocol mixin, and the + rollups are exercised against the bases rather than through the leaves. A + live leaf on the non-live base would publish a mean of rates over live data + under a live-sounding score key, and pass everything else. + """ + cls = get_task_class(name) + assert issubclass(cls, BfclV3Task) + group_key = cls.GROUP_KEY + assert group_key == expected diff --git a/tests/unit/tasks/bfcl_v3/test_report.py b/tests/unit/tasks/bfcl_v3/test_report.py new file mode 100644 index 00000000..4a7c8139 --- /dev/null +++ b/tests/unit/tasks/bfcl_v3/test_report.py @@ -0,0 +1,172 @@ +"""Cover the two BFCL v3 group rollups: what each mean is over, and what carries +an interval. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import pytest + +from sieval.community.bfcl_v3 import ( + calculate_unweighted_accuracy, + calculate_weighted_accuracy, +) +from sieval.core.tasks.metrics import ( + DENOMINATOR_FIELD, + DENOMINATOR_REQUESTED, + SCORE_KEY_FIELD, +) + + +def test_unweighted_is_the_mean_of_rates_not_the_pooled_rate(): + # 1/1 and 0/99: the pooled rate is 0.01, the unweighted mean is 0.5. + # Every cell carries all THREE keys upstream reads, `display_accuracy` + # included -- both helpers dereference it unconditionally, so a two-key + # cell raises `KeyError` rather than returning a wrong number. + cells = [ + {"accuracy": 1.0, "total_count": 1, "display_accuracy": 1.0}, + {"accuracy": 0.0, "total_count": 99, "display_accuracy": 0.0}, + ] + assert calculate_unweighted_accuracy(cells)["accuracy"] == pytest.approx(0.5) + assert calculate_weighted_accuracy(cells)["accuracy"] == pytest.approx(0.01) + + +@pytest.mark.anyio +async def test_non_live_rollup_nests_simple_ast(non_live_report): + # Percentage points throughout: simple 100 (400 rows), java 0 (100), + # javascript 0 (50) -> simple_ast = (100 + 0 + 0) / 3 + assert non_live_report["simple_ast"] == pytest.approx(100 / 3) + # ast_summary = mean(simple_ast, multiple, parallel, parallel_multiple) + assert non_live_report["ast_summary"] == pytest.approx( + (100 / 3 + 100.0 + 100.0 + 100.0) / 4 + ) + # overall additionally folds in irrelevance + assert non_live_report["non_live_overall_acc"] == pytest.approx( + (100 / 3 + 100.0 + 100.0 + 100.0 + 100.0) / 5 + ) + assert non_live_report["score"] == non_live_report["non_live_overall_acc"] + + +@pytest.mark.anyio +async def test_non_live_headline_carries_no_interval(non_live_report): + for key in ( + "score_ci95", + "non_live_overall_acc_ci95", + "simple_ast_ci95", + "ast_summary_ci95", + ): + assert key not in non_live_report, ( + f"{key} is an unweighted mean of category rates; an interval on it " + "would bracket a different statistic than the number beside it." + ) + # And the population key that would have to sit beside such an interval is + # absent too -- the pair is emitted whole or not at all. + assert "n_problems" not in non_live_report + + +@pytest.mark.anyio +async def test_non_live_per_category_rates_all_carry_the_triple(non_live_report): + units = non_live_report["ci95_units"] + for category in ( + "simple", + "multiple", + "parallel", + "parallel_multiple", + "java", + "javascript", + "irrelevance", + ): + assert f"{category}_ci95" in non_live_report + assert f"n_{category}" in non_live_report + assert units[category] == f"n_{category}" + + +@pytest.mark.anyio +async def test_live_headline_carries_an_interval(live_report): + assert "score_ci95" in live_report + assert "n_problems" in live_report + assert live_report["ci95_units"]["score"] == "n_problems" + assert "ast_summary_ci95" in live_report + # The headline and its aliased twin are ONE number, carrying ONE interval. + assert live_report["live_overall_acc"] == live_report["score"] + assert live_report["live_overall_acc_ci95"] == live_report["score_ci95"] + assert live_report["ci95_units"]["live_overall_acc"] == "n_problems" + + +@pytest.mark.anyio +async def test_live_rollups_are_pooled_over_rows_not_means_of_rates(live_report): + """Weighted, so both rollups are the rate over the union of their rows. + + The six categories run from 16.7 to 80.1 at sizes from 16 to 1053, so the + unweighted mean of them (41.1) is nowhere near the pooled rate -- which is + what makes the choice of helper visible in the number rather than only in + the source. + """ + assert live_report["live_overall_acc"] == pytest.approx(100 * 1426 / 2251) + assert live_report["ast_summary"] == pytest.approx(100 * 982 / 1351) + # `ast_summary` is over the four AST categories only: the two relevance sets + # join at the overall, and folding them in here would make it the headline. + assert live_report["n_ast"] == 1351 + assert live_report["n_problems"] == 2251 + assert live_report["ci95_units"]["ast_summary"] == "n_ast" + + +@pytest.mark.anyio +async def test_every_live_interval_brackets_the_rate_printed_beside_it(live_report): + """The published rate and its bounds have to be in the same units. + + Both come off the same per-sample list: the rate is that list's mean scaled + to percentage points, the bounds are estimated from it as a probability and + scaled the same way. Scale the list instead of the rate and the estimator + reads every rate above one point as a saturated set, which still publishes + a triple -- a plausible-looking interval that brackets 100 rather than the + number it is printed beside. + """ + units = live_report["ci95_units"] + assert units, "a live report with no declared interval proves nothing here" + for metric in units: + low, high = live_report[f"{metric}_ci95"] + assert low < live_report[metric] < high, ( + f"{metric} is {live_report[metric]} but its interval is [{low}, {high}]" + ) + assert 0.0 <= low < high <= 100.0 + + +@pytest.mark.anyio +async def test_a_missing_sample_scores_wrong_not_excluded(non_live_report_short): + # 399 of simple's 400 came back, all correct. The denominator is still 400. + assert non_live_report_short["simple"] == pytest.approx(100 * 399 / 400) + assert non_live_report_short["n_simple"] == 400 + + +@pytest.mark.anyio +async def test_a_failed_sample_moves_nothing_but_the_fail_count( + non_live_report_short, non_live_report_with_fails +): + """The denominator is DECLARED, so charging a failure is already done. + + A failed sample has no category to be attributed to -- one that failed in + `preprocess` never read a row -- and under this policy it does not need + one: it is missing from its category's numerator and the denominator never + moved. So `fails` is the only key that reacts to it. + """ + assert non_live_report_short["fails"] == 0 + assert non_live_report_with_fails["fails"] == 1 + assert non_live_report_with_fails["score"] == non_live_report_short["score"] + assert non_live_report_with_fails["simple"] == non_live_report_short["simple"] + + +@pytest.mark.anyio +async def test_both_groups_declare_the_column_and_the_population( + non_live_report, live_report +): + """`score_key` is built from the group key, so no static reader resolves it. + + It names a key the report writes, and the report is the only place that can + be checked -- which is why it is asserted here rather than left to the + preflight, whose rule 4 skips a computed key. + """ + for report, group in ((non_live_report, "non_live"), (live_report, "live")): + assert report[SCORE_KEY_FIELD] == f"{group}_overall_acc" + assert report[SCORE_KEY_FIELD] in report + assert report[report[SCORE_KEY_FIELD]] == report["score"] + assert report[DENOMINATOR_FIELD] == DENOMINATOR_REQUESTED diff --git a/tests/unit/tasks/test_import_discipline_family.py b/tests/unit/tasks/test_import_discipline_family.py index e84ef93d..d4dfedbe 100644 --- a/tests/unit/tasks/test_import_discipline_family.py +++ b/tests/unit/tasks/test_import_discipline_family.py @@ -32,6 +32,17 @@ # task module (under `sieval.tasks.`) -> modules its registration must not pull. FORBIDDEN: dict[str, tuple[str, ...]] = { + # `ast_parse` reaches tree-sitter, which is behind the optional `bfcl-v3` + # extra. It is needed only by the Prompt protocol -- but there by every + # category, not just java/javascript: `ast_parse` imports both source + # parsers at module scope, so touching it at all pulls them. All four + # leaves are listed because they share `_base.py`, so a module-scope import + # there would pull tree-sitter for the FC leaves too, which never parse + # source text at all. + "bfcl_v3.bfcl_v3_non_live_0shot_gen": ("tree_sitter",), + "bfcl_v3.bfcl_v3_non_live_0shot_gen_fc": ("tree_sitter",), + "bfcl_v3.bfcl_v3_live_0shot_gen": ("tree_sitter",), + "bfcl_v3.bfcl_v3_live_0shot_gen_fc": ("tree_sitter",), # math_verify is behind the `math` group and is slow to import. "aime_2024_0shot_gen": ("math_verify",), "aime_2025_0shot_gen": ("math_verify",),