feat(tasks): add BFCL v3 — single-turn function calling, both protocols

.github/workflows/ci.yml

@@ -27,9 +27,13 @@ env:
   # is tested at all. nltk alone is not enough -- the pre-2020 grader reaches
   # only nltk, so without sqlparse the test-suite metric and the hardened
   # execution under it are silently uncollected here while passing locally.
+  # `bfcl-v3` is installed rather than left out like `hle`/`multi-if`: it gates
+  # the java/javascript decode path, the one thing the vendored source-text
+  # parsers exist to make work, and an untested path silently rots. The cost is
+  # one small C extension (tree-sitter) built from sdist per run.
   INSTALL_GROUPS: >-
-    -G drop -G ifbench -G ifeval -G iheval -G math -G scicode -G spider -G test
-    -G dev
+    -G bfcl-v3 -G drop -G ifbench -G ifeval -G iheval -G math -G scicode -G spider
+    -G test -G dev
 
 jobs:
   changes:

pyproject.toml

@@ -36,6 +36,21 @@ Issues = "https://github.com/scitix/sieval/issues"
 Changelog = "https://github.com/scitix/sieval/blob/main/CHANGELOG.md"
 
 [project.optional-dependencies]
+bfcl-v3 = [
+  # Reached only by the Prompt protocol, and there by every category rather
+  # than just `java` and `javascript`: upstream's `ast_parse` dispatches those
+  # two to tree-sitter parsers rather than Python's `ast`, but it imports both
+  # source parsers at module scope, so touching `ast_parse` at all requires
+  # them. The FC protocol never parses source text and does not need this.
+  #
+  # Pinned to upstream's own versions. tree_sitter 0.21.3 predates the 0.22 API
+  # break -- the vendored parsers call `Language(ptr, name)` and
+  # `Parser.set_language()`, both removed later -- so this is a hard pin, not a
+  # floor. It has no wheel for Python 3.13 and builds from sdist.
+  "tree-sitter==0.21.3",
+  "tree-sitter-java==0.21.0",
+  "tree-sitter-javascript==0.21.4",
+]
 drop = ["numpy<=2.2", "scipy>=1.16.3"]
 # numpy backs the vendored HLE `calib_err` (calibration error). No grader/judge
 # extras: the judge is reached through the base `openai` client like any model.

scripts/check_layer_imports.py

@@ -456,6 +456,15 @@ def _check_relative_scope(path: Path, tree: ast.AST) -> list[str]:
     Scoped to the sieval package only: ``scripts/`` files are standalone modules,
     not a package, so a relative import there fails at runtime and needs no
     lint. (The pre-commit hook feeds both trees; this check narrows on purpose.)
+
+    ``sieval/community/`` is **not** exempt, and a vendored drop is not a reason
+    to make it one. A vendored file that reaches across packages has already had
+    that import rewritten — upstream's own was absolute (``from
+    bfcl_eval.constants...``), and a copy cannot keep it — so spelling the
+    replacement absolutely costs exactly what spelling it ``..x`` costs: one
+    line, already being edited. Exempting the tree to permit the relative form
+    would buy nothing and would drop this rule over every future drop, which is
+    a ratchet: lint exemptions are not narrowed back.
     """
     if _get_layer(path) is None:
         return []

scripts/check_preflight.py

@@ -1164,6 +1164,35 @@ def lock_drift(
     return report
 
 
+# Python's own warning output, which lands on the same stderr the import
+# checker writes violations to: `<file>:<lineno>: <Category>: <text>`.
+_PY_WARNING_LINE = re.compile(r":\d+: \w*Warning: ")
+
+
+def _import_violation_lines(stderr: str) -> list[str]:
+    """Return only the lines of *stderr* that are import-policy violations.
+
+    ``check_layer_imports.py`` parses every file it is handed, and several
+    vendored modules raise ``SyntaxWarning`` on parse. Those warnings land on
+    stderr on **every** run, passing or failing, so counting raw stderr lines
+    made two real violations read as twenty-six and turned a two-line
+    regression into an apparently unfixable pile.
+
+    A warning occupies two lines — the ``<file>:<lineno>: <Category>:`` header
+    and an indented echo of the offending source. Violations are always printed
+    flush-left, so dropping indented lines is safe and also handles a
+    multi-line warning body.
+    """
+    violations: list[str] = []
+    for line in stderr.strip().splitlines():
+        if not line.strip():
+            continue
+        if line[:1].isspace() or _PY_WARNING_LINE.search(line):
+            continue
+        violations.append(line)
+    return violations
+
+
 class PreflightRunner:
     """Orchestrates preflight checks."""
 
@@ -3047,13 +3076,18 @@ def check_imports(self) -> list[CheckResult]:
         #
         # KNOWN divergence, not parity: pre-commit additionally applies the
         # global `exclude: ^(sieval/community/|vendor/)`, so it skips
-        # `sieval/community/` while this wrapper checks it. Inert today (every
-        # relative import under `community/` is a bare level-1 `from . import
-        # x`), but a future vendored drop using `from ..x import y` would pass
-        # pre-commit and fail preflight, and the only offered fix would be to
-        # edit code kept byte-identical to upstream. Fixing it is a design call
-        # — hoisting the exemption into `_check_file` would also drop the
-        # private-access check's coverage of `community/`.
+        # `sieval/community/` for every hook while this wrapper checks it. That
+        # makes this wrapper the only enforcement `community/` gets, which is
+        # the reason to keep feeding it the tree rather than to carve it out.
+        #
+        # The vendored drop this comment used to predict has arrived — a file
+        # reaching across packages, where upstream's own import was absolute and
+        # the copy could not keep it. It is spelled absolutely, so the rule is
+        # satisfied without an exemption: rewriting that import was unavoidable
+        # either way, and the relative spelling was the only version that would
+        # have needed one. Do not add the carve-out here, in `_check_file`, or
+        # in `_check_relative_scope` — it would silently drop private-module
+        # protection over a tree nothing else checks.
         enforced_py = [
             f
             for f in self._git_tracked_files(".py")
@@ -3072,7 +3106,12 @@ def check_imports(self) -> list[CheckResult]:
         if result.returncode == 0:
             return [CheckResult("PASS", "check_imports", "no import-policy violations")]
 
-        errors = [line for line in result.stderr.strip().splitlines() if line.strip()]
+        errors = _import_violation_lines(result.stderr)
+        if not errors:
+            # Non-zero exit with nothing recognisable as a violation means the
+            # checker's output shape moved. Show the raw stderr rather than
+            # reporting a reassuring "0 violation(s)" on a failing run.
+            errors = [ln for ln in result.stderr.strip().splitlines() if ln.strip()]
         return [
             CheckResult(
                 "FAIL",

sieval/community/CLAUDE.md

@@ -26,7 +26,19 @@ inline where it would swamp a diff against upstream. Do not add more original
 code here without the same argument; a helper with one caller belongs in that
 caller's module.
 
+`bfcl_v3/_safe_eval.py` is the second, and with exactly **one** caller it shows
+what "the same argument" means. That caller is a *vendored* file
+(`bfcl_v3/parser.py`, whose `ast.BinOp` branch upstream resolves with `eval` over
+model output), so inlining the guard would both enlarge a diff that exists to be
+compared against upstream and put a security boundary inside a file that cannot
+be linted. "One caller" is the rule for ordinary helpers; a boundary that has to
+stay separately auditable is not one.
+
 The package-wide `ruff` / `mypy` / `pre-commit` exclusions exist to keep vendored
-code byte-identical and cover this file too, which is the wrong default for a
-security boundary. Until they are narrowed, lint it by hand:
-`ruff check --config 'exclude=["vendor"]' sieval/community/_sympy_guards.py`.
+code byte-identical and cover both files too, which is the wrong default for a
+security boundary. Until they are narrowed, lint them by hand:
+
+```
+ruff check --config 'exclude=["vendor"]' sieval/community/_sympy_guards.py
+ruff check --config 'exclude=["vendor"]' sieval/community/bfcl_v3/_safe_eval.py
+```

sieval/community/bfcl_v3/__init__.py

@@ -0,0 +1,90 @@
+"""BFCL v3 (single-turn) grader, vendored from ShishirPatil/gorilla.
+
+Source: https://github.com/ShishirPatil/gorilla
+Revision: v1.3 (ea13468e4423454d0c213704fb87cf7cb3990433)
+License: Apache-2.0
+
+Upstream's evaluation surface is `bfcl_eval`, which imports its own submodules
+by absolute path. A vendored copy cannot keep those imports, so rewriting them
+is the only edit made to any vendored file -- with one exception: `parser.py`'s
+`ast.BinOp` branch calls `_safe_eval.safe_eval` where upstream `eval`s a
+model-authored expression. Each vendored module records upstream's blob SHA at
+the pin and its exact edits, so the copy can be re-derived from a fresh checkout.
+
+Whole-file: `ast_checker.py`, `type_mappings.py`, `type_convertor/*.py`,
+`source_parser/*.py`, and `prompts.py` (upstream's `constants/default_prompts`).
+Partial -- named subsets of a larger upstream file, listed in each module's own
+docstring: `parser.py`, `tool_convert.py`, `output_checks.py`, `preprocess.py`,
+`aggregate.py`.
+
+First-party: `_model_config.py` supplies the `MODEL_CONFIG_MAPPING` global
+`ast_checker.convert_func_name` reads, since upstream's 2044-line registry has no
+entry for sieval's models. `_tables.py` holds the category tables shared by the
+dataset loader and the tasks, neither of which may reach into the other's private
+module. `_safe_eval.py` is the deviation `parser.py` carries, held outside that
+file so the boundary stays lintable -- vendored code is not.
+
+AI-Generated Code - Claude Opus 5 (Anthropic)
+"""
+
+from typing import TYPE_CHECKING
+
+from ._model_config import set_underscore_to_dot
+from ._tables import (
+    CALL_EXPECTED,
+    GOLDLESS_CATEGORIES,
+    LANGUAGE_BY_CATEGORY,
+    LIVE_COUNTS,
+    NON_LIVE_COUNTS,
+)
+from .aggregate import calculate_unweighted_accuracy, calculate_weighted_accuracy
+from .ast_checker import ast_checker
+from .output_checks import is_empty_output, is_function_calling_format_output
+from .preprocess import (
+    func_doc_language_specific_pre_processing,
+    system_prompt_pre_processing_chat_model,
+)
+from .prompts import DEFAULT_SYSTEM_PROMPT
+from .tool_convert import convert_to_tool
+
+if TYPE_CHECKING:
+    # A module `__getattr__` types *every* attribute of the package as its own
+    # return type, so `ast_parse` would resolve to `object` and every call site
+    # would read as calling a non-callable. An explicitly declared symbol takes
+    # precedence over the fallback; `TYPE_CHECKING` is False at runtime, so the
+    # import below stays deferred.
+    from .parser import ast_parse
+
+
+__all__ = [
+    "ast_checker",
+    "ast_parse",
+    "convert_to_tool",
+    "is_function_calling_format_output",
+    "is_empty_output",
+    "func_doc_language_specific_pre_processing",
+    "system_prompt_pre_processing_chat_model",
+    "calculate_weighted_accuracy",
+    "calculate_unweighted_accuracy",
+    "DEFAULT_SYSTEM_PROMPT",
+    "set_underscore_to_dot",
+    "NON_LIVE_COUNTS",
+    "LIVE_COUNTS",
+    "GOLDLESS_CATEGORIES",
+    "CALL_EXPECTED",
+    "LANGUAGE_BY_CATEGORY",
+]
+
+
+def __getattr__(name: str) -> object:
+    """`ast_parse` is deferred because `parser.py` is the only module here that
+    reaches tree-sitter, which lives in the optional `bfcl-v3` extra. Importing
+    it eagerly makes that extra a hard requirement for importing this package at
+    all -- and `sieval/datasets/` imports it for two pure-data tables, so the
+    whole dataset registry died on a base install.
+    """
+    if name == "ast_parse":
+        from .parser import ast_parse
+
+        return ast_parse
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

sieval/community/bfcl_v3/_model_config.py

@@ -0,0 +1,48 @@
+"""The `MODEL_CONFIG_MAPPING` global upstream's `ast_checker` expects.
+
+`convert_func_name` reads `MODEL_CONFIG_MAPPING[model].underscore_to_dot` to
+decide whether a gold function name's dots must be rewritten to underscores
+before comparison. Upstream carries a 2044-line registry of its own evaluated
+models to answer that one question. sieval's models are not in it, so a verbatim
+copy would `KeyError`.
+
+Rather than delete the call (removing a helper is as much a deviation as adding
+one), supply the global the file expects. The flag is a property of the
+*transport*, not of a model: it is True exactly when the function schemas
+travelled through an API whose tool names cannot contain `.`.
+
+AI-Generated Code - Claude Opus 5 (Anthropic)
+"""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class _ModelEntry:
+    underscore_to_dot: bool
+
+
+class _ModelConfigMapping:
+    """Answers for any model name, because the flag does not depend on one."""
+
+    def __init__(self) -> None:
+        self._underscore_to_dot = False
+
+    def set_underscore_to_dot(self, enabled: bool) -> None:
+        self._underscore_to_dot = enabled
+
+    def __getitem__(self, _model_name: str) -> _ModelEntry:
+        return _ModelEntry(underscore_to_dot=self._underscore_to_dot)
+
+
+MODEL_CONFIG_MAPPING = _ModelConfigMapping()
+
+
+def set_underscore_to_dot(enabled: bool) -> None:
+    """Set the flag for the protocol about to be graded.
+
+    Prompt protocol → False: schemas reach the model verbatim, dots survive.
+    FC protocol → True: `convert_to_tool` rewrote `.`→`_` on the way out, so
+    gold must be rewritten to match.
+    """
+    MODEL_CONFIG_MAPPING.set_underscore_to_dot(enabled)