diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 4f01ad6..8fe09ef 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -9,7 +9,13 @@ //! templates); //! * sentinel pipes leaked into the markers, in any position: //! `<|tool_call>…`, `<|tool_call|>…<|tool_call|>`, -//! `…`; +//! `…`, including the fullwidth `|` those templates +//! actually emit; +//! * a `DeepSeek` DSML marker on the tag itself, +//! `<|DSML|tool_call>…` — the same marker +//! [`super::invoke_xml`] already accepts on ``, which this family +//! used to miss, so a `deepseek` turn that chose the tag form over the +//! invoke form parsed as prose and the call was silently dropped; //! * a `call:` prefix before the body; //! * a fenced block instead of a tag, ```` ```tool_call … ``` ````, sometimes //! closed by a stray ``; @@ -39,11 +45,17 @@ use crate::types::{CallSource, ParseOptions, ParsedToolCall}; pub(crate) struct Tagged; /// Any tag-family marker: ``, ``, ``, with -/// pipes, a slash, or whitespace leaked in, and an optional attribute list. -/// `` (plural, a JSON key) and `` do not match: -/// the name must end at a pipe, slash, whitespace, or `>`. -static TAG_RE: LazyLock> = - LazyLock::new(|| Regex::new(r"(?i)<[|/\s]*tool[_-]?call(?:[|/\s]*|\s+[^>]*)>").ok()); +/// pipes (ASCII `|` or the fullwidth `|` chat templates emit), a slash, +/// whitespace, or a `DeepSeek` DSML marker leaked in, and an optional +/// attribute list. `` (plural, a JSON key, and the DSML wrapper +/// element) and `` do not match: the name must end at a pipe, +/// slash, whitespace, or `>`. +static TAG_RE: LazyLock> = LazyLock::new(|| { + Regex::new( + r"(?i)<[|\u{ff5c}/\s]*(?:DSML[|\u{ff5c}/\s]*)?tool[_-]?call(?:[|\u{ff5c}/\s]*|\s+[^>]*)>", + ) + .ok() +}); /// Openers a fenced block can carry. `` ```tool_calls `` (plural) is listed /// separately from `` ```tool_call `` rather than relying on a prefix match: diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 93bfbb6..aca11db 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -513,3 +513,63 @@ fn a_bare_trailing_opener_is_dropped_not_shown() { let (text, _) = parse("before not-json"); assert_eq!(text, "before not-json"); } + +// ── DeepSeek DSML on the tag family ───────────────────────────────────────── + +/// `deepseek` emits its DSML marker on the `tool_call` tag, not only on +/// ``, and with the fullwidth bar its template actually uses. +/// Observed live from `deepseek-v4-flash` driving the code dialect: the call +/// parsed as prose, so the turn ended with the model's lead-in and the tool +/// was never run. +#[test] +fn dsml_marker_on_the_tool_call_tag_parses() { + let raw = concat!( + "I'll look it up.\n\n", + "<|DSML|tool_call>\n", + "{\"name\": \"GMAIL_FETCH_EMAILS\", \"arguments\": {\"max_results\": 1}}\n", + "" + ); + let (text, calls) = crate::parse::parse_tool_calls(raw); + assert_eq!(calls.len(), 1, "DSML tool_call tag must parse: {calls:?}"); + assert_eq!(calls[0].name, "GMAIL_FETCH_EMAILS"); + assert_eq!(text.trim(), "I'll look it up."); +} + +/// Doubled bars and the ASCII spelling are the same marker. +#[test] +fn dsml_marker_variants_on_the_tag_parse() { + for open_close in [ + ("<||DSML||tool_call>", ""), + ("<|DSML|tool_call>", ""), + ("<|DSML|tool_call>", "<|DSML|tool_call>"), + ] { + let raw = format!( + "{}\n{{\"name\": \"echo\", \"arguments\": {{}}}}\n{}", + open_close.0, open_close.1 + ); + let (_, calls) = crate::parse::parse_tool_calls(&raw); + assert_eq!( + calls.len(), + 1, + "variant {open_close:?} must parse: {calls:?}" + ); + assert_eq!(calls[0].name, "echo"); + } +} + +/// The DSML *wrapper* element is plural and is not a call marker; treating it +/// as one would open a block on the wrapper and close it on the first inner +/// tag, losing the call inside. +#[test] +fn the_plural_dsml_wrapper_is_not_a_tag_marker() { + let raw = concat!( + "<|DSML|tool_calls>\n", + "<|DSML|tool_call>\n", + "{\"name\": \"echo\", \"arguments\": {}}\n", + "\n", + "" + ); + let (_, calls) = crate::parse::parse_tool_calls(raw); + assert_eq!(calls.len(), 1, "the inner call is the only call: {calls:?}"); + assert_eq!(calls[0].name, "echo"); +}