From 3afbc1c13f38fa2183b8d8125d765ba18a0386a5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 09:05:41 +0530 Subject: [PATCH 1/6] fix(parse): treat a doubled opener as one block DeepSeek V4 under a code dialect wraps the call twice: \n\nNAME(...)\n\n. Positional pairing closed the first tag on the empty body, dropped the call, and let the whole block leak into the visible reply. An opener followed by nothing but whitespace now continues the same block, and the extra closer is swallowed. Co-authored-by: Medulla --- .../src/parse/grammar/tagged.rs | 56 +++++++++++++++---- .../tinytools-agent/src/parse/test/tagged.rs | 39 +++++++++++++ 2 files changed, 85 insertions(+), 10 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 66d3dbc..037abcc 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -117,20 +117,50 @@ impl Tagged { let Some(opener) = next_opener(text, from) else { return Probe::None; }; - let after = &text[opener.body_start..]; + let mut body_start = opener.body_start; let close = match opener.kind { - OpenerKind::Tag => TAG_RE - .as_ref() - .and_then(|re| re.find(after)) - .map(|m| (m.start(), m.end())), - OpenerKind::Invoke => after.find("").map(|i| (i, i + "".len())), - OpenerKind::Fence => fence_close(after), + OpenerKind::Tag => { + // Positional pairing means a doubled opener — + // `\n\nNAME(...)\n\n`, + // which `DeepSeek` V4 emits under a code dialect — would + // otherwise close the first tag on an empty body and lose the + // call. An opener followed by nothing but whitespace is the + // same block starting again, so the scan moves past it; the + // matching extra closer is consumed below. + let re = TAG_RE.as_ref(); + loop { + let after = &text[body_start..]; + let Some(m) = re.and_then(|re| re.find(after)) else { + break None; + }; + let is_opener = !is_closing_marker(m.as_str()); + if is_opener && after[..m.start()].trim().is_empty() { + body_start += m.end(); + continue; + } + break Some((m.start(), m.end())); + } + } + OpenerKind::Invoke => { + let after = &text[body_start..]; + after.find("").map(|i| (i, i + "".len())) + } + OpenerKind::Fence => fence_close(&text[body_start..]), }; + let after = &text[body_start..]; if let Some((body_end, close_end)) = close { let body = &after[..body_end]; - let end = opener.body_start + close_end; + let rest = &after[close_end..]; + // Swallow the closers a doubled opener left behind, so no stray + // `` survives into the visible text. Only then: a + // block followed by prose keeps its exact end, whitespace included. + let end = if opener.kind == OpenerKind::Tag && rest.trim_start().starts_with("`, `<|/tool_call|>`). +fn is_closing_marker(marker: &str) -> bool { + marker[1..] + .trim_start_matches(['|', ' ', '\t']) + .starts_with('/') +} + /// The earliest opener at or after `from`: a non-closing tag-family marker, /// the bare `` literal, or a fence opener. fn next_opener(text: &str, from: usize) -> Option { @@ -199,8 +236,7 @@ fn next_opener(text: &str, from: usize) -> Option { if let Some(re) = TAG_RE.as_ref() { for m in re.find_iter(&text[from..]) { // A marker with a slash is a closer, never an opener. - let inner = &m.as_str()[1..]; - if inner.trim_start_matches(['|', ' ', '\t']).starts_with('/') { + if is_closing_marker(m.as_str()) { continue; } consider( diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index f5d3b43..dc45b54 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -415,3 +415,42 @@ fn a_code_call_to_an_unknown_tool_is_not_a_call() { let (_, calls) = parse_tool_calls_with_pformat(response, &echo_registry()); assert!(calls.is_empty()); } + +// ── Doubled tags ──────────────────────────────────────────────────────────── +// +// `DeepSeek` V4 under a code dialect wraps the block twice: +// `\n\nNAME(...)\n\n`. Positional +// pairing used to close the first tag on the empty body and drop the call, +// and the whole thing leaked into the visible reply. + +#[test] +fn a_doubled_opener_is_one_block_and_its_extra_closer_is_swallowed() { + let response = "I'll search.\n\n\n\necho(value=\"kashmir\")\n\n"; + let (narrative, calls) = parse_tool_calls_with_pformat(response, &echo_registry()); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!(calls[0].name, "echo"); + assert_eq!(calls[0].arguments, serde_json::json!({"value": "kashmir"})); + assert_eq!(narrative, "I'll search."); + assert!(!narrative.contains("tool_call"), "{narrative:?}"); +} + +#[test] +fn a_doubled_opener_around_a_json_body_parses_too() { + let (text, calls) = parse( + "\n\n{\"name\":\"echo\",\"arguments\":{\"value\":\"x\"}}\n\n\nafter", + ); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments, serde_json::json!({"value": "x"})); + assert_eq!(text, "after"); +} + +#[test] +fn two_adjacent_blocks_are_still_two_blocks() { + // The doubled-opener rule only fires on a whitespace-only gap; a real + // body between two openers is still the first block's body. + let text = "{\"name\":\"one\",\"arguments\":{}}{\"name\":\"two\",\"arguments\":{}}"; + let (_, calls) = parse(text); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].name, "one"); + assert_eq!(calls[1].name, "two"); +} From 7348bb9f360475b30d5ad6d42d7d8090b09b3177 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 09:11:09 +0530 Subject: [PATCH 2/6] fix(parse): drop a bare trailing opener instead of showing it Co-authored-by: Medulla --- crates/tinytools-agent/src/parse/grammar/tagged.rs | 12 ++++++++++++ crates/tinytools-agent/src/parse/test/tagged.rs | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 037abcc..410f3d5 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -182,6 +182,18 @@ impl Tagged { }; } + // Batch: an opener with nothing after it is a call the model started + // and never wrote (a truncated or abandoned block). There is nothing + // to recover and nothing worth showing, so it is dropped rather than + // left in the visible text as a bare ``. + if opener.kind == OpenerKind::Tag && after.trim().is_empty() { + return Probe::Found(Block { + start: opener.start, + end: text.len(), + decoded: Decoded::Malformed { body_chars: 0 }, + }); + } + // Batch: no closer. Recover a balanced JSON body if one starts here. let recovered = find_json_end(after) .and_then(|json_end| { diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index dc45b54..5d50b73 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -454,3 +454,15 @@ fn two_adjacent_blocks_are_still_two_blocks() { assert_eq!(calls[0].name, "one"); assert_eq!(calls[1].name, "two"); } + +#[test] +fn a_bare_trailing_opener_is_dropped_not_shown() { + // An abandoned block at the end of a reply carries no call and no + // information; showing `` to the user is never right. + let (text, calls) = parse("Let me fetch a few sites directly.\n\n\n"); + assert!(calls.is_empty()); + assert_eq!(text, "Let me fetch a few sites directly."); + // A block with real (if unparseable) content is still kept as text. + let (text, _) = parse("before not-json"); + assert_eq!(text, "before not-json"); +} From 82eaca536c37a0288e4c45a6d84f2b2edc2c5e9b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 12:23:21 +0530 Subject: [PATCH 3/6] test(jev): cover family-strategy branches to clear the 90% file gate crates/tinytools-jev/src/family.rs was at 84.91% line coverage, failing CI's per-file coverage gate (inherited unchanged from main via PR #19). Add tests for the abstain path, the too-many-families and reserved-none-family errors, the needs-tool-floor clear, merge's none-beats-every-member and missing-member-probability branches, the oversized-family retriever cut (both the matched and the fallback-to-first-members cases), the family-summary 600-character truncation, and join_all's re-poll loop for a future that is Pending on its first poll. Co-authored-by: Medulla --- crates/tinytools-jev/src/test.rs | 364 +++++++++++++++++++++++++++++++ 1 file changed, 364 insertions(+) diff --git a/crates/tinytools-jev/src/test.rs b/crates/tinytools-jev/src/test.rs index e48a096..b533d30 100644 --- a/crates/tinytools-jev/src/test.rs +++ b/crates/tinytools-jev/src/test.rs @@ -359,3 +359,367 @@ async fn family_then_decide_with_one_family_skips_the_family_stage() { Some("SLACK_SEND_MESSAGE") ); } + +/// Builds a decision from `(key, probability)` pairs, defaulting the fields +/// the family-stage branches below do not exercise. +fn family_decision(pairs: &[(&str, f64)], needs_tool: Option) -> JevDecision { + JevDecision { + probabilities: pairs + .iter() + .map(|(key, value)| ((*key).to_owned(), *value)) + .collect(), + choice_confidence: 0.7, + needs_tool, + input_tokens: Some(10), + attempts: 1, + } +} + +#[tokio::test] +async fn family_then_decide_rejects_reserved_none_family_name() { + let candidates = vec![ + RankCandidate::new("a", "does a").with_family("none"), + RankCandidate::new("b", "does b").with_family("real"), + ]; + let (ranker, _) = ranker_with_strategy( + [("real", 0.5), ("none", 0.5), ("_", 0.0)], + Some(0.9), + JevStrategy::FamilyThenDecide, + ); + let result = ranker + .rank("do it", &RankContext::empty(), &candidates, 2) + .await; + assert!(matches!(result, Err(RankError::InvalidInput { .. }))); +} + +/// A ranker configured with an explicit strategy, reusing the fixed-decision +/// [`FakeEvaluator`]. +fn ranker_with_strategy( + probabilities: [(&str, f64); 3], + needs_tool: Option, + strategy: JevStrategy, +) -> (JevRanker, Arc) { + let evaluator = Arc::new(FakeEvaluator { + decision: JevDecision { + probabilities: probabilities + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect::>(), + choice_confidence: 0.8, + needs_tool, + input_tokens: Some(10), + attempts: 1, + }, + seen: Mutex::new(vec![]), + }); + ( + JevRanker::new( + evaluator.clone(), + JevRankerConfig::new().with_strategy(strategy), + ), + evaluator, + ) +} + +#[tokio::test] +async fn family_then_decide_rejects_more_families_than_max_candidates() { + let candidates: Vec = (0..=JevRankerConfig::MAX_CANDIDATES) + .map(|i| { + RankCandidate::new(format!("tool_{i}"), "does something") + .with_family(format!("family_{i}")) + }) + .collect(); + let (ranker, _) = ranker_with_strategy( + [("x", 0.5), ("y", 0.5), ("none", 0.0)], + Some(0.9), + JevStrategy::FamilyThenDecide, + ); + let result = ranker + .rank("do it", &RankContext::empty(), &candidates, 2) + .await; + assert!(matches!(result, Err(RankError::InvalidInput { .. }))); +} + +#[tokio::test] +async fn family_then_decide_abstains_when_the_family_stage_prefers_none() { + #[derive(Debug)] + struct AbstainingEvaluator { + seen: Mutex>, + } + #[async_trait::async_trait] + impl JevEvaluator for AbstainingEvaluator { + async fn evaluate(&self, request: &JevRequest) -> Result { + if let Ok(mut seen) = self.seen.lock() { + seen.push(request.clone()); + } + Ok(family_decision( + &[("slack", 0.05), ("gmail", 0.05), ("none", 0.9)], + Some(0.9), + )) + } + } + let evaluator = Arc::new(AbstainingEvaluator { + seen: Mutex::new(vec![]), + }); + let ranker = JevRanker::new( + evaluator.clone(), + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let ranking = ranker + .rank_detailed("ping alex", &RankContext::empty(), &family_catalogue(), 3) + .await; + assert!( + ranking + .as_ref() + .is_ok_and(|r| r.hits.is_empty() && r.families.is_empty()) + ); + assert!( + ranking + .as_ref() + .is_ok_and(|r| (r.none_probability - 0.9).abs() < 1e-9) + ); + assert_eq!( + evaluator.seen.lock().map_or(0, |s| s.len()), + 1, + "only the family stage ran" + ); +} + +#[tokio::test] +async fn family_then_decide_clears_hits_when_the_chosen_family_says_no_tool_is_needed() { + let evaluator = Arc::new(FakeEvaluator { + decision: family_decision( + &[ + ("SLACK_SEND_MESSAGE", 0.8), + ("SLACK_LIST", 0.1), + ("none", 0.1), + ], + Some(0.2), + ), + seen: Mutex::new(vec![]), + }); + let ranker = JevRanker::new( + evaluator, + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let only_slack: Vec = family_catalogue() + .into_iter() + .filter(|c| c.family.as_deref() == Some("slack")) + .collect(); + let ranking = ranker + .rank_detailed("ping alex", &RankContext::empty(), &only_slack, 3) + .await; + assert!(ranking.is_ok_and(|r| r.hits.is_empty())); +} + +#[tokio::test] +async fn family_then_decide_drops_a_family_whose_none_beats_every_member() { + let evaluator = Arc::new(FakeEvaluator { + decision: family_decision( + &[ + ("SLACK_SEND_MESSAGE", 0.2), + ("SLACK_LIST", 0.1), + ("none", 0.9), + ], + Some(0.9), + ), + seen: Mutex::new(vec![]), + }); + let ranker = JevRanker::new( + evaluator, + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let only_slack: Vec = family_catalogue() + .into_iter() + .filter(|c| c.family.as_deref() == Some("slack")) + .collect(); + let ranking = ranker + .rank_detailed("ping alex", &RankContext::empty(), &only_slack, 3) + .await; + assert!(ranking.is_ok_and(|r| r.hits.is_empty())); +} + +#[tokio::test] +async fn family_then_decide_skips_a_member_missing_from_the_decision() { + let evaluator = Arc::new(FakeEvaluator { + decision: family_decision(&[("SLACK_SEND_MESSAGE", 0.8), ("none", 0.05)], Some(0.9)), + seen: Mutex::new(vec![]), + }); + let ranker = JevRanker::new( + evaluator, + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let only_slack: Vec = family_catalogue() + .into_iter() + .filter(|c| c.family.as_deref() == Some("slack")) + .collect(); + let ranking = ranker + .rank_detailed("ping alex", &RankContext::empty(), &only_slack, 3) + .await + .ok(); + // `SLACK_LIST` has no probability in the decision above, so `merge` must + // skip it rather than panic or fabricate a score for it. + assert_eq!(ranking.as_ref().map(|r| r.hits.len()), Some(1)); + assert_eq!( + ranking + .as_ref() + .and_then(|r| r.hits.first()) + .map(|h| h.key.as_str()), + Some("SLACK_SEND_MESSAGE") + ); +} + +#[tokio::test] +async fn family_then_decide_cuts_an_oversized_family_via_the_retriever() { + let members: Vec = (0..300) + .map(|i| { + let tag = if i < 150 { "alpha" } else { "beta" }; + RankCandidate::new(format!("tool_{i}"), format!("{tag} candidate number {i}")) + }) + .collect(); + let evaluator = Arc::new(FakeEvaluator { + decision: family_decision(&[("none", 0.05)], Some(0.9)), + seen: Mutex::new(vec![]), + }); + let ranker = JevRanker::new( + evaluator.clone(), + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let _ = ranker + .rank_detailed("alpha", &RankContext::empty(), &members, 3) + .await + .ok(); + let shown = evaluator + .seen + .lock() + .ok() + .and_then(|seen| seen.first().map(|r| r.options.len())) + .unwrap_or(0); + // `none` plus the alpha-tagged half, cut to at most `MAX_CANDIDATES`. + assert!(shown > 1 && shown <= JevRankerConfig::MAX_CANDIDATES + 1); +} + +#[tokio::test] +async fn family_then_decide_falls_back_to_the_first_members_when_retrieval_finds_nothing() { + let members: Vec = (0..300) + .map(|i| RankCandidate::new(format!("tool_{i}"), "shared filler text shared filler")) + .collect(); + let evaluator = Arc::new(FakeEvaluator { + decision: family_decision(&[("none", 0.05)], Some(0.9)), + seen: Mutex::new(vec![]), + }); + let ranker = JevRanker::new( + evaluator.clone(), + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let _ = ranker + .rank_detailed("shared", &RankContext::empty(), &members, 3) + .await + .ok(); + let shown = evaluator + .seen + .lock() + .ok() + .and_then(|seen| seen.first().map(|r| r.options.len())) + .unwrap_or(0); + // The retriever finds nothing distinguishable, so the fallback keeps the + // first `MAX_CANDIDATES` members (plus `none`) in their original order. + assert_eq!(shown, JevRankerConfig::MAX_CANDIDATES + 1); +} + +#[tokio::test] +async fn family_summary_truncates_past_600_characters() { + #[derive(Debug)] + struct RecordingEvaluator { + seen: Mutex>, + } + #[async_trait::async_trait] + impl JevEvaluator for RecordingEvaluator { + async fn evaluate(&self, request: &JevRequest) -> Result { + if let Ok(mut seen) = self.seen.lock() { + seen.push(request.clone()); + } + Ok(family_decision( + &[("big", 1.0), ("small", 0.0), ("none", 0.0)], + Some(0.9), + )) + } + } + let mut candidates: Vec = (0..12) + .map(|i| { + RankCandidate::new( + format!("TOOL_WITH_A_VERY_DESCRIPTIVE_LONG_NAME_NUMBER_{i:03}_THAT_PADS_LENGTH"), + "does something", + ) + .with_family("big") + }) + .collect(); + candidates.push(RankCandidate::new("SMALL_TOOL", "does one thing").with_family("small")); + let evaluator = Arc::new(RecordingEvaluator { + seen: Mutex::new(vec![]), + }); + let ranker = JevRanker::new( + evaluator.clone(), + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let _ = ranker + .rank_detailed("do it", &RankContext::empty(), &candidates, 3) + .await + .ok(); + let big_description = evaluator + .seen + .lock() + .ok() + .and_then(|seen| { + seen.first().map(|r| { + r.options + .iter() + .find(|o| o.key == "big") + .map(|o| o.description.clone()) + .unwrap_or_default() + }) + }) + .unwrap_or_default(); + assert!( + big_description.ends_with('…'), + "long family summary should be truncated: {big_description}" + ); + assert!(big_description.chars().count() <= 601); +} + +#[tokio::test] +async fn join_all_resolves_a_future_that_is_pending_on_its_first_poll() { + #[derive(Debug)] + struct YieldingEvaluator; + #[async_trait::async_trait] + impl JevEvaluator for YieldingEvaluator { + async fn evaluate(&self, _request: &JevRequest) -> Result { + // Forces at least one `Poll::Pending` before this future resolves, + // exercising `join_all`'s re-poll loop. + tokio::task::yield_now().await; + Ok(family_decision( + &[("SLACK_SEND_MESSAGE", 0.8), ("none", 0.1)], + Some(0.9), + )) + } + } + let ranker = JevRanker::new( + Arc::new(YieldingEvaluator), + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let only_slack: Vec = family_catalogue() + .into_iter() + .filter(|c| c.family.as_deref() == Some("slack")) + .collect(); + let ranking = ranker + .rank_detailed("ping alex", &RankContext::empty(), &only_slack, 3) + .await + .ok(); + assert_eq!( + ranking + .as_ref() + .and_then(|r| r.hits.first()) + .map(|h| h.key.as_str()), + Some("SLACK_SEND_MESSAGE") + ); +} From bbccbc6285f571a6a76e784f547bfcb604a36060 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 12:30:18 +0530 Subject: [PATCH 4/6] fix(parse): only swallow the doubled block's own extra closers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The doubled-``-opener cleanup used `strip_leading_close_tags`, which eats *any* leading `` marker. That over-consumed unrelated narrative markup right after a normal call (e.g. `\nvisible` lost ``) and, because it only understands `` syntax, missed the pipe-form duplicate closer `<|/tool_call|>` DeepSeek's dialect can emit. Track how many extra openers the doubled-block scan skipped and swallow exactly that many tag-family closers (matched by the same TAG_RE/ is_closing_marker grammar as every opener), never an unrelated closing tag. In streaming mode, if the buffered text runs out before that count of closers is confirmed one way or the other, report the block as still pending instead of finalizing early — otherwise a fragment boundary that lands between the doubled block's two closers could let the second one leak into the visible stream as plain text. Co-authored-by: Medulla --- .../src/parse/grammar/tagged.rs | 75 ++++++++++++++++--- .../tinytools-agent/src/parse/test/tagged.rs | 25 +++++++ crates/tinytools-agent/src/stream/test.rs | 15 ++++ 3 files changed, 103 insertions(+), 12 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 410f3d5..300baeb 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -119,15 +119,18 @@ impl Tagged { }; let mut body_start = opener.body_start; + // How many extra openers a doubled block skipped, so the matching + // number of extra closers — never an unrelated closing tag such as + // `` — can be swallowed below. `DeepSeek` V4 doubles both the + // opener and the closer under a code dialect: + // `\n\nNAME(...)\n\n`. + let mut skipped = 0usize; let close = match opener.kind { OpenerKind::Tag => { - // Positional pairing means a doubled opener — - // `\n\nNAME(...)\n\n`, - // which `DeepSeek` V4 emits under a code dialect — would - // otherwise close the first tag on an empty body and lose the - // call. An opener followed by nothing but whitespace is the - // same block starting again, so the scan moves past it; the - // matching extra closer is consumed below. + // Positional pairing means a doubled opener would otherwise + // close the first tag on an empty body and lose the call. An + // opener followed by nothing but whitespace is the same + // block starting again, so the scan moves past it. let re = TAG_RE.as_ref(); loop { let after = &text[body_start..]; @@ -137,6 +140,7 @@ impl Tagged { let is_opener = !is_closing_marker(m.as_str()); if is_opener && after[..m.start()].trim().is_empty() { body_start += m.end(); + skipped += 1; continue; } break Some((m.start(), m.end())); @@ -153,11 +157,21 @@ impl Tagged { if let Some((body_end, close_end)) = close { let body = &after[..body_end]; let rest = &after[close_end..]; - // Swallow the closers a doubled opener left behind, so no stray - // `` survives into the visible text. Only then: a - // block followed by prose keeps its exact end, whitespace included. - let end = if opener.kind == OpenerKind::Tag && rest.trim_start().starts_with("` — so no stray + // `` survives into the visible text while narrative + // markup after a normal call is left untouched. + let end = if opener.kind == OpenerKind::Tag && skipped > 0 { + match swallow_extra_closers(rest, skipped, mode) { + Some(consumed) => body_start + close_end + consumed, + // Streaming: more input could still bring the matching + // closer, so the block is not safe to finalize yet. + None => { + return Probe::Pending { + start: opener.start, + }; + } + } } else { body_start + close_end }; @@ -235,6 +249,43 @@ fn is_closing_marker(marker: &str) -> bool { .starts_with('/') } +/// Swallows up to `max` tag-family closers from the front of `rest` +/// (whitespace between them ignored), returning the byte count consumed. +/// Only a recognized closer — matched by [`TAG_RE`], the same grammar as +/// every opener — is ever eaten, so unrelated markup such as `` is +/// left for the narrative. In [`ScanMode::Stream`], `None` means the text +/// ends before it is clear whether another closer is still coming, so the +/// caller must hold the block back rather than finalize it early. +fn swallow_extra_closers(rest: &str, max: usize, mode: ScanMode) -> Option { + let re = TAG_RE.as_ref()?; + let mut consumed = 0usize; + for _ in 0..max { + let after = &rest[consumed..]; + let trimmed = after.trim_start(); + let skipped_ws = after.len() - trimmed.len(); + if trimmed.is_empty() { + // Nothing here yet: in batch mode that is simply the end of the + // response, in stream mode a closer could still be on its way. + return if mode == ScanMode::Stream { + None + } else { + Some(consumed) + }; + } + let Some(m) = re.find(trimmed) else { + return Some(consumed); + }; + if m.start() != 0 { + return Some(consumed); + } + if !is_closing_marker(m.as_str()) { + return Some(consumed); + } + consumed += skipped_ws + m.end(); + } + Some(consumed) +} + /// The earliest opener at or after `from`: a non-closing tag-family marker, /// the bare `` literal, or a fence opener. fn next_opener(text: &str, from: usize) -> Option { diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 5d50b73..0ee275d 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -455,6 +455,31 @@ fn two_adjacent_blocks_are_still_two_blocks() { assert_eq!(calls[1].name, "two"); } +#[test] +fn a_doubled_opener_does_not_swallow_an_unrelated_closing_tag() { + // Only the extra `` a doubled opener leaves behind is + // protocol furniture; a real closing tag right after it (``, from + // whatever markup the model echoed) is narrative and must survive. + let (text, calls) = parse( + "\n\n{\"name\":\"echo\",\"arguments\":{}}\n\nvisible", + ); + assert_eq!(calls.len(), 1); + assert_eq!(text, "visible"); +} + +#[test] +fn a_doubled_opener_swallows_a_pipe_form_duplicate_closer() { + // `TAG_RE` matches the pipe-form closer `<|/tool_call|>` too, so the + // doubled-opener path must recognize it as a closer to swallow, not + // leave it dangling in `rest` for `strip_leading_close_tags` (which only + // understands ``) to miss. + let (text, calls) = parse( + "<|tool_call|>\n<|tool_call|>\n{\"name\":\"echo\",\"arguments\":{}}\n<|/tool_call|>\n<|/tool_call|>\nafter", + ); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!(text, "after"); +} + #[test] fn a_bare_trailing_opener_is_dropped_not_shown() { // An abandoned block at the end of a reply carries no call and no diff --git a/crates/tinytools-agent/src/stream/test.rs b/crates/tinytools-agent/src/stream/test.rs index 270e899..000c89f 100644 --- a/crates/tinytools-agent/src/stream/test.rs +++ b/crates/tinytools-agent/src/stream/test.rs @@ -252,3 +252,18 @@ fn a_code_call_split_mid_string_is_released_once_and_never_shown() { assert!(!out.contains("echo("), "markup leaked: {out:?}"); assert!(out.contains("Sure.") && out.contains("done")); } + +#[test] +fn a_doubled_blocks_extra_closer_split_across_fragments_never_leaks() { + // The doubled opener's *inner* closer can arrive in one fragment and the + // matching extra closer in the next. The block must stay pending across + // that boundary rather than finalize on the inner closer alone and let + // the later `` fall through as visible text. + let (out, calls) = scrub_all(&[ + "before \n\n{\"name\":\"x\",\"arguments\":{}}\n\n", + " after", + ]); + assert_eq!(out, "before after"); + assert_eq!(calls, 1); + assert!(!out.contains("tool_call"), "{out:?}"); +} From 092c582ad41152dc066789096aba40b3d687a8c1 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 12:42:47 +0530 Subject: [PATCH 5/6] fix(parse): hold a partial duplicate closer and fix whitespace-form closer detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-up gaps in bbccbc6's extra-closer swallow, both flagged against the pushed head: - `swallow_extra_closers` only checked `rest` for emptiness before deciding to hold in stream mode. A fragment boundary landing *inside* the extra closer itself (`` in the next) fell through TAG_RE's `find` returning no match, so the block finalized early and the partial marker leaked as narrative text, with its tail following as more narrative once the rest arrived. Hold instead when the remaining text is a non-empty case-insensitive prefix of a canonical closer spelling and contains no `>` yet — the same practical, non-exhaustive literal-prefix trade-off `pending_opener` already makes for openers. - `is_closing_marker` only trimmed space and tab before checking for the leading `/`, but `TAG_RE`'s `\s` matches any whitespace. A closer with a leaked newline or other whitespace, such as `<\n/tool_call>`, matched the regex as one marker but was not recognized as a *closing* one, so the doubled-block swallow left it behind for the narrative to leak. Co-authored-by: Medulla --- .../src/parse/grammar/tagged.rs | 36 ++++++++++++++++--- .../tinytools-agent/src/parse/test/tagged.rs | 13 +++++++ crates/tinytools-agent/src/stream/test.rs | 15 ++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 300baeb..d54bff4 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -243,19 +243,43 @@ impl Tagged { } /// Whether a tag-family marker is a closer (``, `<|/tool_call|>`). +/// Skips the same whitespace class `TAG_RE`'s `\s` does (not just space and +/// tab), so a marker like `<\n/tool_call>` — which the regex matches as one +/// marker — is still recognized as a closer here. fn is_closing_marker(marker: &str) -> bool { marker[1..] - .trim_start_matches(['|', ' ', '\t']) + .trim_start_matches(|c: char| c == '|' || c.is_whitespace()) .starts_with('/') } +/// Canonical closer spellings [`swallow_extra_closers`] holds a partial +/// match of in stream mode. Not exhaustive of everything `TAG_RE` accepts +/// (arbitrary interleaved pipes and whitespace) — the same practical +/// trade-off [`pending_opener`]'s literal list already makes for openers. +const CLOSER_PREFIXES: &[&str] = &["` yet and is a case-insensitive prefix of one of [`CLOSER_PREFIXES`]. +fn could_still_become_a_closer(trimmed: &str) -> bool { + if trimmed.contains('>') { + return false; + } + CLOSER_PREFIXES.iter().any(|literal| { + let n = trimmed.len().min(literal.len()); + trimmed.is_char_boundary(n) && trimmed[..n].eq_ignore_ascii_case(&literal[..n]) + }) +} + /// Swallows up to `max` tag-family closers from the front of `rest` /// (whitespace between them ignored), returning the byte count consumed. /// Only a recognized closer — matched by [`TAG_RE`], the same grammar as /// every opener — is ever eaten, so unrelated markup such as `` is /// left for the narrative. In [`ScanMode::Stream`], `None` means the text -/// ends before it is clear whether another closer is still coming, so the -/// caller must hold the block back rather than finalize it early. +/// ends, or breaks off mid-marker, before it is clear whether another closer +/// is still coming, so the caller must hold the block back rather than +/// finalize it early and let a partial marker such as `` tail arrives in a later fragment. fn swallow_extra_closers(rest: &str, max: usize, mode: ScanMode) -> Option { let re = TAG_RE.as_ref()?; let mut consumed = 0usize; @@ -273,7 +297,11 @@ fn swallow_extra_closers(rest: &str, max: usize, mode: ScanMode) -> Option` is one complete closer marker; `is_closing_marker` + // must classify it as such too, or the extra closer is left behind for + // the narrative to leak. + let (text, calls) = parse( + "\n\n{\"name\":\"echo\",\"arguments\":{}}\n\n<\n/tool_call>\nafter", + ); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!(text, "after"); +} + #[test] fn a_bare_trailing_opener_is_dropped_not_shown() { // An abandoned block at the end of a reply carries no call and no diff --git a/crates/tinytools-agent/src/stream/test.rs b/crates/tinytools-agent/src/stream/test.rs index 000c89f..fa1dbb3 100644 --- a/crates/tinytools-agent/src/stream/test.rs +++ b/crates/tinytools-agent/src/stream/test.rs @@ -267,3 +267,18 @@ fn a_doubled_blocks_extra_closer_split_across_fragments_never_leaks() { assert_eq!(calls, 1); assert!(!out.contains("tool_call"), "{out:?}"); } + +#[test] +fn a_doubled_blocks_extra_closer_split_mid_marker_never_leaks() { + // The fragment boundary can land *inside* the extra closer itself, not + // just before it: `` in the next. That + // partial marker must be held rather than released as text once its + // first fragment is scanned. + let (out, calls) = scrub_all(&[ + "before \n\n{\"name\":\"x\",\"arguments\":{}}\n\n after", + ]); + assert_eq!(out, "before after"); + assert_eq!(calls, 1); + assert!(!out.contains("tool_"), "{out:?}"); +} From 4b28b3812e07fafdb4a4d387b684f54bd22b4b21 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 12:46:17 +0530 Subject: [PATCH 6/6] fix(parse): stop swallowing narrative markup after an unclosed opener's recovered JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The batch fallback for an opener with no tag-family marker anywhere after it (so no closer at all was found) still ran strip_leading_close_tags on whatever followed the recovered JSON body, removing any leading tag — same over-broad-stripping bug as the doubled-opener path already fixed, just in a sibling branch flagged separately on this push. Since no tag-family marker exists in the remaining text by construction of this branch, there is nothing here to clean up: end the block at the JSON boundary and leave trailing markup, tool-call-related or not, in the narrative. strip_leading_close_tags has no remaining caller once this path stops using it, so it is removed along with its now-pointless direct unit test. Co-authored-by: Medulla --- .../tinytools-agent/src/parse/grammar/tagged.rs | 11 +++++++---- crates/tinytools-agent/src/parse/json_values.rs | 16 ---------------- crates/tinytools-agent/src/parse/test/engine.rs | 11 +---------- crates/tinytools-agent/src/parse/test/tagged.rs | 13 +++++++++++-- 4 files changed, 19 insertions(+), 32 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index d54bff4..4f01ad6 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -29,7 +29,7 @@ use regex::Regex; use super::{Block, Decoded, Grammar, Probe, ScanMode, find_ci, pending_opener, prefer_pending}; use crate::parse::call_object::{AliasPolicy, read_calls}; use crate::parse::json_values::{ - extract_first_json_value_with_end, extract_json_values, find_json_end, strip_leading_close_tags, + extract_first_json_value_with_end, extract_json_values, find_json_end, }; use crate::repair::json::{recover_object, strip_code_fence}; use crate::types::{CallSource, ParseOptions, ParsedToolCall}; @@ -224,9 +224,12 @@ impl Tagged { CallSource::TaggedJson, ); if !calls.is_empty() { - let rest = &after[consumed..]; - let stripped = strip_leading_close_tags(rest); - let end = text.len() - stripped.len(); + // No tag-family marker exists anywhere after this opener (the + // TAG_RE scan above found none), so nothing here is protocol + // furniture to clean up — stopping at the JSON boundary + // leaves any trailing markup, tool-call-related or not, in + // the narrative rather than guessing which closer it was. + let end = body_start + consumed; return Probe::Found(Block { start: opener.start, end, diff --git a/crates/tinytools-agent/src/parse/json_values.rs b/crates/tinytools-agent/src/parse/json_values.rs index 108ad67..d0ede48 100644 --- a/crates/tinytools-agent/src/parse/json_values.rs +++ b/crates/tinytools-agent/src/parse/json_values.rs @@ -109,19 +109,3 @@ pub(crate) fn find_json_end(input: &str) -> Option { None } - -/// Drops any run of leading closing tags (``) and the whitespace around -/// them. A truncated closing tag with no `>` consumes the rest. -#[must_use] -pub(crate) fn strip_leading_close_tags(mut input: &str) -> &str { - loop { - let trimmed = input.trim_start(); - if !trimmed.starts_with("') else { - return ""; - }; - input = &trimmed[close_end + 1..]; - } -} diff --git a/crates/tinytools-agent/src/parse/test/engine.rs b/crates/tinytools-agent/src/parse/test/engine.rs index d3e709d..66e26e9 100644 --- a/crates/tinytools-agent/src/parse/test/engine.rs +++ b/crates/tinytools-agent/src/parse/test/engine.rs @@ -1,9 +1,7 @@ //! The scan engine: protected fences, name resolution, helpers, diagnostics. use super::{parse, parse_known}; -use crate::parse::json_values::{ - extract_first_json_value_with_end, find_json_end, strip_leading_close_tags, -}; +use crate::parse::json_values::{extract_first_json_value_with_end, find_json_end}; use crate::parse::protected::fence_ranges; use crate::parse::{ extract_json_values, parse_arguments_value, parse_tool_call_value, @@ -192,13 +190,6 @@ fn json_scanners_cover_common_edge_cases() { assert!(extracted.1 > 0); assert!(extract_first_json_value_with_end("no json here").is_none()); - assert_eq!( - strip_leading_close_tags(" hi "), - "hi " - ); - assert_eq!(strip_leading_close_tags("plain"), "plain"); - assert_eq!(strip_leading_close_tags(" ` + // right after the recovered JSON is narrative, not a stray tool-call + // closer — it must not be swallowed as if it were one. + let (text, calls) = parse("{\"name\":\"echo\",\"arguments\":{}}visible"); + assert_eq!(calls.len(), 1); + assert_eq!(text, "visible"); +} + #[test] fn unclosed_tag_without_json_is_kept_as_text() { let (text, calls) = parse("before not-json"); @@ -471,8 +481,7 @@ fn a_doubled_opener_does_not_swallow_an_unrelated_closing_tag() { fn a_doubled_opener_swallows_a_pipe_form_duplicate_closer() { // `TAG_RE` matches the pipe-form closer `<|/tool_call|>` too, so the // doubled-opener path must recognize it as a closer to swallow, not - // leave it dangling in `rest` for `strip_leading_close_tags` (which only - // understands ``) to miss. + // leave it dangling as narrative text. let (text, calls) = parse( "<|tool_call|>\n<|tool_call|>\n{\"name\":\"echo\",\"arguments\":{}}\n<|/tool_call|>\n<|/tool_call|>\nafter", );