fix(parse): treat a doubled <tool_call> opener as one block

crates/tinytools-agent/src/parse/grammar/tagged.rs

@@ -29,7 +29,7 @@ use regex::Regex;
 use super::{Block, Decoded, Grammar, Probe, ScanMode, find_ci, pending_opener, prefer_pending};
 use crate::parse::call_object::{AliasPolicy, read_calls};
 use crate::parse::json_values::{
-    extract_first_json_value_with_end, extract_json_values, find_json_end, strip_leading_close_tags,
+    extract_first_json_value_with_end, extract_json_values, find_json_end,
 };
 use crate::repair::json::{recover_object, strip_code_fence};
 use crate::types::{CallSource, ParseOptions, ParsedToolCall};
@@ -117,20 +117,64 @@ impl Tagged {
         let Some(opener) = next_opener(text, from) else {
             return Probe::None;
         };
-        let after = &text[opener.body_start..];
+        let mut body_start = opener.body_start;
 
+        // How many extra openers a doubled block skipped, so the matching
+        // number of extra closers — never an unrelated closing tag such as
+        // `</div>` — can be swallowed below. `DeepSeek` V4 doubles both the
+        // opener and the closer under a code dialect:
+        // `<tool_call>\n<tool_call>\nNAME(...)\n</tool_call>\n</tool_call>`.
+        let mut skipped = 0usize;
         let close = match opener.kind {
-            OpenerKind::Tag => TAG_RE
-                .as_ref()
-                .and_then(|re| re.find(after))
-                .map(|m| (m.start(), m.end())),
-            OpenerKind::Invoke => after.find("</invoke>").map(|i| (i, i + "</invoke>".len())),
-            OpenerKind::Fence => fence_close(after),
+            OpenerKind::Tag => {
+                // Positional pairing means a doubled opener would otherwise
+                // close the first tag on an empty body and lose the call. An
+                // opener followed by nothing but whitespace is the same
+                // block starting again, so the scan moves past it.
+                let re = TAG_RE.as_ref();
+                loop {
+                    let after = &text[body_start..];
+                    let Some(m) = re.and_then(|re| re.find(after)) else {
+                        break None;
+                    };
+                    let is_opener = !is_closing_marker(m.as_str());
+                    if is_opener && after[..m.start()].trim().is_empty() {
+                        body_start += m.end();
+                        skipped += 1;
+                        continue;
+                    }
+                    break Some((m.start(), m.end()));
+                }
+            }
+            OpenerKind::Invoke => {
+                let after = &text[body_start..];
+                after.find("</invoke>").map(|i| (i, i + "</invoke>".len()))
+            }
+            OpenerKind::Fence => fence_close(&text[body_start..]),
         };
+        let after = &text[body_start..];
 
         if let Some((body_end, close_end)) = close {
             let body = &after[..body_end];
-            let end = opener.body_start + close_end;
+            let rest = &after[close_end..];
+            // Swallow only the closers a doubled opener left behind — never
+            // an unrelated closing tag such as `</div>` — so no stray
+            // `</tool_call>` survives into the visible text while narrative
+            // markup after a normal call is left untouched.
+            let end = if opener.kind == OpenerKind::Tag && skipped > 0 {
+                match swallow_extra_closers(rest, skipped, mode) {
+                    Some(consumed) => body_start + close_end + consumed,
+                    // Streaming: more input could still bring the matching
+                    // closer, so the block is not safe to finalize yet.
+                    None => {
+                        return Probe::Pending {
+                            start: opener.start,
+                        };
+                    }
+                }
+            } else {
+                body_start + close_end
+            };
             let calls = decode_body(body, options);
             let decoded = if calls.is_empty() {
                 Decoded::Malformed {
@@ -152,6 +196,18 @@ impl Tagged {
             };
         }
 
+        // Batch: an opener with nothing after it is a call the model started
+        // and never wrote (a truncated or abandoned block). There is nothing
+        // to recover and nothing worth showing, so it is dropped rather than
+        // left in the visible text as a bare `<tool_call>`.
+        if opener.kind == OpenerKind::Tag && after.trim().is_empty() {
+            return Probe::Found(Block {
+                start: opener.start,
+                end: text.len(),
+                decoded: Decoded::Malformed { body_chars: 0 },
+            });
+        }
+
         // Batch: no closer. Recover a balanced JSON body if one starts here.
         let recovered = find_json_end(after)
             .and_then(|json_end| {
@@ -168,9 +224,12 @@ impl Tagged {
                 CallSource::TaggedJson,
             );
             if !calls.is_empty() {
-                let rest = &after[consumed..];
-                let stripped = strip_leading_close_tags(rest);
-                let end = text.len() - stripped.len();
+                // No tag-family marker exists anywhere after this opener (the
+                // TAG_RE scan above found none), so nothing here is protocol
+                // furniture to clean up — stopping at the JSON boundary
+                // leaves any trailing markup, tool-call-related or not, in
+                // the narrative rather than guessing which closer it was.
+                let end = body_start + consumed;
                 return Probe::Found(Block {
                     start: opener.start,
                     end,
@@ -186,6 +245,78 @@ impl Tagged {
     }
 }
 
+/// Whether a tag-family marker is a closer (`</tool_call>`, `<|/tool_call|>`).
+/// Skips the same whitespace class `TAG_RE`'s `\s` does (not just space and
+/// tab), so a marker like `<\n/tool_call>` — which the regex matches as one
+/// marker — is still recognized as a closer here.
+fn is_closing_marker(marker: &str) -> bool {
+    marker[1..]
+        .trim_start_matches(|c: char| c == '|' || c.is_whitespace())
+        .starts_with('/')
+}
+
+/// Canonical closer spellings [`swallow_extra_closers`] holds a partial
+/// match of in stream mode. Not exhaustive of everything `TAG_RE` accepts
+/// (arbitrary interleaved pipes and whitespace) — the same practical
+/// trade-off [`pending_opener`]'s literal list already makes for openers.
+const CLOSER_PREFIXES: &[&str] = &["</tool_call", "</toolcall", "</tool-call", "<|/tool_call"];
+
+/// Whether `trimmed` — which `TAG_RE` did not match as a complete marker —
+/// could still grow into a tag-family closer once more input arrives: it has
+/// no `>` yet and is a case-insensitive prefix of one of [`CLOSER_PREFIXES`].
+fn could_still_become_a_closer(trimmed: &str) -> bool {
+    if trimmed.contains('>') {
+        return false;
+    }
+    CLOSER_PREFIXES.iter().any(|literal| {
+        let n = trimmed.len().min(literal.len());
+        trimmed.is_char_boundary(n) && trimmed[..n].eq_ignore_ascii_case(&literal[..n])
+    })
+}
+
+/// Swallows up to `max` tag-family closers from the front of `rest`
+/// (whitespace between them ignored), returning the byte count consumed.
+/// Only a recognized closer — matched by [`TAG_RE`], the same grammar as
+/// every opener — is ever eaten, so unrelated markup such as `</div>` is
+/// left for the narrative. In [`ScanMode::Stream`], `None` means the text
+/// ends, or breaks off mid-marker, before it is clear whether another closer
+/// is still coming, so the caller must hold the block back rather than
+/// finalize it early and let a partial marker such as `</tool_` leak out as
+/// text before its `call>` tail arrives in a later fragment.
+fn swallow_extra_closers(rest: &str, max: usize, mode: ScanMode) -> Option<usize> {
+    let re = TAG_RE.as_ref()?;
+    let mut consumed = 0usize;
+    for _ in 0..max {
+        let after = &rest[consumed..];
+        let trimmed = after.trim_start();
+        let skipped_ws = after.len() - trimmed.len();
+        if trimmed.is_empty() {
+            // Nothing here yet: in batch mode that is simply the end of the
+            // response, in stream mode a closer could still be on its way.
+            return if mode == ScanMode::Stream {
+                None
+            } else {
+                Some(consumed)
+            };
+        }
+        let Some(m) = re.find(trimmed) else {
+            return if mode == ScanMode::Stream && could_still_become_a_closer(trimmed) {
+                None
+            } else {
+                Some(consumed)
+            };
+        };
+        if m.start() != 0 {
+            return Some(consumed);
+        }
+        if !is_closing_marker(m.as_str()) {
+            return Some(consumed);
+        }
+        consumed += skipped_ws + m.end();
+    }
+    Some(consumed)
+}
+
 /// The earliest opener at or after `from`: a non-closing tag-family marker,
 /// the bare `<invoke>` literal, or a fence opener.
 fn next_opener(text: &str, from: usize) -> Option<Opener> {
@@ -199,8 +330,7 @@ fn next_opener(text: &str, from: usize) -> Option<Opener> {
     if let Some(re) = TAG_RE.as_ref() {
         for m in re.find_iter(&text[from..]) {
             // A marker with a slash is a closer, never an opener.
-            let inner = &m.as_str()[1..];
-            if inner.trim_start_matches(['|', ' ', '\t']).starts_with('/') {
+            if is_closing_marker(m.as_str()) {
                 continue;
             }
             consider(

crates/tinytools-agent/src/parse/json_values.rs

@@ -109,19 +109,3 @@ pub(crate) fn find_json_end(input: &str) -> Option<usize> {
 
     None
 }
-
-/// Drops any run of leading closing tags (`</x>`) and the whitespace around
-/// them. A truncated closing tag with no `>` consumes the rest.
-#[must_use]
-pub(crate) fn strip_leading_close_tags(mut input: &str) -> &str {
-    loop {
-        let trimmed = input.trim_start();
-        if !trimmed.starts_with("</") {
-            return trimmed;
-        }
-        let Some(close_end) = trimmed.find('>') else {
-            return "";
-        };
-        input = &trimmed[close_end + 1..];
-    }
-}

crates/tinytools-agent/src/parse/test/engine.rs

@@ -1,9 +1,7 @@
 //! The scan engine: protected fences, name resolution, helpers, diagnostics.
 
 use super::{parse, parse_known};
-use crate::parse::json_values::{
-    extract_first_json_value_with_end, find_json_end, strip_leading_close_tags,
-};
+use crate::parse::json_values::{extract_first_json_value_with_end, find_json_end};
 use crate::parse::protected::fence_ranges;
 use crate::parse::{
     extract_json_values, parse_arguments_value, parse_tool_call_value,
@@ -192,13 +190,6 @@ fn json_scanners_cover_common_edge_cases() {
     assert!(extracted.1 > 0);
     assert!(extract_first_json_value_with_end("no json here").is_none());
 
-    assert_eq!(
-        strip_leading_close_tags(" </tool_call>  </invoke> hi "),
-        "hi "
-    );
-    assert_eq!(strip_leading_close_tags("plain"), "plain");
-    assert_eq!(strip_leading_close_tags(" </broken"), "");
-
     let values = extract_json_values("before {\"a\":1} [1,2] after");
     assert_eq!(
         values,

crates/tinytools-agent/src/parse/test/tagged.rs

@@ -80,6 +80,16 @@ fn unclosed_tag_with_balanced_json_still_recovers() {
     assert_eq!(calls.len(), 1);
 }
 
+#[test]
+fn unclosed_tag_recovery_preserves_unrelated_markup_after_the_json() {
+    // No tag-family marker exists anywhere after this opener, so a `</div>`
+    // right after the recovered JSON is narrative, not a stray tool-call
+    // closer — it must not be swallowed as if it were one.
+    let (text, calls) = parse("<toolcall>{\"name\":\"echo\",\"arguments\":{}}</div>visible");
+    assert_eq!(calls.len(), 1);
+    assert_eq!(text, "</div>visible");
+}
+
 #[test]
 fn unclosed_tag_without_json_is_kept_as_text() {
     let (text, calls) = parse("before <tool-call>not-json");
@@ -415,3 +425,91 @@ fn a_code_call_to_an_unknown_tool_is_not_a_call() {
     let (_, calls) = parse_tool_calls_with_pformat(response, &echo_registry());
     assert!(calls.is_empty());
 }
+
+// ── Doubled tags ────────────────────────────────────────────────────────────
+//
+// `DeepSeek` V4 under a code dialect wraps the block twice:
+// `<tool_call>\n<tool_call>\nNAME(...)\n</tool_call>\n</tool_call>`. Positional
+// pairing used to close the first tag on the empty body and drop the call,
+// and the whole thing leaked into the visible reply.
+
+#[test]
+fn a_doubled_opener_is_one_block_and_its_extra_closer_is_swallowed() {
+    let response = "I'll search.\n\n<tool_call>\n<tool_call>\necho(value=\"kashmir\")\n</tool_call>\n</tool_call>";
+    let (narrative, calls) = parse_tool_calls_with_pformat(response, &echo_registry());
+    assert_eq!(calls.len(), 1, "{calls:?}");
+    assert_eq!(calls[0].name, "echo");
+    assert_eq!(calls[0].arguments, serde_json::json!({"value": "kashmir"}));
+    assert_eq!(narrative, "I'll search.");
+    assert!(!narrative.contains("tool_call"), "{narrative:?}");
+}
+
+#[test]
+fn a_doubled_opener_around_a_json_body_parses_too() {
+    let (text, calls) = parse(
+        "<tool_call>\n<tool_call>\n{\"name\":\"echo\",\"arguments\":{\"value\":\"x\"}}\n</tool_call>\n</tool_call>\nafter",
+    );
+    assert_eq!(calls.len(), 1);
+    assert_eq!(calls[0].arguments, serde_json::json!({"value": "x"}));
+    assert_eq!(text, "after");
+}
+
+#[test]
+fn two_adjacent_blocks_are_still_two_blocks() {
+    // The doubled-opener rule only fires on a whitespace-only gap; a real
+    // body between two openers is still the first block's body.
+    let text = "<tool_call>{\"name\":\"one\",\"arguments\":{}}</tool_call><tool_call>{\"name\":\"two\",\"arguments\":{}}</tool_call>";
+    let (_, calls) = parse(text);
+    assert_eq!(calls.len(), 2);
+    assert_eq!(calls[0].name, "one");
+    assert_eq!(calls[1].name, "two");
+}
+
+#[test]
+fn a_doubled_opener_does_not_swallow_an_unrelated_closing_tag() {
+    // Only the extra `</tool_call>` a doubled opener leaves behind is
+    // protocol furniture; a real closing tag right after it (`</div>`, from
+    // whatever markup the model echoed) is narrative and must survive.
+    let (text, calls) = parse(
+        "<tool_call>\n<tool_call>\n{\"name\":\"echo\",\"arguments\":{}}\n</tool_call>\n</div>visible",
+    );
+    assert_eq!(calls.len(), 1);
+    assert_eq!(text, "</div>visible");
+}
+
+#[test]
+fn a_doubled_opener_swallows_a_pipe_form_duplicate_closer() {
+    // `TAG_RE` matches the pipe-form closer `<|/tool_call|>` too, so the
+    // doubled-opener path must recognize it as a closer to swallow, not
+    // leave it dangling as narrative text.
+    let (text, calls) = parse(
+        "<|tool_call|>\n<|tool_call|>\n{\"name\":\"echo\",\"arguments\":{}}\n<|/tool_call|>\n<|/tool_call|>\nafter",
+    );
+    assert_eq!(calls.len(), 1, "{calls:?}");
+    assert_eq!(text, "after");
+}
+
+#[test]
+fn a_doubled_opener_swallows_a_newline_leaked_duplicate_closer() {
+    // `TAG_RE`'s `\s` matches any whitespace, not just space and tab, so
+    // `<\n/tool_call>` is one complete closer marker; `is_closing_marker`
+    // must classify it as such too, or the extra closer is left behind for
+    // the narrative to leak.
+    let (text, calls) = parse(
+        "<tool_call>\n<tool_call>\n{\"name\":\"echo\",\"arguments\":{}}\n</tool_call>\n<\n/tool_call>\nafter",
+    );
+    assert_eq!(calls.len(), 1, "{calls:?}");
+    assert_eq!(text, "after");
+}
+
+#[test]
+fn a_bare_trailing_opener_is_dropped_not_shown() {
+    // An abandoned block at the end of a reply carries no call and no
+    // information; showing `<tool_call>` to the user is never right.
+    let (text, calls) = parse("Let me fetch a few sites directly.\n\n<tool_call>\n");
+    assert!(calls.is_empty());
+    assert_eq!(text, "Let me fetch a few sites directly.");
+    // A block with real (if unparseable) content is still kept as text.
+    let (text, _) = parse("before <tool-call>not-json");
+    assert_eq!(text, "before <tool-call>not-json");
+}

crates/tinytools-agent/src/stream/test.rs

@@ -252,3 +252,33 @@ fn a_code_call_split_mid_string_is_released_once_and_never_shown() {
     assert!(!out.contains("echo("), "markup leaked: {out:?}");
     assert!(out.contains("Sure.") && out.contains("done"));
 }
+
+#[test]
+fn a_doubled_blocks_extra_closer_split_across_fragments_never_leaks() {
+    // The doubled opener's *inner* closer can arrive in one fragment and the
+    // matching extra closer in the next. The block must stay pending across
+    // that boundary rather than finalize on the inner closer alone and let
+    // the later `</tool_call>` fall through as visible text.
+    let (out, calls) = scrub_all(&[
+        "before <tool_call>\n<tool_call>\n{\"name\":\"x\",\"arguments\":{}}\n</tool_call>\n",
+        "</tool_call> after",
+    ]);
+    assert_eq!(out, "before  after");
+    assert_eq!(calls, 1);
+    assert!(!out.contains("tool_call"), "{out:?}");
+}
+
+#[test]
+fn a_doubled_blocks_extra_closer_split_mid_marker_never_leaks() {
+    // The fragment boundary can land *inside* the extra closer itself, not
+    // just before it: `</tool_` in one fragment, `call>` in the next. That
+    // partial marker must be held rather than released as text once its
+    // first fragment is scanned.
+    let (out, calls) = scrub_all(&[
+        "before <tool_call>\n<tool_call>\n{\"name\":\"x\",\"arguments\":{}}\n</tool_call>\n</tool_",
+        "call> after",
+    ]);
+    assert_eq!(out, "before  after");
+    assert_eq!(calls, 1);
+    assert!(!out.contains("tool_"), "{out:?}");
+}