From bf469e2cbdbd918d5f4e49fb28a81ce4275d2f29 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 20:11:06 +0300 Subject: [PATCH 1/3] fix(parse): accept DeepSeek DSML markers and fullwidth pipes on tool call tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tagged grammar now matches tool call tags that carry a `DeepSeek` DSML marker (e.g., `<｜DSML｜tool_call>`) or use the fullwidth pipe `｜` that chat templates actually emit, both of which were previously missed. This prevents a `deepseek` turn that uses the tag form from being parsed as prose and silently dropping the call. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/parse/grammar/tagged.rs | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 4f01ad6..25c1266 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -9,7 +9,13 @@ //! templates); //! * sentinel pipes leaked into the markers, in any position: //! `<|tool_call>…`, `<|tool_call|>…<|tool_call|>`, -//! `…`; +//! `…`, including the fullwidth `｜` those templates +//! actually emit; +//! * a `DeepSeek` DSML marker on the tag itself, +//! `<｜DSML｜tool_call>…` — the same marker +//! [`super::invoke_xml`] already accepts on ``, which this family +//! used to miss, so a `deepseek` turn that chose the tag form over the +//! invoke form parsed as prose and the call was silently dropped; //! * a `call:` prefix before the body; //! * a fenced block instead of a tag, ```` ```tool_call … ``` ````, sometimes //! closed by a stray ``; @@ -39,11 +45,15 @@ use crate::types::{CallSource, ParseOptions, ParsedToolCall}; pub(crate) struct Tagged; /// Any tag-family marker: ``, ``, ``, with -/// pipes, a slash, or whitespace leaked in, and an optional attribute list. -/// `` (plural, a JSON key) and `` do not match: -/// the name must end at a pipe, slash, whitespace, or `>`. -static TAG_RE: LazyLock> = - LazyLock::new(|| Regex::new(r"(?i)<[|/\s]*tool[_-]?call(?:[|/\s]*|\s+[^>]*)>").ok()); +/// pipes (ASCII `|` or the fullwidth `｜` chat templates emit), a slash, +/// whitespace, or a `DeepSeek` DSML marker leaked in, and an optional +/// attribute list. `` (plural, a JSON key, and the DSML wrapper +/// element) and `` do not match: the name must end at a pipe, +/// slash, whitespace, or `>`. +static TAG_RE: LazyLock> = LazyLock::new(|| { + Regex::new(r"(?i)<[|\u{ff5c}/\s]*(?:DSML[|\u{ff5c}/\s]*)?tool[_-]?call(?:[|\u{ff5c}/\s]*|\s+[^>]*)>") + .ok() +}); /// Openers a fenced block can carry. `` ```tool_calls `` (plural) is listed /// separately from `` ```tool_call `` rather than relying on a prefix match: From 245e749057f96d68c94aef79a9340238fff9055e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 20:11:23 +0300 Subject: [PATCH 2/3] test(parse): add tests for DeepSeek DSML tool-call tag variants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add test coverage for DeepSeek's DSML marker on the `tool_call` tag, including fullwidth bar variants and the ASCII spelling, to ensure these formats are correctly parsed as tool calls rather than prose. Also verify that the plural `<｜DSML｜tool_calls>` wrapper element is not mistakenly treated as a call marker, which would cause the inner call to be lost. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinytools-agent/src/parse/test/tagged.rs | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 93bfbb6..d02d794 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -513,3 +513,59 @@ fn a_bare_trailing_opener_is_dropped_not_shown() { let (text, _) = parse("before not-json"); assert_eq!(text, "before not-json"); } + +// ── DeepSeek DSML on the tag family ───────────────────────────────────────── + +/// `deepseek` emits its DSML marker on the `tool_call` tag, not only on +/// ``, and with the fullwidth bar its template actually uses. +/// Observed live from `deepseek-v4-flash` driving the code dialect: the call +/// parsed as prose, so the turn ended with the model's lead-in and the tool +/// was never run. +#[test] +fn dsml_marker_on_the_tool_call_tag_parses() { + let raw = concat!( + "I'll look it up.\n\n", + "<｜DSML｜tool_call>\n", + "{\"name\": \"GMAIL_FETCH_EMAILS\", \"arguments\": {\"max_results\": 1}}\n", + "" + ); + let (text, calls) = crate::parse::parse_tool_calls(raw); + assert_eq!(calls.len(), 1, "DSML tool_call tag must parse: {calls:?}"); + assert_eq!(calls[0].name, "GMAIL_FETCH_EMAILS"); + assert_eq!(text.trim(), "I'll look it up."); +} + +/// Doubled bars and the ASCII spelling are the same marker. +#[test] +fn dsml_marker_variants_on_the_tag_parse() { + for open_close in [ + ("<｜｜DSML｜｜tool_call>", ""), + ("<|DSML|tool_call>", ""), + ("<｜DSML｜tool_call>", "<｜DSML｜tool_call>"), + ] { + let raw = format!( + "{}\n{{\"name\": \"echo\", \"arguments\": {{}}}}\n{}", + open_close.0, open_close.1 + ); + let (_, calls) = crate::parse::parse_tool_calls(&raw); + assert_eq!(calls.len(), 1, "variant {open_close:?} must parse: {calls:?}"); + assert_eq!(calls[0].name, "echo"); + } +} + +/// The DSML *wrapper* element is plural and is not a call marker; treating it +/// as one would open a block on the wrapper and close it on the first inner +/// tag, losing the call inside. +#[test] +fn the_plural_dsml_wrapper_is_not_a_tag_marker() { + let raw = concat!( + "<｜DSML｜tool_calls>\n", + "<｜DSML｜tool_call>\n", + "{\"name\": \"echo\", \"arguments\": {}}\n", + "\n", + "" + ); + let (_, calls) = crate::parse::parse_tool_calls(raw); + assert_eq!(calls.len(), 1, "the inner call is the only call: {calls:?}"); + assert_eq!(calls[0].name, "echo"); +} From b47ccd1a58ec1bb0de51f8a732261bda701bbab6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 20:11:40 +0300 Subject: [PATCH 3/3] fix(parse): reformat regex and assertion for readability Reformat the `TAG_RE` regex construction and the `dsml_marker_variants_on_the_tag_parse` test assertion to use multi-line formatting, improving code readability without changing any behavior. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-agent/src/parse/grammar/tagged.rs | 6 ++++-- crates/tinytools-agent/src/parse/test/tagged.rs | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 25c1266..8fe09ef 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -51,8 +51,10 @@ pub(crate) struct Tagged; /// element) and `` do not match: the name must end at a pipe, /// slash, whitespace, or `>`. static TAG_RE: LazyLock> = LazyLock::new(|| { - Regex::new(r"(?i)<[|\u{ff5c}/\s]*(?:DSML[|\u{ff5c}/\s]*)?tool[_-]?call(?:[|\u{ff5c}/\s]*|\s+[^>]*)>") - .ok() + Regex::new( + r"(?i)<[|\u{ff5c}/\s]*(?:DSML[|\u{ff5c}/\s]*)?tool[_-]?call(?:[|\u{ff5c}/\s]*|\s+[^>]*)>", + ) + .ok() }); /// Openers a fenced block can carry. `` ```tool_calls `` (plural) is listed diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index d02d794..aca11db 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -548,7 +548,11 @@ fn dsml_marker_variants_on_the_tag_parse() { open_close.0, open_close.1 ); let (_, calls) = crate::parse::parse_tool_calls(&raw); - assert_eq!(calls.len(), 1, "variant {open_close:?} must parse: {calls:?}"); + assert_eq!( + calls.len(), + 1, + "variant {open_close:?} must parse: {calls:?}" + ); assert_eq!(calls[0].name, "echo"); } }