From 43610828ce10f3aba4debbb689aba0f977d5f06e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 22 Sep 2026 22:57:54 +0000 Subject: [PATCH 1/2] chat : accept packed XML tool calls in the Qwen3-Coder parser MiMo-V2.6-Distill-Qwen-9B uses the Qwen3-Coder XML tool-call format but emits it with no newlines between tags: v The parser hardcoded "\n\n", ">\n", "\n" and "\n", so the first argument of a packed call consumed every later tag, the grammar never accepted, and generation ran to max_tokens with tool_calls empty. Make the newlines around the tags optional. Argument values are scanned for "\n" before "", so output from models that emit the newlines parses exactly as before. Add parse tests for the packed, half-packed, and packed parallel forms. --- common/chat.cpp | 29 +++++++++++++++++++++-------- tests/test-chat.cpp | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 743ecde0a77e..b5d0655ab827 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1246,9 +1246,18 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ // Tool call parser if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto arg_close = p.tool_arg_close(p.literal("\n\n")); + // Newlines around the XML tags are optional: Qwen3-Coder emits `\nv\n\n`, + // MiMo-V2.6-Distill-Qwen-9B packs them as `v`. + // With a hardcoded "\n\n" the first packed argument swallowed every later + // tag, the grammar never accepted, and generation ran to max_tokens. + // Scan longest-first so a trailing "\n" stays out of the value for models that emit it. + const std::vector arg_close_scan = { "\n", "" }; + auto arg_close = p.tool_arg_close(p.optional(p.literal("\n")) + p.literal("")); + // ac() builds its automaton from the delimiter alone, so keep it to the bare + // "" (a longer variant is a prefix trap) and absorb trailing whitespace outside. auto arg_string = p.rule("xml-arg-string", - p.ac(p.tool_arg_string_value(p.until("\n\n")) + arg_close, "\n\n")); + p.ac(p.tool_arg_string_value(p.until_one_of(arg_close_scan)) + arg_close, + std::string("")) + p.space()); auto tool_choice = p.choice(); foreach_function(inputs.tools, [&](const json & tool) { @@ -1265,11 +1274,13 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) { auto rule_name = "tool-" + name + "-arg-" + param_name; - auto arg_open = p.tool_arg_open("\n"); + // one optional "\n", not space(), so further leading whitespace stays in the value + auto arg_open = p.tool_arg_open("" + p.optional(p.literal("\n"))); auto arg_value = schema_info.resolves_to_string(param_schema) ? arg_string : - p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close; + p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close + p.space(); auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); @@ -1283,9 +1294,10 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ args = args + p.zero_or_more(p.choice(optional_args)); } - auto func = p.tool(p.tool_open("\n") + + auto func = p.tool(p.tool_open("" + p.optional(p.literal("\n"))) + p.tool_args(args) + - p.tool_close(p.literal("\n"))); + p.tool_close(p.literal("") + p.space())); tool_choice |= p.rule("tool-" + name, func); }); @@ -1293,11 +1305,12 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; auto tool_call_body = tool_choice + "" + p.space(); - auto tool_call = p.rule("tool-call", "\n" + tool_call_body); + auto tool_call_open = p.literal("") + p.optional(p.literal("\n")); + auto tool_call = p.rule("tool-call", tool_call_open + tool_call_body); // Qwen3-Coder models may occasionally omit the token. auto tool_call_first = is_qwen3_coder ? - p.rule("tool-call-first", p.optional(p.literal("\n")) + tool_call_body) : + p.rule("tool-call-first", p.optional(tool_call_open) + tool_call_body) : tool_call; auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first; diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 7918f0ffcf48..316a1dd9af9e 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -3511,6 +3511,39 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_reconstruction() .run(); + // MiMo-V2.6-Distill-Qwen-9B packs the tags with no newlines. Parse-only: the template + // renders the newline form, so there is no reconstruction to check. + tst.test("1") + .tools({ special_function_tool }) + .expect(message_assist_call) + .run(); + + // Half-packed: newline after the opening tags, none after . + tst.test( + "\n" + "\n" + "\n" + "1") + .tools({ special_function_tool }) + .expect(message_assist_call) + .run(); + + // Packed and parallel: a strict "\n\n" delimiter swallowed the second call + // into the first argument. + tst.test( + "1" + "1" + "2") + .parallel_tool_calls(true) + .tools({ + special_function_tool, special_function_tool_with_optional_param + }) + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} }, + }) + .run(); + // Some models skip the opening and go straight to tst.test( "\n" From 8b9c5b2e0f93ed38c484ce2240c3d814c08ac89a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 22 Sep 2026 23:16:42 +0000 Subject: [PATCH 2/2] chat : keep literal in Qwen3-Coder string arguments A string value now ends at only when the next tag follows, so a literal inside the value is kept, as it was before the packed-format change. The closing tag is confirmed with a lookahead so a partial stream does not close the value early. The grammar accepts the original newline form or the packed form, so constrained sampling for newline-emitting models is unchanged. Shorten comments and add a test for a literal in a value. --- common/chat.cpp | 27 ++++++++++++++++----------- tests/test-chat.cpp | 24 +++++++++++++++++++----- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index b5d0655ab827..095f92d2fa25 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1246,18 +1246,24 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ // Tool call parser if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - // Newlines around the XML tags are optional: Qwen3-Coder emits `\nv\n\n`, - // MiMo-V2.6-Distill-Qwen-9B packs them as `v`. - // With a hardcoded "\n\n" the first packed argument swallowed every later - // tag, the grammar never accepted, and generation ran to max_tokens. - // Scan longest-first so a trailing "\n" stays out of the value for models that emit it. - const std::vector arg_close_scan = { "\n", "" }; + // Newlines around tags are optional (MiMo-V2.6-Distill-Qwen-9B emits none). A value ends + // at only when the next tag follows, so a literal is kept. + std::vector arg_close_scan; + for (const char * nl : { "\n", "" }) { + for (const char * ws : { "", "\n", "\n\n", "\r\n", " " }) { + for (const char * next : { "" }) { + arg_close_scan.push_back(std::string(nl) + "" + ws + next); + } + } + } auto arg_close = p.tool_arg_close(p.optional(p.literal("\n")) + p.literal("")); - // ac() builds its automaton from the delimiter alone, so keep it to the bare - // "" (a longer variant is a prefix trap) and absorb trailing whitespace outside. + // peek: do not close the value early while streaming + auto next_tag = p.peek(p.space() + p.choice({ p.literal("") })); + auto arg_value = p.tool_arg_string_value(p.until_one_of(arg_close_scan)) + + p.tool_arg_close(p.optional(p.literal("\n")) + p.literal("") + next_tag); + // grammar: newline form or packed form auto arg_string = p.rule("xml-arg-string", - p.ac(p.tool_arg_string_value(p.until_one_of(arg_close_scan)) + arg_close, - std::string("")) + p.space()); + p.choice({ p.ac(arg_value, "\n\n"), p.ac(arg_value, "") }) + p.space()); auto tool_choice = p.choice(); foreach_function(inputs.tools, [&](const json & tool) { @@ -1274,7 +1280,6 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) { auto rule_name = "tool-" + name + "-arg-" + param_name; - // one optional "\n", not space(), so further leading whitespace stays in the value auto arg_open = p.tool_arg_open("" + p.optional(p.literal("\n"))); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 316a1dd9af9e..1f543d4cf0b6 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -3511,14 +3511,13 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_reconstruction() .run(); - // MiMo-V2.6-Distill-Qwen-9B packs the tags with no newlines. Parse-only: the template - // renders the newline form, so there is no reconstruction to check. + // Packed, no newlines (MiMo-V2.6-Distill-Qwen-9B) tst.test("1") .tools({ special_function_tool }) .expect(message_assist_call) .run(); - // Half-packed: newline after the opening tags, none after . + // Half-packed tst.test( "\n" "\n" @@ -3528,8 +3527,23 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect(message_assist_call) .run(); - // Packed and parallel: a strict "\n\n" delimiter swallowed the second call - // into the first argument. + // Literal inside a string value + tst.test( + "\n" + "\n" + "\n" + "print(\"\")\n" + "\n" + "\n" + "") + .tools({ python_tool }) + .expect_tool_calls({ + { "python", R"j({"code": "print(\"\")"})j", {} }, + }) + .expect_reconstruction() + .run(); + + // Packed parallel calls tst.test( "1" "1"