From 7653ac1e857eeb00378066cfcd1b0bd215f0e176 Mon Sep 17 00:00:00 2001
From: co-seven <linxi.cai@spacemit.com>
Date: Fri, 21 Aug 2026 09:30:01 +0000
Subject: [PATCH 1/2] feat(server): integrate isolated multi-ASR service

---
 common/arg.cpp                                |   7 +
 common/common.h                               |   1 +
 tools/server/server-context.cpp               |  91 +++++-
 tools/smt-mtmd/CMakeLists.txt                 |  12 +
 .../smt/multi-asr/multi-asr-common.cpp        |  94 ++++++
 .../smt-mtmd/smt/multi-asr/multi-asr-common.h | 105 +++++++
 .../smt/multi-asr/multi-asr-decoder.cpp       | 288 ++++++++++++++++++
 .../smt/multi-asr/multi-asr-decoder.h         |  67 ++++
 .../smt/multi-asr/multi-asr-encoder.cpp       |  89 ++++++
 .../smt/multi-asr/multi-asr-encoder.h         |  36 +++
 .../smt/multi-asr/multi-asr-orchestrator.cpp  | 188 ++++++++++++
 .../smt/multi-asr/multi-asr-orchestrator.h    |  86 ++++++
 .../smt/multi-asr/multi-asr-service.cpp       |  81 +++++
 .../smt/multi-asr/multi-asr-service.h         |  25 ++
 14 files changed, 1166 insertions(+), 4 deletions(-)
 create mode 100644 tools/smt-mtmd/smt/multi-asr/multi-asr-common.cpp
 create mode 100644 tools/smt-mtmd/smt/multi-asr/multi-asr-common.h
 create mode 100644 tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.cpp
 create mode 100644 tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.h
 create mode 100644 tools/smt-mtmd/smt/multi-asr/multi-asr-encoder.cpp
 create mode 100644 tools/smt-mtmd/smt/multi-asr/multi-asr-encoder.h
 create mode 100644 tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.cpp
 create mode 100644 tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.h
 create mode 100644 tools/smt-mtmd/smt/multi-asr/multi-asr-service.cpp
 create mode 100644 tools/smt-mtmd/smt/multi-asr/multi-asr-service.h

diff --git a/common/arg.cpp b/common/arg.cpp
index 73be78a6a9c..18eb4c05a0c 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -2381,6 +2381,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.smt_config_dir = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_MTMD}).set_env("LLAMA_ARG_SMT_CONFIG_DIR"));
+    add_opt(common_arg(
+        {"--smt-multi-asr"},
+        "enable the isolated legacy multi-ASR FIFO service (default: disabled)",
+        [](common_params & params) {
+            params.smt_multi_asr = true;
+        }
+    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_SMT_MULTI_ASR"));
 #endif
     add_opt(common_arg(
         {"--image", "--audio", "--video"}, "FILE",
diff --git a/common/common.h b/common/common.h
index 5aa8d46b56d..d8ede71d9b2 100644
--- a/common/common.h
+++ b/common/common.h
@@ -598,6 +598,7 @@ struct common_params {
 #if defined(LLAMA_SERVER_SMT_MTMD)
     std::string media_backend = "auto"; // multimodal backend: auto|mtmd|smt
     std::string smt_config_dir;         // SMT config dir (config.json + ONNX)
+    bool smt_multi_asr = false;         // opt in to the isolated legacy multi-ASR FIFO service
 #endif
     std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
     int image_min_tokens = -1;
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 585a993f9f1..69f74f30444 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -8,6 +8,9 @@
 #include "server-queue.h"
 #include "server-schema.h"
 #include "server-stream.h"
+#if defined(LLAMA_SERVER_SMT_MTMD)
+#include "multi-asr-service.h"
+#endif
 
 #include "build-info.h"
 #include "common.h"
@@ -894,6 +897,9 @@ struct server_context_impl {
     server_media_context * media = nullptr;
     mtmd_context * mctx = nullptr;
     const llama_vocab * vocab = nullptr;
+#if defined(LLAMA_SERVER_SMT_MTMD)
+    std::unique_ptr<smt_multi_asr_service> smt_asr_service;
+#endif
 
     server_queue    queue_tasks;
     server_response queue_results;
@@ -972,6 +978,11 @@ struct server_context_impl {
     int64_t t_last_load_progress_ms = 0;
 
     void destroy() {
+#if defined(LLAMA_SERVER_SMT_MTMD)
+        // The isolated ASR context borrows model_tgt; tear it down before the
+        // owning common_init_result releases the model.
+        smt_asr_service.reset();
+#endif
         spec.reset();
         spec_init.reset();
 
@@ -1050,8 +1061,10 @@ struct server_context_impl {
 #if defined(LLAMA_SERVER_SMT_MTMD)
         const bool has_smt_media = !params.smt_config_dir.empty() &&
                 (params.media_backend == "auto" || params.media_backend == "smt");
+        const bool use_multi_asr = has_smt_media && !has_mmproj && params.smt_multi_asr;
 #else
         const bool has_smt_media = false;
+        const bool use_multi_asr = false;
 #endif
         const bool has_draft = params.speculative.has_dft();
         const bool spec_mtp = std::find(params_base.speculative.types.begin(),
@@ -1059,6 +1072,24 @@ struct server_context_impl {
                                         COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end();
         const bool has_spec = has_draft || spec_mtp;
 
+#if defined(LLAMA_SERVER_SMT_MTMD)
+        // Initialize the legacy ASR stack before loading the server's own
+        // llama model.  SpacemiT's backend keeps process-wide state, and the
+        // proven standalone implementation is sensitive to a second model
+        // being loaded ahead of it.
+        if (use_multi_asr && !smt_asr_service) {
+            try {
+                smt_asr_service = std::make_unique<smt_multi_asr_service>();
+                smt_asr_service->init(nullptr, params_base);
+                SRV_INF("%s", "initialized isolated SMT multi-ASR service\n");
+            } catch (const std::exception & e) {
+                SRV_ERR("failed to initialize isolated SMT multi-ASR service: %s\n", e.what());
+                smt_asr_service.reset();
+                return false;
+            }
+        }
+#endif
+
         if (callback_state) {
             std::vector<std::string> stages = {"text_model"};
             if (has_spec) {
@@ -1078,7 +1109,9 @@ struct server_context_impl {
         }
 
         std::unique_ptr<media_worker> media_worker_init;
-        if (has_mmproj || has_smt_media) {
+        // Default SMT audio keeps the original generic media path. Only the
+        // explicit multi-ASR mode uses the isolated legacy service.
+        if (has_mmproj || (has_smt_media && !use_multi_asr)) {
             std::string worker_backend = "mtmd";
 #if defined(LLAMA_SERVER_SMT_MTMD)
             if (params_base.media_backend == "smt" || (params_base.media_backend == "auto" && !has_mmproj && has_smt_media)) {
@@ -1248,7 +1281,7 @@ struct server_context_impl {
             load_progress_callback(1.0f, &load_progress_spec);
         }
 
-        if (has_mmproj || has_smt_media) {
+        if (has_mmproj || (has_smt_media && !use_multi_asr)) {
             if (callback_state) {
                 callback_state(SERVER_STATE_LOADING, {{"stage", has_mmproj ? "mmproj_model" : "media_model"}});
             }
@@ -1534,7 +1567,7 @@ struct server_context_impl {
                 /* chat_template_kwargs  */ params_base.default_template_kwargs,
                 /* tmpls                 */ std::move(chat_templates),
                 /* allow_image           */ media ? media->supports_vision() : false,
-                /* allow_audio           */ media ? media->supports_audio()  : false,
+                /* allow_audio           */ (media ? media->supports_audio() : false) || smt_asr_service != nullptr,
                 /* allow_video           */ media ? media->supports_video()  : false,
                 /* enable_thinking       */ enable_thinking,
                 /* reasoning_budget      */ params_base.sampling.reasoning_budget_tokens,
@@ -4030,7 +4063,8 @@ server_context_meta server_context::get_meta() const {
         /* model_aliases          */ impl->model_aliases,
         /* model_tags             */ impl->model_tags,
         /* model_path             */ impl->params_base.model.path,
-        /* has_mtmd               */ impl->media != nullptr && impl->media->supports_prompt_embeddings(),
+        /* has_mtmd               */ (impl->media != nullptr && impl->media->supports_prompt_embeddings()) ||
+                                      impl->smt_asr_service != nullptr,
         /* has_inp_image          */ impl->chat_params.allow_image,
         /* has_inp_audio          */ impl->chat_params.allow_audio,
         /* has_inp_video          */ impl->chat_params.allow_video,
@@ -4116,6 +4150,55 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
     int32_t sse_ping_interval = params.sse_ping_interval;
 
     try {
+        // Audio-only SMT requests use the isolated legacy ASR orchestrator.
+        // This synchronous facade is intentionally ahead of task creation:
+        // no server slot, PEG parser, prompt cache, or shared KV sequence is
+        // touched by these requests.
+#if defined(LLAMA_SERVER_SMT_MTMD)
+        const std::string raw_prompt = data.contains("prompt") && data.at("prompt").is_string()
+                ? data.at("prompt").get<std::string>() : std::string();
+        // oaicompat_chat_params_parse() has already decoded input_audio into
+        // `files`; chat-template rendering may remove the media marker from
+        // the resulting prompt, so do not use the rendered text to decide
+        // whether this is an audio request.
+        const bool has_audio_media = !files.empty();
+        if (ctx_server.smt_asr_service && has_audio_media) {
+            multi_asr_request asr_result;
+            const int32_t n_predict = json_value(data, "max_tokens", 256);
+            const std::string user_prompt = raw_prompt;
+            const bool ok = ctx_server.smt_asr_service->submit(files.front(), user_prompt, n_predict, asr_result);
+            if (!ok) {
+                res->error(format_error_response(asr_result.error.empty() ? "ASR request failed" : asr_result.error,
+                                                 ERROR_TYPE_SERVER));
+                return res;
+            }
+            const json choice = {
+                {"index", 0},
+                {"message", {{"role", "assistant"}, {"content", asr_result.text}}},
+                {"finish_reason", "stop"},
+            };
+            res->ok({
+                {"id", completion_id},
+                {"object", "chat.completion"},
+                {"created", std::time(nullptr)},
+                {"model", meta->model_name},
+                {"choices", json::array({choice})},
+                {"usage", {{"prompt_tokens", asr_result.timings.n_audio_tokens},
+                            {"completion_tokens", asr_result.timings.n_out_tokens},
+                            {"total_tokens", asr_result.timings.n_audio_tokens + asr_result.timings.n_out_tokens}}},
+                {"multi_asr_timings", {
+                    {"queue_ms", asr_result.timings.queue_ms},
+                    {"encode_ms", asr_result.timings.encode_ms},
+                    {"prefill_ms", asr_result.timings.prefill_ms},
+                    {"decode_ms", asr_result.timings.decode_ms},
+                    {"total_ms", asr_result.timings.total_ms},
+                    {"n_audio_tokens", asr_result.timings.n_audio_tokens},
+                    {"n_out_tokens", asr_result.timings.n_out_tokens},
+                }},
+            });
+            return res;
+        }
+#endif
         std::vector<server_task> tasks;
 
         const auto & prompt = data.at("prompt");
diff --git a/tools/smt-mtmd/CMakeLists.txt b/tools/smt-mtmd/CMakeLists.txt
index 017ffeb553b..4cb795a1a5f 100644
--- a/tools/smt-mtmd/CMakeLists.txt
+++ b/tools/smt-mtmd/CMakeLists.txt
@@ -45,6 +45,16 @@ if(LLAMA_SERVER_SMT_MTMD)
     target_sources(${TARGET} PRIVATE
         smt/smt-audio-wrapper.cpp
         smt/smt-audio-wrapper.h
+        smt/multi-asr/multi-asr-common.cpp
+        smt/multi-asr/multi-asr-common.h
+        smt/multi-asr/multi-asr-encoder.cpp
+        smt/multi-asr/multi-asr-encoder.h
+        smt/multi-asr/multi-asr-decoder.cpp
+        smt/multi-asr/multi-asr-decoder.h
+        smt/multi-asr/multi-asr-orchestrator.cpp
+        smt/multi-asr/multi-asr-orchestrator.h
+        smt/multi-asr/multi-asr-service.cpp
+        smt/multi-asr/multi-asr-service.h
         smt/smt-media-common.cpp
         smt/smt-media-common.h
         smt/smt-profile.h
@@ -64,8 +74,10 @@ if(LLAMA_SERVER_SMT_MTMD)
     )
     target_include_directories(${TARGET} PUBLIC
         ${CMAKE_CURRENT_SOURCE_DIR}/smt
+        ${CMAKE_CURRENT_SOURCE_DIR}/smt/multi-asr
     )
     target_include_directories(${TARGET} PRIVATE
+        ${CMAKE_SOURCE_DIR}/common
         ${CMAKE_SOURCE_DIR}/tools/mtmd
         ${CMAKE_SOURCE_DIR}/vendor
         ${CMAKE_SOURCE_DIR}/ggml/src
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-common.cpp b/tools/smt-mtmd/smt/multi-asr/multi-asr-common.cpp
new file mode 100644
index 00000000000..034b39bbee3
--- /dev/null
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-common.cpp
@@ -0,0 +1,94 @@
+#include "multi-asr-common.h"
+
+#include <cctype>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <sstream>
+#include <string>
+
+// Parse one segment: either a single core "8" or a dash range "8-11".
+// Returns true on success and sets the corresponding mask bits.
+static bool parse_one_segment(const std::string & seg, bool (&mask)[GGML_MAX_N_THREADS]) {
+    if (seg.empty()) {
+        return false;
+    }
+
+    const size_t dash = seg.find('-');
+    if (dash == std::string::npos) {
+        // single cpu index
+        char *     endp = nullptr;
+        const long cpu  = std::strtol(seg.c_str(), &endp, 10);
+        if (endp == seg.c_str() || *endp != '\0' || cpu < 0 || cpu >= GGML_MAX_N_THREADS) {
+            return false;
+        }
+        mask[cpu] = true;
+        return true;
+    }
+
+    // dash range "lo-hi"
+    const std::string lo_s = seg.substr(0, dash);
+    const std::string hi_s = seg.substr(dash + 1);
+    char *            e1   = nullptr;
+    char *            e2   = nullptr;
+    const long        lo   = std::strtol(lo_s.c_str(), &e1, 10);
+    const long        hi   = std::strtol(hi_s.c_str(), &e2, 10);
+    if (e1 == lo_s.c_str() || *e1 != '\0' || e2 == hi_s.c_str() || *e2 != '\0') {
+        return false;
+    }
+    if (lo < 0 || hi < 0 || lo >= GGML_MAX_N_THREADS || hi >= GGML_MAX_N_THREADS || lo > hi) {
+        return false;
+    }
+    for (long c = lo; c <= hi; ++c) {
+        mask[c] = true;
+    }
+    return true;
+}
+
+int multi_asr_parse_cpu_range(const std::string & range, bool (&mask)[GGML_MAX_N_THREADS]) {
+    std::memset(mask, 0, sizeof(bool) * GGML_MAX_N_THREADS);
+    if (range.empty()) {
+        return 0;
+    }
+
+    // Split on commas; each part is a single cpu or a dash range.
+    std::stringstream ss(range);
+    std::string       part;
+    while (std::getline(ss, part, ',')) {
+        // trim ASCII whitespace
+        size_t b = 0;
+        size_t e = part.size();
+        while (b < e && std::isspace((unsigned char) part[b])) {
+            ++b;
+        }
+        while (e > b && std::isspace((unsigned char) part[e - 1])) {
+            --e;
+        }
+        const std::string seg = part.substr(b, e - b);
+        if (seg.empty()) {
+            continue;
+        }
+        if (!parse_one_segment(seg, mask)) {
+            return -1;
+        }
+    }
+
+    int count = 0;
+    for (int i = 0; i < GGML_MAX_N_THREADS; ++i) {
+        count += mask[i] ? 1 : 0;
+    }
+    return count;
+}
+
+std::string multi_asr_cpu_mask_to_string(const bool (&mask)[GGML_MAX_N_THREADS]) {
+    std::string out;
+    for (int i = 0; i < GGML_MAX_N_THREADS; ++i) {
+        if (mask[i]) {
+            if (!out.empty()) {
+                out += ",";
+            }
+            out += std::to_string(i);
+        }
+    }
+    return out;
+}
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-common.h b/tools/smt-mtmd/smt/multi-asr/multi-asr-common.h
new file mode 100644
index 00000000000..50ea6d21eb3
--- /dev/null
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-common.h
@@ -0,0 +1,105 @@
+#pragma once
+
+// Multi-ASR 4-way concurrent orchestrator — common types & config.
+//
+// Design (see docs/../multi-asr-design.md, scheme B1 "错峰流水"):
+//   - encoder stage : 1 ONNX session (smt_audio_context), pinned to encoder cores,
+//                     serial encode -> produces audio embedding chunk.
+//   - decoder stage : 1 llama_context slot, pinned to decoder cores, serial decode.
+//   - orchestrator  : FIFO queue (>N waits) + encode/decode threads that overlap
+//                     E(N+1) with D(N).
+// Core split (encoder cores vs decoder cores) is a RUNTIME INPUT, not hardcoded,
+// so the optimal ratio can be swept experimentally.
+
+#include "ggml.h"  // GGML_MAX_N_THREADS
+#include "llama.h"
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+
+struct multi_asr_params {
+    // model / backend
+    std::string model_path;      // gguf text model (decoder)
+    std::string smt_config_dir;  // dir with config.json + encoder ONNX
+    bool        warmup = true;
+
+    // HTTP
+    std::string host = "0.0.0.0";
+    int32_t     port = 8080;
+
+    // concurrency
+    int32_t n_parallel = 4;   // max in-flight requests before queueing (>N waits)
+    int32_t queue_max  = 64;  // hard cap on total (in-flight + waiting); reject beyond
+
+    // --- core split (the tunable input variable) ---
+    // encoder cores: set via the ONNX EP fields in config.json (spacemit_ep_*).
+    // decoder cores: applied to the llama_context ggml thread affinity (cpumask).
+    // Accepts a CPU range string like "10,11,14,15" or "12-15". Empty = leave backend default.
+    std::string decoder_cpu_range;      // e.g. "10,11,14,15"                or "12-15"
+    int32_t     decoder_n_threads = 0;  // 0 = derive from decoder_cpu_range size
+
+    // generation
+    int32_t n_predict = 256;  // max output tokens per request
+    int32_t n_ctx     = 0;    // 0 = from model / config.json context_size
+    int32_t n_batch   = 2048;
+
+    // pipeline
+    bool enable_pipeline = true;  // false = strict serial baseline (variable ②)
+};
+
+// ---------------------------------------------------------------------------
+// Per-request lifecycle
+// ---------------------------------------------------------------------------
+
+enum class multi_asr_stage : uint8_t {
+    queued   = 0,
+    encoding = 1,
+    decoding = 2,
+    done     = 3,
+    failed   = 4,
+};
+
+// Per-stage timing (ms), filled as the request flows through the pipeline.
+struct multi_asr_timings {
+    double  queue_ms       = 0.0;  // time spent waiting in FIFO before encode start
+    double  encode_ms      = 0.0;  // ONNX encoder wall time
+    double  prefill_ms     = 0.0;  // audio embedding prefill into gguf
+    double  decode_ms      = 0.0;  // token generation
+    double  total_ms       = 0.0;  // end-to-end (accept -> response)
+    int32_t n_audio_tokens = 0;
+    int32_t n_out_tokens   = 0;
+};
+
+// A single ASR request as it moves through encode -> decode.
+struct multi_asr_request {
+    uint64_t             id = 0;
+    std::vector<uint8_t> audio;   // raw wav bytes (decoded input)
+    std::string          prompt;  // text prompt, e.g. "language Chinese<asr_text>"
+    int32_t              n_predict = 256;
+
+    // filled by encoder stage: audio embedding (n_audio_tokens * hidden_size floats)
+    std::vector<float> embd;
+    int32_t            n_audio_tokens = 0;
+
+    // result
+    std::string       text;
+    multi_asr_stage   stage = multi_asr_stage::queued;
+    std::string       error;
+    multi_asr_timings timings;
+};
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+// Parse a CPU range string ("8,9,12,13" or "8-11") into a boolean affinity mask.
+// Returns the number of selected CPUs, or -1 on parse error.
+int multi_asr_parse_cpu_range(const std::string & range, bool (&mask)[GGML_MAX_N_THREADS]);
+
+// Convert a boolean affinity mask back to a compact "8,9,12,13" string (for logging).
+std::string multi_asr_cpu_mask_to_string(const bool (&mask)[GGML_MAX_N_THREADS]);
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.cpp b/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.cpp
new file mode 100644
index 00000000000..f685156428e
--- /dev/null
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.cpp
@@ -0,0 +1,288 @@
+#include "multi-asr-decoder.h"
+
+#include "ggml.h"
+#include "sampling.h"
+
+#include <algorithm>
+#include <cstring>
+#include <stdexcept>
+#include <vector>
+
+// Build the Qwen3-ASR prompt around the audio, mirroring
+// format_qwen3asr_audio_prompt() in tools/mtmd/mtmd-cli-smt.cpp:
+//   <|im_start|>system\n<|im_end|>\n<|im_start|>user\n <AUDIO> <|im_end|>\n<|im_start|>assistant\n<user text>
+// The audio embedding (+ <|audio_start|>/<|audio_end|>) is spliced where <AUDIO> is.
+// We return the text BEFORE the audio and the text AFTER the audio separately.
+struct qwen3asr_prompt_split {
+    std::string before;  // up to and including "<|im_start|>user\n"
+    std::string after;   // "<|im_end|>\n<|im_start|>assistant\n" + user text
+};
+
+static qwen3asr_prompt_split build_qwen3asr_prompt(const std::string & user_text) {
+    qwen3asr_prompt_split s;
+    s.before = "<|im_start|>system\n<|im_end|>\n<|im_start|>user\n";
+    s.after  = "<|im_end|>\n<|im_start|>assistant\n" + user_text;
+    return s;
+}
+
+multi_asr_decoder::~multi_asr_decoder() {
+    if (owns_context_ && lctx_ && !init_) {
+        llama_free(lctx_);
+    }
+    lctx_  = nullptr;
+    model_ = nullptr;
+}
+
+void multi_asr_decoder::init(const multi_asr_params & params, int64_t hidden_size) {
+    hidden_size_ = hidden_size;
+    n_batch_     = params.n_batch;
+
+    // Build common_params for model load + threading/affinity + sampling.
+    params_.model.path = params.model_path;
+    params_.n_ctx      = params.n_ctx;  // 0 -> from model
+    params_.n_batch    = params.n_batch;
+    params_.warmup     = params.warmup;
+
+    // Decoder thread count. Core binding for the ggml IME backend is set via the
+    // SPACEMIT_PERFER_CORE_ID environment variable (see run_server.sh), not here.
+    if (params.decoder_n_threads > 0) {
+        params_.cpuparams.n_threads = params.decoder_n_threads;
+    }
+
+    init_ = common_init_from_params(params_);
+    if (!init_) {
+        throw std::runtime_error("multi_asr_decoder: common_init_from_params returned null");
+    }
+    model_ = init_->model();
+    lctx_  = init_->context();
+    if (!model_ || !lctx_) {
+        throw std::runtime_error("multi_asr_decoder: failed to load model / create context");
+    }
+    vocab_ = llama_model_get_vocab(model_);
+
+    // Validate embedding dim.
+    const int model_n_embd = llama_model_n_embd(model_);
+    if ((int64_t) model_n_embd != hidden_size_) {
+        throw std::runtime_error("multi_asr_decoder: model n_embd (" + std::to_string(model_n_embd) +
+                                 ") != encoder hidden_size (" + std::to_string(hidden_size_) + ")");
+    }
+
+    // Qwen3-ASR audio boundary special tokens.
+    tok_audio_beg_ = common_tokenize(vocab_, "<|audio_start|>", /*add_special*/ false, /*parse_special*/ true);
+    tok_audio_end_ = common_tokenize(vocab_, "<|audio_end|>", /*add_special*/ false, /*parse_special*/ true);
+    owns_context_ = true;
+}
+
+void multi_asr_decoder::init_shared(llama_model * model, const multi_asr_params & params, int64_t hidden_size) {
+    if (model == nullptr) {
+        throw std::runtime_error("multi_asr_decoder: null shared model");
+    }
+    hidden_size_ = hidden_size;
+    n_batch_ = params.n_batch;
+    params_.model.path = params.model_path;
+    params_.n_ctx = params.n_ctx;
+    params_.n_batch = params.n_batch;
+    params_.n_ubatch = std::min(params.n_batch, 512);
+    params_.warmup = false;
+    if (params.decoder_n_threads > 0) {
+        params_.cpuparams.n_threads = params.decoder_n_threads;
+        params_.cpuparams_batch.n_threads = params.decoder_n_threads;
+    }
+    auto cparams = common_context_params_to_llama(params_);
+    cparams.n_seq_max = 1;
+    cparams.n_outputs_max = 1;
+    lctx_ = llama_init_from_model(model, cparams);
+    if (!lctx_) {
+        throw std::runtime_error("multi_asr_decoder: failed to create private context from shared model");
+    }
+    model_ = model;
+    vocab_ = llama_model_get_vocab(model_);
+    if (llama_model_n_embd(model_) != hidden_size_) {
+        llama_free(lctx_);
+        lctx_ = nullptr;
+        throw std::runtime_error("multi_asr_decoder: shared model embedding size mismatch");
+    }
+    tok_audio_beg_ = common_tokenize(vocab_, "<|audio_start|>", false, true);
+    tok_audio_end_ = common_tokenize(vocab_, "<|audio_end|>", false, true);
+    owns_context_ = true;
+}
+
+int multi_asr_decoder::decode_tokens(const std::vector<llama_token> & tokens, llama_pos & n_past, bool logits_last) {
+    if (tokens.empty()) {
+        return 0;
+    }
+    llama_batch batch = llama_batch_init(n_batch_, 0, 1);
+    size_t      i     = 0;
+    while (i < tokens.size()) {
+        batch.n_tokens = 0;
+        for (; i < tokens.size() && batch.n_tokens < n_batch_; ++i) {
+            const int32_t j    = batch.n_tokens;
+            batch.token[j]     = tokens[i];
+            batch.pos[j]       = n_past + j;
+            batch.n_seq_id[j]  = 1;
+            batch.seq_id[j][0] = 0;
+            batch.logits[j]    = false;
+            batch.n_tokens++;
+        }
+        if (logits_last && i == tokens.size()) {
+            batch.logits[batch.n_tokens - 1] = true;
+        }
+        if (llama_decode(lctx_, batch) != 0) {
+            llama_batch_free(batch);
+            return 1;
+        }
+        n_past += batch.n_tokens;
+    }
+    llama_batch_free(batch);
+    return 0;
+}
+
+int multi_asr_decoder::decode_embd(const float * embd, int n_tokens, llama_pos & n_past, bool logits_last) {
+    const int                   n_embd = (int) hidden_size_;
+    std::vector<llama_pos>      pos((size_t) n_tokens);
+    std::vector<int32_t>        n_seq_id((size_t) n_tokens);
+    std::vector<llama_seq_id>   seq_id_store((size_t) n_tokens);
+    std::vector<llama_seq_id *> seq_ids((size_t) n_tokens + 1);
+    std::vector<int8_t>         logits((size_t) n_tokens);
+    for (int i = 0; i < n_tokens; ++i) {
+        seq_id_store[i] = 0;
+        seq_ids[i]      = &seq_id_store[i];
+    }
+    seq_ids[n_tokens] = nullptr;
+
+    int processed = 0;
+    while (processed < n_tokens) {
+        const int  batch_size    = std::min(n_batch_, n_tokens - processed);
+        const bool is_last_batch = (processed + batch_size >= n_tokens);
+        for (int i = 0; i < batch_size; ++i) {
+            pos[processed + i]      = n_past + processed + i;
+            n_seq_id[processed + i] = 1;
+            logits[processed + i]   = (logits_last && is_last_batch && i == batch_size - 1) ? 1 : 0;
+        }
+        llama_batch batch = {
+            /*n_tokens =*/batch_size,
+            /*token    =*/nullptr,
+            /*embd     =*/const_cast<float *>(embd) + (size_t) processed * n_embd,
+            /*pos      =*/pos.data() + processed,
+            /*n_seq_id =*/n_seq_id.data() + processed,
+            /*seq_id   =*/seq_ids.data() + processed,
+            /*logits   =*/logits.data() + processed,
+        };
+        if (llama_decode(lctx_, batch) != 0) {
+            return 1;
+        }
+        processed += batch_size;
+    }
+    n_past += n_tokens;
+    return 0;
+}
+
+bool multi_asr_decoder::decode(multi_asr_request & req) {
+    if (!lctx_) {
+        req.error = "decoder not initialized";
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+    if (req.embd.empty() || req.n_audio_tokens <= 0) {
+        req.error = "decoder: empty audio embedding";
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+
+    // Fresh KV for this request (single slot, one request at a time).
+    llama_memory_clear(llama_get_memory(lctx_), true);
+
+    llama_pos                   n_past = 0;
+    const qwen3asr_prompt_split split  = build_qwen3asr_prompt(req.prompt);
+
+    const int64_t t_prefill0 = ggml_time_ms();
+
+    // 1) text before audio (with BOS/special parsing)
+    const std::vector<llama_token> toks_before =
+        common_tokenize(vocab_, split.before, /*add_special*/ true, /*parse_special*/ true);
+    if (decode_tokens(toks_before, n_past, /*logits_last*/ false) != 0) {
+        req.error = "decoder: failed to decode prompt prefix";
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+
+    // 2) <|audio_start|>
+    if (decode_tokens(tok_audio_beg_, n_past, /*logits_last*/ false) != 0) {
+        req.error = "decoder: failed to decode audio-begin";
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+
+    // 3) audio embedding
+    if (decode_embd(req.embd.data(), req.n_audio_tokens, n_past, /*logits_last*/ false) != 0) {
+        req.error = "decoder: failed to decode audio embedding";
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+
+    // 4) <|audio_end|>
+    if (decode_tokens(tok_audio_end_, n_past, /*logits_last*/ false) != 0) {
+        req.error = "decoder: failed to decode audio-end";
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+
+    // 5) assistant prefix + user text (logits on last -> ready to sample)
+    const std::vector<llama_token> toks_after =
+        common_tokenize(vocab_, split.after, /*add_special*/ false, /*parse_special*/ true);
+    if (decode_tokens(toks_after, n_past, /*logits_last*/ true) != 0) {
+        req.error = "decoder: failed to decode assistant prefix";
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+
+    req.timings.prefill_ms = (double) (ggml_time_ms() - t_prefill0);
+
+    // 6) generation loop
+    const int64_t    t_decode0 = ggml_time_ms();
+    common_sampler * smpl      = common_sampler_init(model_, params_.sampling);
+    if (!smpl) {
+        req.error = "decoder: failed to init sampler";
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+
+    std::vector<llama_token> generated;
+    const int                n_predict = req.n_predict > 0 ? req.n_predict : 256;
+    llama_batch              gen_batch = llama_batch_init(1, 0, 1);
+    bool                     ok        = true;
+    for (int i = 0; i < n_predict; ++i) {
+        const llama_token tok = common_sampler_sample(smpl, lctx_, -1);
+        common_sampler_accept(smpl, tok, true);
+        if (llama_vocab_is_eog(vocab_, tok)) {
+            break;
+        }
+        generated.push_back(tok);
+
+        gen_batch.n_tokens     = 1;
+        gen_batch.token[0]     = tok;
+        gen_batch.pos[0]       = n_past++;
+        gen_batch.n_seq_id[0]  = 1;
+        gen_batch.seq_id[0][0] = 0;
+        gen_batch.logits[0]    = true;
+        if (llama_decode(lctx_, gen_batch) != 0) {
+            req.error = "decoder: llama_decode failed during generation";
+            ok        = false;
+            break;
+        }
+    }
+    llama_batch_free(gen_batch);
+
+    req.timings.decode_ms    = (double) (ggml_time_ms() - t_decode0);
+    req.timings.n_out_tokens = (int32_t) generated.size();
+
+    if (ok) {
+        req.text  = common_detokenize(lctx_, generated, /*special*/ false);
+        req.stage = multi_asr_stage::done;
+    } else {
+        req.stage = multi_asr_stage::failed;
+    }
+
+    common_sampler_free(smpl);
+    return ok;
+}
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.h b/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.h
new file mode 100644
index 00000000000..e26c4799056
--- /dev/null
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.h
@@ -0,0 +1,67 @@
+#pragma once
+
+// Decoder stage: owns the gguf text model + a single llama_context (one slot).
+// Runs on its own worker thread, pinned to the decoder core set.
+// Given a request whose `embd` was filled by the encoder stage, it:
+//   1. decodes the Qwen3-ASR prompt prefix text tokens,
+//   2. decodes <|audio_start|> + audio embedding + <|audio_end|>,
+//   3. decodes the assistant prefix text tokens,
+//   4. generates output tokens serially until EOG / n_predict.
+//
+// This mirrors the faithful path in tools/mtmd/mtmd-cli-smt.cpp (eval_message_smt
+// + generate_response) but split so encode already happened on the encoder core set.
+
+#include "common.h"
+#include "llama.h"
+#include "multi-asr-common.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+class multi_asr_decoder {
+  public:
+    multi_asr_decoder() = default;
+    ~multi_asr_decoder();
+
+    multi_asr_decoder(const multi_asr_decoder &)             = delete;
+    multi_asr_decoder & operator=(const multi_asr_decoder &) = delete;
+
+    // Load gguf model + create llama_context. `cpu_range` pins the ggml compute
+    // threads to the decoder core set. `hidden_size` must match the encoder's
+    // audio embedding dimension (validated against model n_embd).
+    // Throws std::runtime_error on failure.
+    void init(const multi_asr_params & params, int64_t hidden_size);
+
+    // Server integration variant: reuse the already loaded text model while
+    // creating a private llama_context for the legacy ASR decoder.  The model
+    // is owned by llama-server; this object only owns the context.
+    void init_shared(llama_model * model, const multi_asr_params & params, int64_t hidden_size);
+
+    // Decode one request (prefill audio embd + generate). Fills req.text and
+    // timings.prefill_ms / decode_ms / n_out_tokens. Resets KV per request.
+    // Returns true on success; on failure sets req.error and returns false.
+    bool decode(multi_asr_request & req);
+
+    int64_t hidden_size() const { return hidden_size_; }
+
+  private:
+    common_init_result_ptr init_;  // owns model + context lifetime
+    llama_model *          model_ = nullptr;
+    llama_context *        lctx_  = nullptr;
+    bool                   owns_context_ = false;
+    const llama_vocab *    vocab_ = nullptr;
+
+    int64_t hidden_size_ = 0;
+    int32_t n_batch_     = 2048;
+
+    // Qwen3-ASR audio boundary tokens (<|audio_start|>, <|audio_end|>).
+    std::vector<llama_token> tok_audio_beg_;
+    std::vector<llama_token> tok_audio_end_;
+
+    common_params params_;  // holds sampling config for per-request samplers
+
+    // helpers (mirrors mtmd-cli-smt.cpp)
+    int decode_tokens(const std::vector<llama_token> & tokens, llama_pos & n_past, bool logits_last);
+    int decode_embd(const float * embd, int n_tokens, llama_pos & n_past, bool logits_last);
+};
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-encoder.cpp b/tools/smt-mtmd/smt/multi-asr/multi-asr-encoder.cpp
new file mode 100644
index 00000000000..84a296403e9
--- /dev/null
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-encoder.cpp
@@ -0,0 +1,89 @@
+#include "multi-asr-encoder.h"
+
+#include "ggml.h"               // ggml_time_ms
+#include "smt-audio-wrapper.h"  // tools/mtmd/smt-audio-wrapper.h
+
+#include <cstdio>
+#include <fstream>
+#include <stdexcept>
+
+#ifndef _WIN32
+#    include <unistd.h>
+#endif
+
+multi_asr_encoder::multi_asr_encoder()  = default;
+multi_asr_encoder::~multi_asr_encoder() = default;
+
+void multi_asr_encoder::init(const std::string & smt_config_dir, bool warmup) {
+    audio_ = smt_audio_context::create(smt_config_dir, warmup);
+    if (!audio_) {
+        throw std::runtime_error("multi_asr_encoder: failed to create SMT audio context");
+    }
+    hidden_size_ = audio_->hidden_size();
+    if (hidden_size_ <= 0) {
+        throw std::runtime_error("multi_asr_encoder: invalid hidden_size");
+    }
+}
+
+// smt_audio_context::encode_audio() takes a file path, so we spill the raw wav
+// bytes to a temp file (same approach as the server path).
+static std::string write_temp_wav(const std::vector<uint8_t> & data, std::string & err) {
+    char      tmpl[] = "/tmp/multi-asr-XXXXXX";
+    const int fd     = mkstemp(tmpl);
+    if (fd < 0) {
+        err = "mkstemp failed";
+        return {};
+    }
+    const std::string path = tmpl;
+    {
+        std::ofstream f(path, std::ios::binary | std::ios::trunc);
+        if (!f) {
+            close(fd);
+            std::remove(path.c_str());
+            err = "cannot open temp file";
+            return {};
+        }
+        f.write(reinterpret_cast<const char *>(data.data()), (std::streamsize) data.size());
+    }
+    close(fd);
+    return path;
+}
+
+bool multi_asr_encoder::encode(multi_asr_request & req) {
+    if (!audio_) {
+        req.error = "encoder not initialized";
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+
+    std::string       err;
+    const std::string tmp = write_temp_wav(req.audio, err);
+    if (tmp.empty()) {
+        req.error = "multi_asr_encoder: " + err;
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+
+    const int64_t t0 = ggml_time_ms();
+    try {
+        req.embd = audio_->encode_audio(tmp);
+    } catch (const std::exception & e) {
+        std::remove(tmp.c_str());
+        req.error = std::string("multi_asr_encoder: encode_audio failed: ") + e.what();
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+    std::remove(tmp.c_str());
+
+    req.timings.encode_ms = (double) (ggml_time_ms() - t0);
+
+    if (req.embd.empty() || (req.embd.size() % (size_t) hidden_size_) != 0) {
+        req.error = "multi_asr_encoder: invalid embedding shape";
+        req.stage = multi_asr_stage::failed;
+        return false;
+    }
+    req.n_audio_tokens         = (int32_t) (req.embd.size() / (size_t) hidden_size_);
+    req.timings.n_audio_tokens = req.n_audio_tokens;
+    req.stage                  = multi_asr_stage::decoding;
+    return true;
+}
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-encoder.h b/tools/smt-mtmd/smt/multi-asr/multi-asr-encoder.h
new file mode 100644
index 00000000000..7a5c3dff4b3
--- /dev/null
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-encoder.h
@@ -0,0 +1,36 @@
+#pragma once
+
+// Encoder stage: wraps a single SMT audio ONNX context (frontend+backend).
+// Runs on its own worker thread, pinned to the encoder core set.
+// Encodes audio serially (B1: one encode at a time) and produces the audio
+// embedding into the request's `embd` field.
+
+#include "multi-asr-common.h"
+
+#include <memory>
+#include <string>
+
+struct smt_audio_context;  // fwd (tools/mtmd/smt-audio-wrapper.h)
+
+class multi_asr_encoder {
+  public:
+    multi_asr_encoder();
+    ~multi_asr_encoder();
+
+    multi_asr_encoder(const multi_asr_encoder &)             = delete;
+    multi_asr_encoder & operator=(const multi_asr_encoder &) = delete;
+
+    // Load the ONNX encoder from smt_config_dir. Core binding (affinity/threads)
+    // for the ONNX EP comes from config.json. Throws std::runtime_error on failure.
+    void init(const std::string & smt_config_dir, bool warmup);
+
+    // Encode one request's audio -> fills req.embd, req.n_audio_tokens, timings.
+    // Returns true on success; on failure sets req.error and returns false.
+    bool encode(multi_asr_request & req);
+
+    int64_t hidden_size() const { return hidden_size_; }
+
+  private:
+    std::unique_ptr<smt_audio_context> audio_;
+    int64_t                            hidden_size_ = 0;
+};
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.cpp b/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.cpp
new file mode 100644
index 00000000000..06234117abb
--- /dev/null
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.cpp
@@ -0,0 +1,188 @@
+#include "multi-asr-orchestrator.h"
+
+#include "smt-audio-wrapper.h"  // full type for smt_audio_context (unique_ptr dtor)
+#include "ggml.h"
+#include "log.h"
+
+#include <stdexcept>
+#include <utility>
+
+multi_asr_orchestrator::multi_asr_orchestrator() = default;
+
+multi_asr_orchestrator::~multi_asr_orchestrator() {
+    stop();
+}
+
+void multi_asr_orchestrator::init(const multi_asr_params & params) {
+    params_ = params;
+
+    // Encoder must init first so we know hidden_size, then validate decoder.
+    LOG_INF("[multi-asr] initializing encoder (cores from config.json)...\n");
+    encoder_.init(params.smt_config_dir, params.warmup);
+    hidden_size_ = encoder_.hidden_size();
+
+    LOG_INF("[multi-asr] initializing decoder (cores='%s')...\n", params.decoder_cpu_range.c_str());
+    decoder_.init(params, hidden_size_);
+
+    LOG_INF("[multi-asr] ready. hidden_size=%lld n_parallel=%d queue_max=%d pipeline=%s\n", (long long) hidden_size_,
+            params.n_parallel, params.queue_max, params.enable_pipeline ? "on" : "off");
+}
+
+void multi_asr_orchestrator::init_shared(llama_model * model, const multi_asr_params & params) {
+    params_ = params;
+    LOG_INF("[multi-asr] initializing fully isolated legacy decoder\n");
+    encoder_.init(params.smt_config_dir, params.warmup);
+    hidden_size_ = encoder_.hidden_size();
+    decoder_.init(params, hidden_size_);
+    LOG_INF("[multi-asr] isolated legacy decoder ready. hidden_size=%lld\n", (long long) hidden_size_);
+    GGML_UNUSED(model);
+}
+
+void multi_asr_orchestrator::start() {
+    if (running_.exchange(true)) {
+        return;
+    }
+    encode_thread_ = std::thread(&multi_asr_orchestrator::encode_loop, this);
+    decode_thread_ = std::thread(&multi_asr_orchestrator::decode_loop, this);
+}
+
+void multi_asr_orchestrator::stop() {
+    if (!running_.exchange(false)) {
+        return;
+    }
+    intake_cv_.notify_all();
+    ready_cv_.notify_all();
+    if (encode_thread_.joinable()) {
+        encode_thread_.join();
+    }
+    if (decode_thread_.joinable()) {
+        decode_thread_.join();
+    }
+}
+
+bool multi_asr_orchestrator::submit_and_wait(const multi_asr_request & req_in, multi_asr_request & out) {
+    // Backpressure: reject if the system is already at its hard cap.
+    if (in_flight_.load() >= params_.queue_max) {
+        out       = req_in;
+        out.error = "server busy: queue full";
+        out.stage = multi_asr_stage::failed;
+        return false;
+    }
+
+    auto job         = std::make_shared<multi_asr_job>();
+    job->req         = req_in;
+    job->t_submit_ms = ggml_time_ms();
+
+    in_flight_.fetch_add(1);
+    {
+        std::lock_guard<std::mutex> lk(intake_mu_);
+        intake_.push_back(job);
+    }
+    intake_cv_.notify_one();
+
+    // Block until the pipeline finishes this job.
+    {
+        std::unique_lock<std::mutex> lk(job->mu);
+        job->cv.wait(lk, [&] { return job->finished; });
+    }
+
+    out = std::move(job->req);
+    in_flight_.fetch_sub(1);
+    return out.stage != multi_asr_stage::failed;
+}
+
+// Encode worker: pulls from intake, runs ONNX encode on the encoder cores,
+// pushes the encoded job to the ready queue for the decoder.
+void multi_asr_orchestrator::encode_loop() {
+    while (running_.load()) {
+        multi_asr_job_ptr job;
+        {
+            std::unique_lock<std::mutex> lk(intake_mu_);
+            intake_cv_.wait(lk, [&] { return !intake_.empty() || !running_.load(); });
+            if (!running_.load() && intake_.empty()) {
+                break;
+            }
+            job = intake_.front();
+            intake_.pop_front();
+        }
+
+        // In strict-serial (no pipeline) mode, only one job is ever in intake at
+        // a time because submit blocks; the encode->decode handoff below still
+        // serializes naturally since ready_ holds at most one item.
+
+        const int64_t t0          = ggml_time_ms();
+        job->req.timings.queue_ms = (double) (t0 - job->t_submit_ms);
+        job->req.stage            = multi_asr_stage::encoding;
+
+        const bool ok = encoder_.encode(job->req);
+        if (!ok) {
+            // finish immediately with error
+            {
+                std::lock_guard<std::mutex> lk(job->mu);
+                job->req.timings.total_ms = (double) (ggml_time_ms() - job->t_submit_ms);
+                job->finished             = true;
+            }
+            job->cv.notify_one();
+            continue;
+        }
+
+        // In the server facade, keep the legacy FIFO queue but execute the
+        // complete encode->decode transaction on this worker.  The server
+        // process owns a second GGUF context; overlapping SMT encoder work
+        // with GGUF decode corrupts backend state even though the standalone
+        // binary is safe to pipeline.  This path also avoids a cross-worker
+        // wait cycle when strict mode is selected.
+        if (!params_.enable_pipeline) {
+            decoder_.decode(job->req);
+            job->req.timings.total_ms = (double) (ggml_time_ms() - job->t_submit_ms);
+            {
+                std::lock_guard<std::mutex> lk(job->mu);
+                job->finished = true;
+            }
+            job->cv.notify_one();
+            continue;
+        }
+
+        // Hand off to decoder. Bound the ready queue to depth 1 so the encoder
+        // runs at most ONE step ahead of the decoder — the clean B1 staggered
+        // overlap E(N+1)‖D(N), not unbounded runahead.
+        {
+            std::unique_lock<std::mutex> lk(ready_mu_);
+            ready_cv_.wait(lk, [&] { return ready_.empty() || !running_.load(); });
+            if (!running_.load()) {
+                break;
+            }
+            ready_.push_back(job);
+        }
+        ready_cv_.notify_one();
+
+    }
+}
+
+// Decode worker: pulls encoded jobs, runs gguf decode on the decoder cores.
+void multi_asr_orchestrator::decode_loop() {
+    while (running_.load()) {
+        multi_asr_job_ptr job;
+        {
+            std::unique_lock<std::mutex> lk(ready_mu_);
+            ready_cv_.wait(lk, [&] { return !ready_.empty() || !running_.load(); });
+            if (!running_.load() && ready_.empty()) {
+                break;
+            }
+            job = ready_.front();
+            ready_.pop_front();
+        }
+        // Wake the encoder: the ready slot is now free, so E(N+1) can hand off
+        // while we decode this job (the staggered overlap).
+        ready_cv_.notify_one();
+
+        decoder_.decode(job->req);
+        job->req.timings.total_ms = (double) (ggml_time_ms() - job->t_submit_ms);
+
+        {
+            std::lock_guard<std::mutex> lk(job->mu);
+            job->finished = true;
+        }
+        job->cv.notify_one();
+    }
+}
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.h b/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.h
new file mode 100644
index 00000000000..e9dd23d2965
--- /dev/null
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.h
@@ -0,0 +1,86 @@
+#pragma once
+
+// Orchestrator: implements scheme B1 "错峰流水" (staggered pipeline).
+//
+//   intake queue ──► [encode thread](encoder cores) ──► ready queue ──► [decode thread](decoder cores)
+//
+// Two dedicated worker threads run on disjoint core sets, so E(N+1) overlaps
+// D(N). Each stage is serial (one at a time), matching the B1 semantics: at any
+// instant at most one encode + one decode in flight.
+//
+// Concurrency control:
+//   - submit() blocks-or-rejects when total in-flight+waiting >= queue_max.
+//   - `n_parallel` bounds how many requests may be "in the pipeline" before new
+//     submissions wait (the FIFO ">N waits" requirement).
+//   - enable_pipeline=false => strict serial baseline (decode finishes before
+//     next encode starts), for benchmarking variable ②.
+
+#include "multi-asr-common.h"
+#include "multi-asr-decoder.h"
+#include "multi-asr-encoder.h"
+
+#include <atomic>
+#include <condition_variable>
+#include <cstdint>
+#include <deque>
+#include <memory>
+#include <mutex>
+#include <thread>
+
+// A submitted job: shared between the HTTP handler (waiter) and the pipeline.
+struct multi_asr_job {
+    multi_asr_request       req;
+    std::mutex              mu;
+    std::condition_variable cv;
+    bool                    finished    = false;
+    int64_t                 t_submit_ms = 0;  // ggml_time_ms at submit
+};
+
+using multi_asr_job_ptr = std::shared_ptr<multi_asr_job>;
+
+class multi_asr_orchestrator {
+  public:
+    multi_asr_orchestrator();
+    ~multi_asr_orchestrator();
+
+    // Initialize encoder + decoder (loads models, pins cores). Throws on failure.
+    void init(const multi_asr_params & params);
+    void init_shared(llama_model * model, const multi_asr_params & params);
+
+    // Start the encode/decode worker threads.
+    void start();
+
+    // Stop workers and drain. Safe to call once.
+    void stop();
+
+    // Submit a request and BLOCK until it finishes (encode+decode) or fails.
+    // Returns false if rejected due to a full queue (sets job->req.error).
+    bool submit_and_wait(const multi_asr_request & req_in, multi_asr_request & out);
+
+    int64_t hidden_size() const { return hidden_size_; }
+
+  private:
+    multi_asr_params  params_;
+    multi_asr_encoder encoder_;
+    multi_asr_decoder decoder_;
+    int64_t           hidden_size_ = 0;
+
+    // intake -> encode
+    std::deque<multi_asr_job_ptr> intake_;
+    std::mutex                    intake_mu_;
+    std::condition_variable       intake_cv_;
+
+    // encode -> decode (bounded to 1 for B1: only one encoded result waiting)
+    std::deque<multi_asr_job_ptr> ready_;
+    std::mutex                    ready_mu_;
+    std::condition_variable       ready_cv_;
+
+    std::atomic<int32_t> in_flight_{ 0 };  // total accepted, not yet finished
+    std::atomic<bool>    running_{ false };
+
+    std::thread encode_thread_;
+    std::thread decode_thread_;
+
+    void encode_loop();
+    void decode_loop();
+};
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-service.cpp b/tools/smt-mtmd/smt/multi-asr/multi-asr-service.cpp
new file mode 100644
index 00000000000..8178e84f3af
--- /dev/null
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-service.cpp
@@ -0,0 +1,81 @@
+#include "multi-asr-service.h"
+
+#include "common.h"
+
+#include <algorithm>
+#include <cstring>
+
+smt_multi_asr_service::~smt_multi_asr_service() {
+    if (orchestrator_) {
+        orchestrator_->stop();
+    }
+}
+
+void smt_multi_asr_service::init(llama_model * model, const common_params & server_params) {
+    multi_asr_params p;
+    p.model_path = server_params.model.path;
+    p.smt_config_dir = server_params.smt_config_dir;
+    p.warmup = server_params.warmup;
+    p.n_parallel = std::max(1, server_params.n_parallel);
+    p.queue_max = std::max(1, server_params.n_parallel * 8);
+    p.n_predict = server_params.n_predict > 0 ? server_params.n_predict : 256;
+    // Keep the legacy decoder's model/context settings independent from the
+    // HTTP server.  In particular, inheriting the server's reduced n_ctx or
+    // its HTTP thread count changes the standalone ASR graph and can produce
+    // invalid output on SpacemiT's backend.  The old multi-ASR binary uses
+    // model-default context and backend-default decoder threading.
+    p.n_ctx = 0;
+    p.n_batch = 2048;
+    p.decoder_n_threads = 0;
+    // Preserve the old FIFO intake semantics, but serialize each complete
+    // encode->decode transaction inside llama-server.  The standalone binary
+    // can overlap E/D; llama-server owns a second GGUF context in the same
+    // process, where that overlap corrupts backend state.
+    p.enable_pipeline = false;
+
+    orchestrator_ = std::make_unique<multi_asr_orchestrator>();
+    orchestrator_->init_shared(model, p);
+    orchestrator_->start();
+}
+
+bool smt_multi_asr_service::submit(const std::vector<uint8_t> & audio, const std::string & prompt,
+                                   int32_t n_predict, multi_asr_request & result) {
+    if (!orchestrator_) {
+        result.error = "SMT multi-ASR service is not initialized";
+        result.stage = multi_asr_stage::failed;
+        return false;
+    }
+    multi_asr_request req;
+    req.audio = audio;
+    req.prompt = prompt.empty() ? "language Chinese<asr_text>" : prompt;
+    const std::string assistant = "<|im_start|>assistant\n";
+    if (const size_t pos = req.prompt.rfind(assistant); pos != std::string::npos) {
+        const std::string tail = req.prompt.substr(pos + assistant.size());
+        if (!tail.empty()) {
+            req.prompt = tail;
+        }
+    }
+    // Chat-template rendering may place the user instruction before the
+    // assistant generation marker.  Recover the legacy Qwen3-ASR instruction
+    // instead of passing the whole rendered conversation to the decoder.
+    // The server's `prompt` field is usually a fully rendered chat template,
+    // and some templates stringify a missing language as `language None`.
+    // The legacy decoder expects only the short Qwen3-ASR instruction.  Keep a
+    // valid explicit language when it is present; otherwise use the proven
+    // Chinese default.  Never pass a rendered template or a `None` language
+    // through to the decoder.
+    const size_t lang = req.prompt.find("language ");
+    const size_t marker = lang == std::string::npos ? std::string::npos : req.prompt.find("<asr_text>", lang);
+    if (lang != std::string::npos && marker != std::string::npos) {
+        const std::string candidate = req.prompt.substr(lang, marker + std::strlen("<asr_text>") - lang);
+        if (candidate.find("None") == std::string::npos && candidate.find("null") == std::string::npos) {
+            req.prompt = candidate;
+        } else {
+            req.prompt = "language Chinese<asr_text>";
+        }
+    } else {
+        req.prompt = "language Chinese<asr_text>";
+    }
+    req.n_predict = n_predict > 0 ? n_predict : 256;
+    return orchestrator_->submit_and_wait(req, result);
+}
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-service.h b/tools/smt-mtmd/smt/multi-asr/multi-asr-service.h
new file mode 100644
index 00000000000..e0eff7ee6e1
--- /dev/null
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-service.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#include "multi-asr-common.h"
+#include "multi-asr-orchestrator.h"
+
+#include <memory>
+
+// Server-facing facade for the legacy, correctness-proven ASR pipeline.
+// The facade deliberately does not expose llama-server slots or media chunks:
+// requests are admitted FIFO and decoded by the private legacy context.
+class smt_multi_asr_service {
+  public:
+    smt_multi_asr_service() = default;
+    ~smt_multi_asr_service();
+
+    smt_multi_asr_service(const smt_multi_asr_service &) = delete;
+    smt_multi_asr_service & operator=(const smt_multi_asr_service &) = delete;
+
+    void init(llama_model * model, const common_params & server_params);
+    bool submit(const std::vector<uint8_t> & audio, const std::string & prompt,
+                int32_t n_predict, multi_asr_request & result);
+
+  private:
+    std::unique_ptr<multi_asr_orchestrator> orchestrator_;
+};

From ffac955b610c418dfde9dd9eea31b0c33855a3cf Mon Sep 17 00:00:00 2001
From: co-seven <linxi.cai@spacemit.com>
Date: Fri, 21 Aug 2026 09:48:21 +0000
Subject: [PATCH 2/2] fix(server): guard SMT ASR fields for non-SMT builds

---
 tools/server/server-context.cpp | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 69f74f30444..b47a87922c2 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -1560,6 +1560,11 @@ struct server_context_impl {
             // IMPORTANT: chat_params is reused across sleeping / resuming states,
             //            never store llama_context/llama_model pointers in chat_params,
             //            as they may be invalidated after sleeping
+            bool allow_audio = media ? media->supports_audio() : false;
+#if defined(LLAMA_SERVER_SMT_MTMD)
+            allow_audio = allow_audio || smt_asr_service != nullptr;
+#endif
+
             chat_params = {
                 /* use_jinja             */ params_base.use_jinja,
                 /* prefill_assistant     */ params_base.prefill_assistant,
@@ -1567,7 +1572,7 @@ struct server_context_impl {
                 /* chat_template_kwargs  */ params_base.default_template_kwargs,
                 /* tmpls                 */ std::move(chat_templates),
                 /* allow_image           */ media ? media->supports_vision() : false,
-                /* allow_audio           */ (media ? media->supports_audio() : false) || smt_asr_service != nullptr,
+                /* allow_audio           */ allow_audio,
                 /* allow_video           */ media ? media->supports_video()  : false,
                 /* enable_thinking       */ enable_thinking,
                 /* reasoning_budget      */ params_base.sampling.reasoning_budget_tokens,
@@ -4057,14 +4062,18 @@ server_context_meta server_context::get_meta() const {
 
     const char * ftype_name = llama_ftype_name(llama_model_ftype(impl->model_tgt));
 
+    bool has_mtmd = impl->media != nullptr && impl->media->supports_prompt_embeddings();
+#if defined(LLAMA_SERVER_SMT_MTMD)
+    has_mtmd = has_mtmd || impl->smt_asr_service != nullptr;
+#endif
+
     return server_context_meta {
         /* build_info             */ std::string(llama_build_info()),
         /* model_name             */ impl->model_name,
         /* model_aliases          */ impl->model_aliases,
         /* model_tags             */ impl->model_tags,
         /* model_path             */ impl->params_base.model.path,
-        /* has_mtmd               */ (impl->media != nullptr && impl->media->supports_prompt_embeddings()) ||
-                                      impl->smt_asr_service != nullptr,
+        /* has_mtmd               */ has_mtmd,
         /* has_inp_image          */ impl->chat_params.allow_image,
         /* has_inp_audio          */ impl->chat_params.allow_audio,
         /* has_inp_video          */ impl->chat_params.allow_video,