From b371ad5b9f15085364bd1352df7c3850c84d6773 Mon Sep 17 00:00:00 2001
From: co-seven <linxi.cai@spacemit.com>
Date: Wed, 26 Aug 2026 06:40:15 +0000
Subject: [PATCH 1/2] feat(asr): optimize continuous batching pipeline

Implement dynamic mixed embedding-prefill and token-decode batching for multi-ASR, with FIFO admission and bounded decoder slots. Keep audio encoding serialized while allowing encoder and decoder execution to overlap, and preserve per-sequence logits and sampling across batched steps.
---
 common/arg.cpp                                |   2 +-
 common/common.h                               |   2 +-
 tools/server/server-context.cpp               | 107 ++--
 tools/smt-mtmd/CMakeLists.txt                 |   1 +
 .../smt-mtmd/smt/multi-asr/multi-asr-common.h |   3 +-
 .../smt/multi-asr/multi-asr-decoder.cpp       | 469 ++++++++++--------
 .../smt/multi-asr/multi-asr-decoder.h         |  88 ++--
 .../smt/multi-asr/multi-asr-orchestrator.cpp  | 213 ++++----
 .../smt/multi-asr/multi-asr-orchestrator.h    |  66 +--
 .../smt/multi-asr/multi-asr-service.cpp       |  38 +-
 .../smt/multi-asr/multi-asr-service.h         |   5 +-
 11 files changed, 518 insertions(+), 476 deletions(-)

diff --git a/common/arg.cpp b/common/arg.cpp
index 18eb4c05a0c..5de5e4d2344 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -2383,7 +2383,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_MTMD}).set_env("LLAMA_ARG_SMT_CONFIG_DIR"));
     add_opt(common_arg(
         {"--smt-multi-asr"},
-        "enable the isolated legacy multi-ASR FIFO service (default: disabled)",
+        "enable Qwen3-ASR prompt compatibility on the native SMT media path (default: disabled)",
         [](common_params & params) {
             params.smt_multi_asr = true;
         }
diff --git a/common/common.h b/common/common.h
index d8ede71d9b2..5c49b39e9a7 100644
--- a/common/common.h
+++ b/common/common.h
@@ -598,7 +598,7 @@ struct common_params {
 #if defined(LLAMA_SERVER_SMT_MTMD)
     std::string media_backend = "auto"; // multimodal backend: auto|mtmd|smt
     std::string smt_config_dir;         // SMT config dir (config.json + ONNX)
-    bool smt_multi_asr = false;         // opt in to the isolated legacy multi-ASR FIFO service
+    bool smt_multi_asr = false;         // use the legacy Qwen3-ASR prompt form on SMT media
 #endif
     std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
     int image_min_tokens = -1;
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index b47a87922c2..4c22ad56d4c 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -979,8 +979,6 @@ struct server_context_impl {
 
     void destroy() {
 #if defined(LLAMA_SERVER_SMT_MTMD)
-        // The isolated ASR context borrows model_tgt; tear it down before the
-        // owning common_init_result releases the model.
         smt_asr_service.reset();
 #endif
         spec.reset();
@@ -1072,24 +1070,6 @@ struct server_context_impl {
                                         COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end();
         const bool has_spec = has_draft || spec_mtp;
 
-#if defined(LLAMA_SERVER_SMT_MTMD)
-        // Initialize the legacy ASR stack before loading the server's own
-        // llama model.  SpacemiT's backend keeps process-wide state, and the
-        // proven standalone implementation is sensitive to a second model
-        // being loaded ahead of it.
-        if (use_multi_asr && !smt_asr_service) {
-            try {
-                smt_asr_service = std::make_unique<smt_multi_asr_service>();
-                smt_asr_service->init(nullptr, params_base);
-                SRV_INF("%s", "initialized isolated SMT multi-ASR service\n");
-            } catch (const std::exception & e) {
-                SRV_ERR("failed to initialize isolated SMT multi-ASR service: %s\n", e.what());
-                smt_asr_service.reset();
-                return false;
-            }
-        }
-#endif
-
         if (callback_state) {
             std::vector<std::string> stages = {"text_model"};
             if (has_spec) {
@@ -1109,8 +1089,8 @@ struct server_context_impl {
         }
 
         std::unique_ptr<media_worker> media_worker_init;
-        // Default SMT audio keeps the original generic media path. Only the
-        // explicit multi-ASR mode uses the isolated legacy service.
+        // SMT multi-ASR uses the normal server media path. The media worker
+        // serializes encoder calls; server slots continuous-batch decoding.
         if (has_mmproj || (has_smt_media && !use_multi_asr)) {
             std::string worker_backend = "mtmd";
 #if defined(LLAMA_SERVER_SMT_MTMD)
@@ -1232,7 +1212,9 @@ struct server_context_impl {
             params_base.load_progress_callback_user_data = &load_progress_text;
         }
 
-        llama_init = common_init_from_params(params_base);
+        // In dedicated multi-ASR mode llama-server owns only the model. The
+        // sole llama_context is created and scheduled by multi-ASR below.
+        llama_init = common_init_from_params(params_base, use_multi_asr);
 
         model_tgt = llama_init->model();
         ctx_tgt   = llama_init->context();
@@ -1244,10 +1226,32 @@ struct server_context_impl {
 
         vocab = llama_model_get_vocab(model_tgt);
 
-        n_ctx = llama_n_ctx(ctx_tgt);
-
         add_bos_token = llama_vocab_get_add_bos(vocab);
 
+#if defined(LLAMA_SERVER_SMT_MTMD)
+        if (use_multi_asr) {
+            try {
+                smt_asr_service = std::make_unique<smt_multi_asr_service>();
+                smt_asr_service->init(model_tgt, params_base);
+                ctx_tgt = smt_asr_service->context();
+                if (ctx_tgt == nullptr) {
+                    throw std::runtime_error("multi-ASR scheduler did not create a decoder context");
+                }
+                SRV_INF("%s", "initialized internal SMT multi-ASR scheduler (serial encoder + continuous-batch decoder)\n");
+            } catch (const std::exception & e) {
+                SRV_ERR("failed to initialize SMT multi-ASR scheduler: %s\n", e.what());
+                smt_asr_service.reset();
+                return false;
+            }
+        }
+#endif
+
+        if (ctx_tgt == nullptr) {
+            SRV_ERR("%s", "failed to create llama context\n");
+            return false;
+        }
+        n_ctx = llama_n_ctx(ctx_tgt);
+
         if (has_spec) {
             // spec_mtp doesn't use load a model internally, so we report 0.0 and 1.0 manually
             load_progress_callback(0.0f, &load_progress_spec);
@@ -4066,7 +4070,6 @@ server_context_meta server_context::get_meta() const {
 #if defined(LLAMA_SERVER_SMT_MTMD)
     has_mtmd = has_mtmd || impl->smt_asr_service != nullptr;
 #endif
-
     return server_context_meta {
         /* build_info             */ std::string(llama_build_info()),
         /* model_name             */ impl->model_name,
@@ -4159,31 +4162,25 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
     int32_t sse_ping_interval = params.sse_ping_interval;
 
     try {
-        // Audio-only SMT requests use the isolated legacy ASR orchestrator.
-        // This synchronous facade is intentionally ahead of task creation:
-        // no server slot, PEG parser, prompt cache, or shared KV sequence is
-        // touched by these requests.
 #if defined(LLAMA_SERVER_SMT_MTMD)
-        const std::string raw_prompt = data.contains("prompt") && data.at("prompt").is_string()
-                ? data.at("prompt").get<std::string>() : std::string();
-        // oaicompat_chat_params_parse() has already decoded input_audio into
-        // `files`; chat-template rendering may remove the media marker from
-        // the resulting prompt, so do not use the rendered text to decide
-        // whether this is an audio request.
-        const bool has_audio_media = !files.empty();
-        if (ctx_server.smt_asr_service && has_audio_media) {
-            multi_asr_request asr_result;
-            const int32_t n_predict = json_value(data, "max_tokens", 256);
-            const std::string user_prompt = raw_prompt;
-            const bool ok = ctx_server.smt_asr_service->submit(files.front(), user_prompt, n_predict, asr_result);
-            if (!ok) {
-                res->error(format_error_response(asr_result.error.empty() ? "ASR request failed" : asr_result.error,
+        // Audio-only Qwen3-ASR requests are owned by the ASR scheduler. The
+        // generic server slot path is intentionally bypassed: it knows how to
+        // batch text tokens, but must not interleave audio embedding prefill
+        // with another request's decoder batch.
+        if (ctx_server.smt_asr_service != nullptr && !files.empty()) {
+            const std::string prompt_text = data.contains("prompt") && data.at("prompt").is_string()
+                    ? data.at("prompt").get<std::string>() : std::string();
+            multi_asr_request result;
+            const int32_t n_predict = json_value(data, "max_tokens", params.n_predict > 0 ? params.n_predict : 256);
+            result.temperature = json_value(data, "temperature", 0.0f);
+            if (!ctx_server.smt_asr_service->submit(files.front(), prompt_text, n_predict, result)) {
+                res->error(format_error_response(result.error.empty() ? "ASR request failed" : result.error,
                                                  ERROR_TYPE_SERVER));
                 return res;
             }
             const json choice = {
                 {"index", 0},
-                {"message", {{"role", "assistant"}, {"content", asr_result.text}}},
+                {"message", {{"role", "assistant"}, {"content", result.text}}},
                 {"finish_reason", "stop"},
             };
             res->ok({
@@ -4192,17 +4189,17 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
                 {"created", std::time(nullptr)},
                 {"model", meta->model_name},
                 {"choices", json::array({choice})},
-                {"usage", {{"prompt_tokens", asr_result.timings.n_audio_tokens},
-                            {"completion_tokens", asr_result.timings.n_out_tokens},
-                            {"total_tokens", asr_result.timings.n_audio_tokens + asr_result.timings.n_out_tokens}}},
+                {"usage", {{"prompt_tokens", result.timings.n_audio_tokens},
+                            {"completion_tokens", result.timings.n_out_tokens},
+                            {"total_tokens", result.timings.n_audio_tokens + result.timings.n_out_tokens}}},
                 {"multi_asr_timings", {
-                    {"queue_ms", asr_result.timings.queue_ms},
-                    {"encode_ms", asr_result.timings.encode_ms},
-                    {"prefill_ms", asr_result.timings.prefill_ms},
-                    {"decode_ms", asr_result.timings.decode_ms},
-                    {"total_ms", asr_result.timings.total_ms},
-                    {"n_audio_tokens", asr_result.timings.n_audio_tokens},
-                    {"n_out_tokens", asr_result.timings.n_out_tokens},
+                    {"queue_ms", result.timings.queue_ms},
+                    {"encode_ms", result.timings.encode_ms},
+                    {"prefill_ms", result.timings.prefill_ms},
+                    {"decode_ms", result.timings.decode_ms},
+                    {"total_ms", result.timings.total_ms},
+                    {"n_audio_tokens", result.timings.n_audio_tokens},
+                    {"n_out_tokens", result.timings.n_out_tokens},
                 }},
             });
             return res;
diff --git a/tools/smt-mtmd/CMakeLists.txt b/tools/smt-mtmd/CMakeLists.txt
index 4cb795a1a5f..6bc5fed0436 100644
--- a/tools/smt-mtmd/CMakeLists.txt
+++ b/tools/smt-mtmd/CMakeLists.txt
@@ -78,6 +78,7 @@ if(LLAMA_SERVER_SMT_MTMD)
     )
     target_include_directories(${TARGET} PRIVATE
         ${CMAKE_SOURCE_DIR}/common
+        ${CMAKE_SOURCE_DIR}/src
         ${CMAKE_SOURCE_DIR}/tools/mtmd
         ${CMAKE_SOURCE_DIR}/vendor
         ${CMAKE_SOURCE_DIR}/ggml/src
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-common.h b/tools/smt-mtmd/smt/multi-asr/multi-asr-common.h
index 50ea6d21eb3..423c1b6419e 100644
--- a/tools/smt-mtmd/smt/multi-asr/multi-asr-common.h
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-common.h
@@ -48,8 +48,6 @@ struct multi_asr_params {
     int32_t n_ctx     = 0;    // 0 = from model / config.json context_size
     int32_t n_batch   = 2048;
 
-    // pipeline
-    bool enable_pipeline = true;  // false = strict serial baseline (variable ②)
 };
 
 // ---------------------------------------------------------------------------
@@ -81,6 +79,7 @@ struct multi_asr_request {
     std::vector<uint8_t> audio;   // raw wav bytes (decoded input)
     std::string          prompt;  // text prompt, e.g. "language Chinese<asr_text>"
     int32_t              n_predict = 256;
+    float                temperature = 0.0f;
 
     // filled by encoder stage: audio embedding (n_audio_tokens * hidden_size floats)
     std::vector<float> embd;
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.cpp b/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.cpp
index f685156428e..5543ae63cc3 100644
--- a/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.cpp
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.cpp
@@ -1,185 +1,119 @@
 #include "multi-asr-decoder.h"
 
 #include "ggml.h"
-#include "sampling.h"
+#include "llama-model.h"
+#include "log.h"
 
 #include <algorithm>
 #include <cstring>
+#include <numeric>
 #include <stdexcept>
-#include <vector>
 
-// Build the Qwen3-ASR prompt around the audio, mirroring
-// format_qwen3asr_audio_prompt() in tools/mtmd/mtmd-cli-smt.cpp:
-//   <|im_start|>system\n<|im_end|>\n<|im_start|>user\n <AUDIO> <|im_end|>\n<|im_start|>assistant\n<user text>
-// The audio embedding (+ <|audio_start|>/<|audio_end|>) is spliced where <AUDIO> is.
-// We return the text BEFORE the audio and the text AFTER the audio separately.
+namespace {
 struct qwen3asr_prompt_split {
-    std::string before;  // up to and including "<|im_start|>user\n"
-    std::string after;   // "<|im_end|>\n<|im_start|>assistant\n" + user text
+    std::string before;
+    std::string after;
 };
 
-static qwen3asr_prompt_split build_qwen3asr_prompt(const std::string & user_text) {
-    qwen3asr_prompt_split s;
-    s.before = "<|im_start|>system\n<|im_end|>\n<|im_start|>user\n";
-    s.after  = "<|im_end|>\n<|im_start|>assistant\n" + user_text;
-    return s;
+qwen3asr_prompt_split build_qwen3asr_prompt(const std::string & instruction) {
+    return {
+        "<|im_start|>system\n<|im_end|>\n<|im_start|>user\n",
+        "<|im_end|>\n<|im_start|>assistant\n" + instruction,
+    };
+}
 }
 
 multi_asr_decoder::~multi_asr_decoder() {
-    if (owns_context_ && lctx_ && !init_) {
+    for (auto & state : slots_) {
+        state.sampler.reset();
+    }
+    if (lctx_ != nullptr) {
         llama_free(lctx_);
     }
-    lctx_  = nullptr;
+    lctx_ = nullptr;
     model_ = nullptr;
 }
 
-void multi_asr_decoder::init(const multi_asr_params & params, int64_t hidden_size) {
-    hidden_size_ = hidden_size;
-    n_batch_     = params.n_batch;
-
-    // Build common_params for model load + threading/affinity + sampling.
-    params_.model.path = params.model_path;
-    params_.n_ctx      = params.n_ctx;  // 0 -> from model
-    params_.n_batch    = params.n_batch;
-    params_.warmup     = params.warmup;
-
-    // Decoder thread count. Core binding for the ggml IME backend is set via the
-    // SPACEMIT_PERFER_CORE_ID environment variable (see run_server.sh), not here.
-    if (params.decoder_n_threads > 0) {
-        params_.cpuparams.n_threads = params.decoder_n_threads;
-    }
-
-    init_ = common_init_from_params(params_);
-    if (!init_) {
-        throw std::runtime_error("multi_asr_decoder: common_init_from_params returned null");
-    }
-    model_ = init_->model();
-    lctx_  = init_->context();
-    if (!model_ || !lctx_) {
-        throw std::runtime_error("multi_asr_decoder: failed to load model / create context");
-    }
-    vocab_ = llama_model_get_vocab(model_);
-
-    // Validate embedding dim.
-    const int model_n_embd = llama_model_n_embd(model_);
-    if ((int64_t) model_n_embd != hidden_size_) {
-        throw std::runtime_error("multi_asr_decoder: model n_embd (" + std::to_string(model_n_embd) +
-                                 ") != encoder hidden_size (" + std::to_string(hidden_size_) + ")");
-    }
-
-    // Qwen3-ASR audio boundary special tokens.
-    tok_audio_beg_ = common_tokenize(vocab_, "<|audio_start|>", /*add_special*/ false, /*parse_special*/ true);
-    tok_audio_end_ = common_tokenize(vocab_, "<|audio_end|>", /*add_special*/ false, /*parse_special*/ true);
-    owns_context_ = true;
-}
-
 void multi_asr_decoder::init_shared(llama_model * model, const multi_asr_params & params, int64_t hidden_size) {
     if (model == nullptr) {
-        throw std::runtime_error("multi_asr_decoder: null shared model");
+        throw std::runtime_error("multi_asr_decoder: null model");
     }
+    model_ = model;
     hidden_size_ = hidden_size;
-    n_batch_ = params.n_batch;
+    n_batch_ = std::max(1, params.n_batch);
+    n_parallel_ = std::max(1, params.n_parallel);
+
+    params_ = {};
     params_.model.path = params.model_path;
     params_.n_ctx = params.n_ctx;
-    params_.n_batch = params.n_batch;
-    params_.n_ubatch = std::min(params.n_batch, 512);
+    params_.n_batch = n_batch_;
+    params_.n_ubatch = std::min(n_batch_, 512);
     params_.warmup = false;
+    params_.n_parallel = n_parallel_;
+    params_.n_outputs_max = n_parallel_;
+    params_.sampling.temp = 0.0f;
     if (params.decoder_n_threads > 0) {
         params_.cpuparams.n_threads = params.decoder_n_threads;
         params_.cpuparams_batch.n_threads = params.decoder_n_threads;
     }
+
     auto cparams = common_context_params_to_llama(params_);
-    cparams.n_seq_max = 1;
-    cparams.n_outputs_max = 1;
-    lctx_ = llama_init_from_model(model, cparams);
-    if (!lctx_) {
-        throw std::runtime_error("multi_asr_decoder: failed to create private context from shared model");
+    cparams.n_seq_max = n_parallel_;
+    cparams.n_outputs_max = n_parallel_;
+    cparams.n_batch = n_batch_;
+    cparams.n_ubatch = std::min(n_batch_, 512);
+    lctx_ = llama_init_from_model(model_, cparams);
+    if (lctx_ == nullptr) {
+        throw std::runtime_error("multi_asr_decoder: failed to create context");
     }
-    model_ = model;
+
     vocab_ = llama_model_get_vocab(model_);
     if (llama_model_n_embd(model_) != hidden_size_) {
-        llama_free(lctx_);
-        lctx_ = nullptr;
-        throw std::runtime_error("multi_asr_decoder: shared model embedding size mismatch");
+        throw std::runtime_error("multi_asr_decoder: model/encoder embedding size mismatch");
     }
+
     tok_audio_beg_ = common_tokenize(vocab_, "<|audio_start|>", false, true);
     tok_audio_end_ = common_tokenize(vocab_, "<|audio_end|>", false, true);
-    owns_context_ = true;
+    load_token_embeddings();
+    slots_.resize(n_parallel_);
+    llama_memory_clear(llama_get_memory(lctx_), true);
 }
 
-int multi_asr_decoder::decode_tokens(const std::vector<llama_token> & tokens, llama_pos & n_past, bool logits_last) {
-    if (tokens.empty()) {
-        return 0;
+void multi_asr_decoder::load_token_embeddings() {
+    const ggml_tensor * tok_embd = model_->tok_embd;
+    if (tok_embd == nullptr || tok_embd->ne[0] != hidden_size_ || tok_embd->ne[1] != llama_vocab_n_tokens(vocab_)) {
+        throw std::runtime_error("multi_asr_decoder: invalid token embedding tensor");
     }
-    llama_batch batch = llama_batch_init(n_batch_, 0, 1);
-    size_t      i     = 0;
-    while (i < tokens.size()) {
-        batch.n_tokens = 0;
-        for (; i < tokens.size() && batch.n_tokens < n_batch_; ++i) {
-            const int32_t j    = batch.n_tokens;
-            batch.token[j]     = tokens[i];
-            batch.pos[j]       = n_past + j;
-            batch.n_seq_id[j]  = 1;
-            batch.seq_id[j][0] = 0;
-            batch.logits[j]    = false;
-            batch.n_tokens++;
-        }
-        if (logits_last && i == tokens.size()) {
-            batch.logits[batch.n_tokens - 1] = true;
-        }
-        if (llama_decode(lctx_, batch) != 0) {
-            llama_batch_free(batch);
-            return 1;
+
+    const size_t nbytes = ggml_nbytes(tok_embd);
+    std::vector<uint8_t> raw(nbytes);
+    ggml_backend_tensor_get(tok_embd, raw.data(), 0, nbytes);
+
+    token_embd_.resize((size_t) tok_embd->ne[0] * (size_t) tok_embd->ne[1]);
+    const ggml_type_traits * traits = ggml_get_type_traits(tok_embd->type);
+    if (tok_embd->type == GGML_TYPE_F32) {
+        std::memcpy(token_embd_.data(), raw.data(), token_embd_.size() * sizeof(float));
+    } else if (traits->to_float != nullptr) {
+        for (int64_t i = 0; i < tok_embd->ne[1]; ++i) {
+            traits->to_float(raw.data() + (size_t) i * tok_embd->nb[1],
+                             token_embd_.data() + (size_t) i * (size_t) hidden_size_, hidden_size_);
         }
-        n_past += batch.n_tokens;
+    } else {
+        throw std::runtime_error(std::string("multi_asr_decoder: cannot dequantize token embedding type ") +
+                                 ggml_type_name(tok_embd->type));
     }
-    llama_batch_free(batch);
-    return 0;
 }
 
-int multi_asr_decoder::decode_embd(const float * embd, int n_tokens, llama_pos & n_past, bool logits_last) {
-    const int                   n_embd = (int) hidden_size_;
-    std::vector<llama_pos>      pos((size_t) n_tokens);
-    std::vector<int32_t>        n_seq_id((size_t) n_tokens);
-    std::vector<llama_seq_id>   seq_id_store((size_t) n_tokens);
-    std::vector<llama_seq_id *> seq_ids((size_t) n_tokens + 1);
-    std::vector<int8_t>         logits((size_t) n_tokens);
-    for (int i = 0; i < n_tokens; ++i) {
-        seq_id_store[i] = 0;
-        seq_ids[i]      = &seq_id_store[i];
+const float * multi_asr_decoder::token_embedding(llama_token token) const {
+    if (token < 0 || token >= llama_vocab_n_tokens(vocab_)) {
+        throw std::runtime_error("multi_asr_decoder: invalid token embedding id");
     }
-    seq_ids[n_tokens] = nullptr;
-
-    int processed = 0;
-    while (processed < n_tokens) {
-        const int  batch_size    = std::min(n_batch_, n_tokens - processed);
-        const bool is_last_batch = (processed + batch_size >= n_tokens);
-        for (int i = 0; i < batch_size; ++i) {
-            pos[processed + i]      = n_past + processed + i;
-            n_seq_id[processed + i] = 1;
-            logits[processed + i]   = (logits_last && is_last_batch && i == batch_size - 1) ? 1 : 0;
-        }
-        llama_batch batch = {
-            /*n_tokens =*/batch_size,
-            /*token    =*/nullptr,
-            /*embd     =*/const_cast<float *>(embd) + (size_t) processed * n_embd,
-            /*pos      =*/pos.data() + processed,
-            /*n_seq_id =*/n_seq_id.data() + processed,
-            /*seq_id   =*/seq_ids.data() + processed,
-            /*logits   =*/logits.data() + processed,
-        };
-        if (llama_decode(lctx_, batch) != 0) {
-            return 1;
-        }
-        processed += batch_size;
-    }
-    n_past += n_tokens;
-    return 0;
+    return token_embd_.data() + (size_t) token * (size_t) hidden_size_;
 }
 
-bool multi_asr_decoder::decode(multi_asr_request & req) {
-    if (!lctx_) {
-        req.error = "decoder not initialized";
+bool multi_asr_decoder::start(multi_asr_request & req, int slot_id) {
+    if (lctx_ == nullptr || slot_id < 0 || slot_id >= (int) slots_.size() || slots_[slot_id].active) {
+        req.error = "decoder slot unavailable";
         req.stage = multi_asr_stage::failed;
         return false;
     }
@@ -189,100 +123,223 @@ bool multi_asr_decoder::decode(multi_asr_request & req) {
         return false;
     }
 
-    // Fresh KV for this request (single slot, one request at a time).
-    llama_memory_clear(llama_get_memory(lctx_), true);
-
-    llama_pos                   n_past = 0;
-    const qwen3asr_prompt_split split  = build_qwen3asr_prompt(req.prompt);
+    auto & state = slots_[slot_id];
+    llama_memory_seq_rm(llama_get_memory(lctx_), slot_id, -1, -1);
+    state = {};
+    state.req = &req;
+    state.n_predict = req.n_predict > 0 ? req.n_predict : 256;
+    state.occupied = true;
+    state.t_prefill_start = ggml_time_ms();
+    state.admission_order = next_admission_order_++;
 
-    const int64_t t_prefill0 = ggml_time_ms();
-
-    // 1) text before audio (with BOS/special parsing)
-    const std::vector<llama_token> toks_before =
-        common_tokenize(vocab_, split.before, /*add_special*/ true, /*parse_special*/ true);
-    if (decode_tokens(toks_before, n_past, /*logits_last*/ false) != 0) {
-        req.error = "decoder: failed to decode prompt prefix";
+    const auto split = build_qwen3asr_prompt(req.prompt);
+    auto append_tokens = [&](const std::vector<llama_token> & tokens) {
+        for (const llama_token token : tokens) {
+            state.prefill.push_back({ token_embedding(token) });
+        }
+    };
+    append_tokens(common_tokenize(vocab_, split.before, true, true));
+    append_tokens(tok_audio_beg_);
+    for (int32_t i = 0; i < req.n_audio_tokens; ++i) {
+        state.prefill.push_back({ req.embd.data() + (size_t) i * (size_t) hidden_size_ });
+    }
+    append_tokens(tok_audio_end_);
+    append_tokens(common_tokenize(vocab_, split.after, false, true));
+    auto sampling = params_.sampling;
+    sampling.temp = req.temperature;
+    state.sampler.reset(common_sampler_init(model_, sampling));
+    if (!state.sampler) {
+        req.error = "decoder: sampler initialization failed";
         req.stage = multi_asr_stage::failed;
+        llama_memory_seq_rm(llama_get_memory(lctx_), slot_id, -1, -1);
+        state = {};
         return false;
     }
+    req.stage = multi_asr_stage::decoding;
+    return true;
+}
 
-    // 2) <|audio_start|>
-    if (decode_tokens(tok_audio_beg_, n_past, /*logits_last*/ false) != 0) {
-        req.error = "decoder: failed to decode audio-begin";
+void multi_asr_decoder::finish_slot(slot & state, bool ok, const std::string & error) {
+    if (state.req == nullptr) {
+        return;
+    }
+    auto & req = *state.req;
+    if (state.active) {
+        req.timings.decode_ms = (double) (ggml_time_ms() - state.t_decode_start);
+    }
+    req.timings.n_out_tokens = (int32_t) state.generated.size();
+    if (ok) {
+        req.text = common_detokenize(lctx_, state.generated, false);
+        req.stage = multi_asr_stage::done;
+    } else {
+        req.error = error.empty() ? "decoder: generation failed" : error;
         req.stage = multi_asr_stage::failed;
-        return false;
     }
+    const int slot_id = (int) (&state - slots_.data());
+    llama_memory_seq_rm(llama_get_memory(lctx_), slot_id, -1, -1);
+    state = {};
+}
 
-    // 3) audio embedding
-    if (decode_embd(req.embd.data(), req.n_audio_tokens, n_past, /*logits_last*/ false) != 0) {
-        req.error = "decoder: failed to decode audio embedding";
-        req.stage = multi_asr_stage::failed;
-        return false;
+void multi_asr_decoder::step() {
+    if (!has_active()) {
+        return;
     }
 
-    // 4) <|audio_end|>
-    if (decode_tokens(tok_audio_end_, n_past, /*logits_last*/ false) != 0) {
-        req.error = "decoder: failed to decode audio-end";
-        req.stage = multi_asr_stage::failed;
-        return false;
+    // Sample one token for every sequence that already has a decoder state.
+    // New requests remain in prefill until their final prompt row produces the
+    // first logits; they join generation on the following scheduler tick.
+    for (auto & state : slots_) {
+        if (!state.active || state.req == nullptr) {
+            continue;
+        }
+        const llama_token tok = common_sampler_sample(state.sampler.get(), lctx_, state.i_batch);
+        common_sampler_accept(state.sampler.get(), tok, true);
+        if (llama_vocab_is_eog(vocab_, tok)) {
+            finish_slot(state, true);
+            continue;
+        }
+        state.generated.push_back(tok);
+        if ((int) state.generated.size() >= state.n_predict) {
+            finish_slot(state, true);
+        }
     }
 
-    // 5) assistant prefix + user text (logits on last -> ready to sample)
-    const std::vector<llama_token> toks_after =
-        common_tokenize(vocab_, split.after, /*add_special*/ false, /*parse_special*/ true);
-    if (decode_tokens(toks_after, n_past, /*logits_last*/ true) != 0) {
-        req.error = "decoder: failed to decode assistant prefix";
-        req.stage = multi_asr_stage::failed;
-        return false;
+    const int n_active = active_count();
+    int n_pending = 0;
+    for (const auto & state : slots_) {
+        if (state.occupied && !state.active && state.prefill_offset < state.prefill.size()) {
+            ++n_pending;
+        }
+    }
+    if (n_active == 0 && n_pending == 0) {
+        return;
     }
 
-    req.timings.prefill_ms = (double) (ggml_time_ms() - t_prefill0);
+    // All rows are F32 embeddings. Generated token ids are looked up in the
+    // model's token embedding table before admission, so the transformer sees
+    // one homogeneous input representation even when prefill and decode rows
+    // coexist in this batch.
+    llama_batch batch = llama_batch_init(n_batch_, (int32_t) hidden_size_, n_parallel_);
 
-    // 6) generation loop
-    const int64_t    t_decode0 = ggml_time_ms();
-    common_sampler * smpl      = common_sampler_init(model_, params_.sampling);
-    if (!smpl) {
-        req.error = "decoder: failed to init sampler";
-        req.stage = multi_asr_stage::failed;
-        return false;
+    int n_batch = 0;
+    std::vector<int> prefill_added(slots_.size(), 0);
+    std::vector<bool> decode_added(slots_.size(), false);
+
+    auto append_active = [&](slot & state) {
+        const int i = n_batch++;
+        const int slot_id = (int) (&state - slots_.data());
+        std::memcpy(batch.embd + (size_t) i * (size_t) hidden_size_,
+                    token_embedding(state.generated.back()), sizeof(float) * (size_t) hidden_size_);
+        batch.pos[i] = state.n_past;
+        batch.n_seq_id[i] = 1;
+        batch.seq_id[i][0] = slot_id;
+        batch.logits[i] = true;
+        state.i_batch = i;
+        decode_added[(size_t) slot_id] = true;
+    };
+
+    // Existing decode work gets first claim on the batch. New prefills use the
+    // remaining budget in FIFO slot order, so a stream of arrivals cannot
+    // starve older generation requests.
+    for (auto & state : slots_) {
+        if (!state.active || state.req == nullptr) {
+            continue;
+        }
+        append_active(state);
     }
 
-    std::vector<llama_token> generated;
-    const int                n_predict = req.n_predict > 0 ? req.n_predict : 256;
-    llama_batch              gen_batch = llama_batch_init(1, 0, 1);
-    bool                     ok        = true;
-    for (int i = 0; i < n_predict; ++i) {
-        const llama_token tok = common_sampler_sample(smpl, lctx_, -1);
-        common_sampler_accept(smpl, tok, true);
-        if (llama_vocab_is_eog(vocab_, tok)) {
-            break;
+    int prefill_budget = n_batch_ - n_batch;
+    // Admit one pending prefill request per scheduler tick. Existing decode
+    // rows have already claimed their budget; the oldest prompt can use all
+    // remaining rows and is split across ticks only when it exceeds n_batch.
+    slot * pending = nullptr;
+    for (auto & state : slots_) {
+        if (state.occupied && !state.active && state.req != nullptr &&
+            state.prefill_offset < state.prefill.size() &&
+            (pending == nullptr || state.admission_order < pending->admission_order)) {
+            pending = &state;
         }
-        generated.push_back(tok);
-
-        gen_batch.n_tokens     = 1;
-        gen_batch.token[0]     = tok;
-        gen_batch.pos[0]       = n_past++;
-        gen_batch.n_seq_id[0]  = 1;
-        gen_batch.seq_id[0][0] = 0;
-        gen_batch.logits[0]    = true;
-        if (llama_decode(lctx_, gen_batch) != 0) {
-            req.error = "decoder: llama_decode failed during generation";
-            ok        = false;
-            break;
+    }
+    if (pending != nullptr && prefill_budget > 0) {
+        auto & state = *pending;
+        const size_t remaining = state.prefill.size() - state.prefill_offset;
+        const int count = std::min<int>(prefill_budget, (int) remaining);
+        const int slot_id = (int) (&state - slots_.data());
+        for (int j = 0; j < count; ++j) {
+            const size_t row = state.prefill_offset + (size_t) j;
+            const auto & item = state.prefill[row];
+            const int i = n_batch++;
+            std::memcpy(batch.embd + (size_t) i * (size_t) hidden_size_, item.data,
+                        sizeof(float) * (size_t) hidden_size_);
+            batch.pos[i] = state.n_past + j;
+            batch.n_seq_id[i] = 1;
+            batch.seq_id[i][0] = slot_id;
+            const bool final_row = row + 1 == state.prefill.size();
+            batch.logits[i] = final_row;
+            if (final_row) {
+                state.i_batch = i;
+            }
         }
+        prefill_added[(size_t) slot_id] = count;
+    }
+    batch.n_tokens = n_batch;
+    if (n_batch == 0) {
+        llama_batch_free(batch);
+        return;
     }
-    llama_batch_free(gen_batch);
 
-    req.timings.decode_ms    = (double) (ggml_time_ms() - t_decode0);
-    req.timings.n_out_tokens = (int32_t) generated.size();
+    if (active_count() > 0 && n_pending > 0) {
+        LOG_DBG("[multi-asr] mixed batch: decode_rows=%d prefill_rows=%d total_rows=%d\n",
+                (int) std::count(decode_added.begin(), decode_added.end(), true),
+                std::accumulate(prefill_added.begin(), prefill_added.end(), 0), n_batch);
+    }
 
-    if (ok) {
-        req.text  = common_detokenize(lctx_, generated, /*special*/ false);
-        req.stage = multi_asr_stage::done;
+    if (llama_decode(lctx_, batch) != 0) {
+        for (auto & state : slots_) {
+            if (state.occupied) {
+                finish_slot(state, false, "decoder: llama_decode failed during continuous batching");
+            }
+        }
     } else {
-        req.stage = multi_asr_stage::failed;
+        for (auto & state : slots_) {
+            if (!state.occupied) {
+                continue;
+            }
+            const int slot_id = (int) (&state - slots_.data());
+            if (decode_added[(size_t) slot_id]) {
+                state.n_past += 1;
+            }
+            state.n_past += prefill_added[(size_t) slot_id];
+            if (prefill_added[(size_t) slot_id] > 0) {
+                state.prefill_offset += prefill_added[(size_t) slot_id];
+                if (state.prefill_offset == state.prefill.size()) {
+                    state.active = true;
+                    state.t_decode_start = ggml_time_ms();
+                    state.req->timings.prefill_ms = (double) (state.t_decode_start - state.t_prefill_start);
+                }
+            }
+        }
+
+    }
+    llama_batch_free(batch);
+}
+
+bool multi_asr_decoder::has_active() const {
+    return occupied_count() > 0;
+}
+
+int multi_asr_decoder::active_count() const {
+    int count = 0;
+    for (const auto & state : slots_) {
+        count += state.active ? 1 : 0;
     }
+    return count;
+}
 
-    common_sampler_free(smpl);
-    return ok;
+int multi_asr_decoder::occupied_count() const {
+    int count = 0;
+    for (const auto & state : slots_) {
+        count += state.occupied ? 1 : 0;
+    }
+    return count;
 }
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.h b/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.h
index e26c4799056..9e073283e46 100644
--- a/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.h
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-decoder.h
@@ -1,67 +1,77 @@
 #pragma once
 
-// Decoder stage: owns the gguf text model + a single llama_context (one slot).
-// Runs on its own worker thread, pinned to the decoder core set.
-// Given a request whose `embd` was filled by the encoder stage, it:
-//   1. decodes the Qwen3-ASR prompt prefix text tokens,
-//   2. decodes <|audio_start|> + audio embedding + <|audio_end|>,
-//   3. decodes the assistant prefix text tokens,
-//   4. generates output tokens serially until EOG / n_predict.
-//
-// This mirrors the faithful path in tools/mtmd/mtmd-cli-smt.cpp (eval_message_smt
-// + generate_response) but split so encode already happened on the encoder core set.
-
 #include "common.h"
 #include "llama.h"
 #include "multi-asr-common.h"
+#include "sampling.h"
 
 #include <memory>
+#include <cstdint>
 #include <string>
 #include <vector>
 
+// ASR-specific decoder scheduler. The encoder remains serial, while prompt
+// embedding prefill and active token generation share one continuous batch.
 class multi_asr_decoder {
   public:
     multi_asr_decoder() = default;
     ~multi_asr_decoder();
 
-    multi_asr_decoder(const multi_asr_decoder &)             = delete;
+    multi_asr_decoder(const multi_asr_decoder &) = delete;
     multi_asr_decoder & operator=(const multi_asr_decoder &) = delete;
 
-    // Load gguf model + create llama_context. `cpu_range` pins the ggml compute
-    // threads to the decoder core set. `hidden_size` must match the encoder's
-    // audio embedding dimension (validated against model n_embd).
-    // Throws std::runtime_error on failure.
-    void init(const multi_asr_params & params, int64_t hidden_size);
-
-    // Server integration variant: reuse the already loaded text model while
-    // creating a private llama_context for the legacy ASR decoder.  The model
-    // is owned by llama-server; this object only owns the context.
+    // Borrow the server's model and create one private context with one KV
+    // sequence per ASR slot.
     void init_shared(llama_model * model, const multi_asr_params & params, int64_t hidden_size);
 
-    // Decode one request (prefill audio embd + generate). Fills req.text and
-    // timings.prefill_ms / decode_ms / n_out_tokens. Resets KV per request.
-    // Returns true on success; on failure sets req.error and returns false.
-    bool decode(multi_asr_request & req);
+    // Prepare one encoded request for admission into a free sequence.
+    bool start(multi_asr_request & req, int slot_id);
 
-    int64_t hidden_size() const { return hidden_size_; }
+    // Run one dynamic prefill/decode scheduler iteration.
+    void step();
+
+    bool has_active() const;
+    int  active_count() const;
+    int  capacity() const { return n_parallel_; }
+    llama_context * context() const { return lctx_; }
 
   private:
-    common_init_result_ptr init_;  // owns model + context lifetime
-    llama_model *          model_ = nullptr;
-    llama_context *        lctx_  = nullptr;
-    bool                   owns_context_ = false;
-    const llama_vocab *    vocab_ = nullptr;
+    struct slot {
+        struct prefill_row {
+            const float * data = nullptr;
+        };
 
-    int64_t hidden_size_ = 0;
-    int32_t n_batch_     = 2048;
+        multi_asr_request * req = nullptr;
+        std::unique_ptr<common_sampler, common_sampler_deleter> sampler;
+        std::vector<llama_token> generated;
+        std::vector<prefill_row> prefill;
+        size_t prefill_offset = 0;
+        llama_pos n_past = 0;
+        int i_batch = 0;
+        int n_predict = 0;
+        int64_t t_decode_start = 0;
+        int64_t t_prefill_start = 0;
+        uint64_t admission_order = 0;
+        bool active = false;
+        bool occupied = false;
+    };
 
-    // Qwen3-ASR audio boundary tokens (<|audio_start|>, <|audio_end|>).
+    llama_model * model_ = nullptr;
+    llama_context * lctx_ = nullptr;
+    const llama_vocab * vocab_ = nullptr;
+    common_params params_;
+
+    int64_t hidden_size_ = 0;
+    int32_t n_batch_ = 2048;
+    int32_t n_parallel_ = 1;
+    std::vector<slot> slots_;
     std::vector<llama_token> tok_audio_beg_;
     std::vector<llama_token> tok_audio_end_;
+    std::vector<float> token_embd_;
+    uint64_t next_admission_order_ = 0;
 
-    common_params params_;  // holds sampling config for per-request samplers
-
-    // helpers (mirrors mtmd-cli-smt.cpp)
-    int decode_tokens(const std::vector<llama_token> & tokens, llama_pos & n_past, bool logits_last);
-    int decode_embd(const float * embd, int n_tokens, llama_pos & n_past, bool logits_last);
+    void load_token_embeddings();
+    const float * token_embedding(llama_token token) const;
+    void finish_slot(slot & state, bool ok, const std::string & error = {});
+    int occupied_count() const;
 };
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.cpp b/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.cpp
index 06234117abb..09b831ae91c 100644
--- a/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.cpp
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.cpp
@@ -1,41 +1,27 @@
 #include "multi-asr-orchestrator.h"
 
-#include "smt-audio-wrapper.h"  // full type for smt_audio_context (unique_ptr dtor)
 #include "ggml.h"
 #include "log.h"
+#include "smt-audio-wrapper.h"
 
+#include <algorithm>
 #include <stdexcept>
-#include <utility>
-
-multi_asr_orchestrator::multi_asr_orchestrator() = default;
 
 multi_asr_orchestrator::~multi_asr_orchestrator() {
     stop();
 }
 
-void multi_asr_orchestrator::init(const multi_asr_params & params) {
-    params_ = params;
-
-    // Encoder must init first so we know hidden_size, then validate decoder.
-    LOG_INF("[multi-asr] initializing encoder (cores from config.json)...\n");
-    encoder_.init(params.smt_config_dir, params.warmup);
-    hidden_size_ = encoder_.hidden_size();
-
-    LOG_INF("[multi-asr] initializing decoder (cores='%s')...\n", params.decoder_cpu_range.c_str());
-    decoder_.init(params, hidden_size_);
-
-    LOG_INF("[multi-asr] ready. hidden_size=%lld n_parallel=%d queue_max=%d pipeline=%s\n", (long long) hidden_size_,
-            params.n_parallel, params.queue_max, params.enable_pipeline ? "on" : "off");
-}
-
 void multi_asr_orchestrator::init_shared(llama_model * model, const multi_asr_params & params) {
+    if (model == nullptr) {
+        throw std::runtime_error("multi_asr_orchestrator: null model");
+    }
     params_ = params;
-    LOG_INF("[multi-asr] initializing fully isolated legacy decoder\n");
-    encoder_.init(params.smt_config_dir, params.warmup);
+    encoder_.init(params_.smt_config_dir, params_.warmup);
     hidden_size_ = encoder_.hidden_size();
-    decoder_.init(params, hidden_size_);
-    LOG_INF("[multi-asr] isolated legacy decoder ready. hidden_size=%lld\n", (long long) hidden_size_);
-    GGML_UNUSED(model);
+    decoder_.init_shared(model, params_, hidden_size_);
+    slot_busy_.assign((size_t) std::max(1, params_.n_parallel), false);
+    LOG_INF("[multi-asr] ready: serial encoder, continuous-batch decoder, slots=%d\n",
+            std::max(1, params_.n_parallel));
 }
 
 void multi_asr_orchestrator::start() {
@@ -61,128 +47,147 @@ void multi_asr_orchestrator::stop() {
 }
 
 bool multi_asr_orchestrator::submit_and_wait(const multi_asr_request & req_in, multi_asr_request & out) {
-    // Backpressure: reject if the system is already at its hard cap.
-    if (in_flight_.load() >= params_.queue_max) {
-        out       = req_in;
-        out.error = "server busy: queue full";
+    if (!running_.load()) {
+        out = req_in;
+        out.error = "multi-ASR service is stopped";
+        out.stage = multi_asr_stage::failed;
+        return false;
+    }
+    if (in_flight_.load() >= std::max(1, params_.queue_max)) {
+        out = req_in;
+        out.error = "multi-ASR queue is full";
         out.stage = multi_asr_stage::failed;
         return false;
     }
 
-    auto job         = std::make_shared<multi_asr_job>();
-    job->req         = req_in;
+    auto job = std::make_shared<multi_asr_job>();
+    job->req = req_in;
     job->t_submit_ms = ggml_time_ms();
-
     in_flight_.fetch_add(1);
+    pending_encode_.fetch_add(1);
     {
-        std::lock_guard<std::mutex> lk(intake_mu_);
+        std::lock_guard<std::mutex> lock(intake_mu_);
         intake_.push_back(job);
     }
     intake_cv_.notify_one();
 
-    // Block until the pipeline finishes this job.
-    {
-        std::unique_lock<std::mutex> lk(job->mu);
-        job->cv.wait(lk, [&] { return job->finished; });
-    }
-
-    out = std::move(job->req);
+    std::unique_lock<std::mutex> lock(job->mu);
+    job->cv.wait(lock, [&] { return job->finished; });
+    out = job->req;
     in_flight_.fetch_sub(1);
     return out.stage != multi_asr_stage::failed;
 }
 
-// Encode worker: pulls from intake, runs ONNX encode on the encoder cores,
-// pushes the encoded job to the ready queue for the decoder.
+void multi_asr_orchestrator::finish_job(const multi_asr_job_ptr & job) {
+    job->req.timings.total_ms = (double) (ggml_time_ms() - job->t_submit_ms);
+    {
+        std::lock_guard<std::mutex> lock(job->mu);
+        job->finished = true;
+    }
+    job->cv.notify_one();
+}
+
 void multi_asr_orchestrator::encode_loop() {
     while (running_.load()) {
         multi_asr_job_ptr job;
         {
-            std::unique_lock<std::mutex> lk(intake_mu_);
-            intake_cv_.wait(lk, [&] { return !intake_.empty() || !running_.load(); });
+            std::unique_lock<std::mutex> lock(intake_mu_);
+            intake_cv_.wait(lock, [&] { return !intake_.empty() || !running_.load(); });
             if (!running_.load() && intake_.empty()) {
-                break;
+                return;
             }
             job = intake_.front();
             intake_.pop_front();
         }
 
-        // In strict-serial (no pipeline) mode, only one job is ever in intake at
-        // a time because submit blocks; the encode->decode handoff below still
-        // serializes naturally since ready_ holds at most one item.
-
-        const int64_t t0          = ggml_time_ms();
-        job->req.timings.queue_ms = (double) (t0 - job->t_submit_ms);
-        job->req.stage            = multi_asr_stage::encoding;
-
-        const bool ok = encoder_.encode(job->req);
-        if (!ok) {
-            // finish immediately with error
-            {
-                std::lock_guard<std::mutex> lk(job->mu);
-                job->req.timings.total_ms = (double) (ggml_time_ms() - job->t_submit_ms);
-                job->finished             = true;
-            }
-            job->cv.notify_one();
+        job->req.stage = multi_asr_stage::encoding;
+        job->req.timings.queue_ms = (double) (ggml_time_ms() - job->t_submit_ms);
+        // Experimental no-gate path: let ORT encoder work overlap GGML/IME
+        // decoder work.  All other FIFO and continuous-batch scheduling stays
+        // unchanged.
+        const bool encode_ok = encoder_.encode(job->req);
+        if (!encode_ok) {
+            pending_encode_.fetch_sub(1);
+            finish_job(job);
+            ready_cv_.notify_one();
             continue;
         }
 
-        // In the server facade, keep the legacy FIFO queue but execute the
-        // complete encode->decode transaction on this worker.  The server
-        // process owns a second GGUF context; overlapping SMT encoder work
-        // with GGUF decode corrupts backend state even though the standalone
-        // binary is safe to pipeline.  This path also avoids a cross-worker
-        // wait cycle when strict mode is selected.
-        if (!params_.enable_pipeline) {
-            decoder_.decode(job->req);
-            job->req.timings.total_ms = (double) (ggml_time_ms() - job->t_submit_ms);
-            {
-                std::lock_guard<std::mutex> lk(job->mu);
-                job->finished = true;
-            }
-            job->cv.notify_one();
-            continue;
-        }
-
-        // Hand off to decoder. Bound the ready queue to depth 1 so the encoder
-        // runs at most ONE step ahead of the decoder — the clean B1 staggered
-        // overlap E(N+1)‖D(N), not unbounded runahead.
         {
-            std::unique_lock<std::mutex> lk(ready_mu_);
-            ready_cv_.wait(lk, [&] { return ready_.empty() || !running_.load(); });
-            if (!running_.load()) {
-                break;
-            }
+            std::lock_guard<std::mutex> lock(ready_mu_);
             ready_.push_back(job);
         }
+        pending_encode_.fetch_sub(1);
         ready_cv_.notify_one();
-
     }
 }
 
-// Decode worker: pulls encoded jobs, runs gguf decode on the decoder cores.
 void multi_asr_orchestrator::decode_loop() {
     while (running_.load()) {
-        multi_asr_job_ptr job;
-        {
-            std::unique_lock<std::mutex> lk(ready_mu_);
-            ready_cv_.wait(lk, [&] { return !ready_.empty() || !running_.load(); });
-            if (!running_.load() && ready_.empty()) {
+        // Fill free decoder slots from the FIFO as soon as encoded requests
+        // arrive. `decoder_.start()` only prepares the request's prompt rows;
+        // its first embedding prefill is admitted by the next mixed batch.
+        while ((int) active_.size() < decoder_.capacity()) {
+            multi_asr_job_ptr job;
+            {
+                std::lock_guard<std::mutex> lock(ready_mu_);
+                if (ready_.empty()) {
+                    break;
+                }
+                job = ready_.front();
+                ready_.pop_front();
+            }
+
+            int slot_id = -1;
+            for (size_t i = 0; i < slot_busy_.size(); ++i) {
+                if (!slot_busy_[i]) {
+                    slot_id = (int) i;
+                    break;
+                }
+            }
+            if (slot_id < 0) {
+                std::lock_guard<std::mutex> lock(ready_mu_);
+                ready_.push_front(job);
                 break;
             }
-            job = ready_.front();
-            ready_.pop_front();
+
+            slot_busy_[(size_t) slot_id] = true;
+            job->slot_id = slot_id;
+            if (!decoder_.start(job->req, slot_id)) {
+                slot_busy_[(size_t) slot_id] = false;
+                finish_job(job);
+            } else {
+                active_.push_back(job);
+            }
         }
-        // Wake the encoder: the ready slot is now free, so E(N+1) can hand off
-        // while we decode this job (the staggered overlap).
-        ready_cv_.notify_one();
 
-        decoder_.decode(job->req);
-        job->req.timings.total_ms = (double) (ggml_time_ms() - job->t_submit_ms);
+        // Every tick admits existing decode rows and the earliest pending audio
+        // prefill rows into one mixed llama_batch. This removes the old wave
+        // barrier while preserving FIFO admission and decode priority.
+        if (decoder_.has_active()) {
+            decoder_.step();
+            // Keep the completion fence for this experimental run, but do not
+            // serialize it with ORT encoder execution.
+            llama_synchronize(decoder_.context());
+            for (auto it = active_.begin(); it != active_.end();) {
+                const auto & job = *it;
+                if (job->req.stage == multi_asr_stage::done || job->req.stage == multi_asr_stage::failed) {
+                    slot_busy_[(size_t) job->slot_id] = false;
+                    finish_job(job);
+                    it = active_.erase(it);
+                } else {
+                    ++it;
+                }
+            }
+            continue;
+        }
 
-        {
-            std::lock_guard<std::mutex> lk(job->mu);
-            job->finished = true;
+        if (active_.empty()) {
+            std::unique_lock<std::mutex> lock(ready_mu_);
+            ready_cv_.wait(lock, [&] { return !running_.load() || !ready_.empty(); });
+            if (!running_.load()) {
+                return;
+            }
         }
-        job->cv.notify_one();
     }
 }
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.h b/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.h
index e9dd23d2965..0879b04059b 100644
--- a/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.h
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-orchestrator.h
@@ -1,19 +1,9 @@
 #pragma once
 
-// Orchestrator: implements scheme B1 "错峰流水" (staggered pipeline).
-//
-//   intake queue ──► [encode thread](encoder cores) ──► ready queue ──► [decode thread](decoder cores)
-//
-// Two dedicated worker threads run on disjoint core sets, so E(N+1) overlaps
-// D(N). Each stage is serial (one at a time), matching the B1 semantics: at any
-// instant at most one encode + one decode in flight.
-//
-// Concurrency control:
-//   - submit() blocks-or-rejects when total in-flight+waiting >= queue_max.
-//   - `n_parallel` bounds how many requests may be "in the pipeline" before new
-//     submissions wait (the FIFO ">N waits" requirement).
-//   - enable_pipeline=false => strict serial baseline (decode finishes before
-//     next encode starts), for benchmarking variable ②.
+// FIFO ASR scheduler:
+//   HTTP submit -> serial SMT encoder -> decoder ready FIFO -> native
+//   continuous-batching generation.  The decoder thread is the sole owner of
+//   its llama_context, so KV sequences and samplers cannot race.
 
 #include "multi-asr-common.h"
 #include "multi-asr-decoder.h"
@@ -26,61 +16,55 @@
 #include <memory>
 #include <mutex>
 #include <thread>
+#include <vector>
 
-// A submitted job: shared between the HTTP handler (waiter) and the pipeline.
 struct multi_asr_job {
-    multi_asr_request       req;
-    std::mutex              mu;
+    multi_asr_request req;
+    std::mutex mu;
     std::condition_variable cv;
-    bool                    finished    = false;
-    int64_t                 t_submit_ms = 0;  // ggml_time_ms at submit
+    bool finished = false;
+    int64_t t_submit_ms = 0;
+    int slot_id = -1;
 };
 
 using multi_asr_job_ptr = std::shared_ptr<multi_asr_job>;
 
 class multi_asr_orchestrator {
   public:
-    multi_asr_orchestrator();
+    multi_asr_orchestrator() = default;
     ~multi_asr_orchestrator();
 
-    // Initialize encoder + decoder (loads models, pins cores). Throws on failure.
-    void init(const multi_asr_params & params);
     void init_shared(llama_model * model, const multi_asr_params & params);
-
-    // Start the encode/decode worker threads.
     void start();
-
-    // Stop workers and drain. Safe to call once.
     void stop();
 
-    // Submit a request and BLOCK until it finishes (encode+decode) or fails.
-    // Returns false if rejected due to a full queue (sets job->req.error).
     bool submit_and_wait(const multi_asr_request & req_in, multi_asr_request & out);
-
     int64_t hidden_size() const { return hidden_size_; }
+    llama_context * context() const { return decoder_.context(); }
 
   private:
-    multi_asr_params  params_;
+    multi_asr_params params_;
     multi_asr_encoder encoder_;
     multi_asr_decoder decoder_;
-    int64_t           hidden_size_ = 0;
+    int64_t hidden_size_ = 0;
 
-    // intake -> encode
     std::deque<multi_asr_job_ptr> intake_;
-    std::mutex                    intake_mu_;
-    std::condition_variable       intake_cv_;
+    std::mutex intake_mu_;
+    std::condition_variable intake_cv_;
 
-    // encode -> decode (bounded to 1 for B1: only one encoded result waiting)
     std::deque<multi_asr_job_ptr> ready_;
-    std::mutex                    ready_mu_;
-    std::condition_variable       ready_cv_;
-
-    std::atomic<int32_t> in_flight_{ 0 };  // total accepted, not yet finished
-    std::atomic<bool>    running_{ false };
-
+    std::mutex ready_mu_;
+    std::condition_variable ready_cv_;
+
+    std::vector<multi_asr_job_ptr> active_;
+    std::vector<bool> slot_busy_;
+    std::atomic<int32_t> in_flight_{ 0 };
+    std::atomic<int32_t> pending_encode_{ 0 };
+    std::atomic<bool> running_{ false };
     std::thread encode_thread_;
     std::thread decode_thread_;
 
     void encode_loop();
     void decode_loop();
+    void finish_job(const multi_asr_job_ptr & job);
 };
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-service.cpp b/tools/smt-mtmd/smt/multi-asr/multi-asr-service.cpp
index 8178e84f3af..d242981cee4 100644
--- a/tools/smt-mtmd/smt/multi-asr/multi-asr-service.cpp
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-service.cpp
@@ -17,21 +17,12 @@ void smt_multi_asr_service::init(llama_model * model, const common_params & serv
     p.smt_config_dir = server_params.smt_config_dir;
     p.warmup = server_params.warmup;
     p.n_parallel = std::max(1, server_params.n_parallel);
-    p.queue_max = std::max(1, server_params.n_parallel * 8);
+    p.queue_max = std::max(p.n_parallel, p.n_parallel * 8);
     p.n_predict = server_params.n_predict > 0 ? server_params.n_predict : 256;
-    // Keep the legacy decoder's model/context settings independent from the
-    // HTTP server.  In particular, inheriting the server's reduced n_ctx or
-    // its HTTP thread count changes the standalone ASR graph and can produce
-    // invalid output on SpacemiT's backend.  The old multi-ASR binary uses
-    // model-default context and backend-default decoder threading.
-    p.n_ctx = 0;
-    p.n_batch = 2048;
-    p.decoder_n_threads = 0;
-    // Preserve the old FIFO intake semantics, but serialize each complete
-    // encode->decode transaction inside llama-server.  The standalone binary
-    // can overlap E/D; llama-server owns a second GGUF context in the same
-    // process, where that overlap corrupts backend state.
-    p.enable_pipeline = false;
+    // The ASR context is sized as a server-style total context. Each slot gets
+    // its own sequence; generation is batched across those sequences.
+    p.n_ctx = server_params.n_ctx;
+    p.n_batch = std::max(1, server_params.n_batch);
 
     orchestrator_ = std::make_unique<multi_asr_orchestrator>();
     orchestrator_->init_shared(model, p);
@@ -45,9 +36,14 @@ bool smt_multi_asr_service::submit(const std::vector<uint8_t> & audio, const std
         result.stage = multi_asr_stage::failed;
         return false;
     }
+
     multi_asr_request req;
     req.audio = audio;
     req.prompt = prompt.empty() ? "language Chinese<asr_text>" : prompt;
+    req.n_predict = n_predict > 0 ? n_predict : 256;
+
+    // The OAI parser may provide a rendered ChatML prompt. Only the short
+    // Qwen3-ASR instruction belongs after the assistant marker.
     const std::string assistant = "<|im_start|>assistant\n";
     if (const size_t pos = req.prompt.rfind(assistant); pos != std::string::npos) {
         const std::string tail = req.prompt.substr(pos + assistant.size());
@@ -55,15 +51,6 @@ bool smt_multi_asr_service::submit(const std::vector<uint8_t> & audio, const std
             req.prompt = tail;
         }
     }
-    // Chat-template rendering may place the user instruction before the
-    // assistant generation marker.  Recover the legacy Qwen3-ASR instruction
-    // instead of passing the whole rendered conversation to the decoder.
-    // The server's `prompt` field is usually a fully rendered chat template,
-    // and some templates stringify a missing language as `language None`.
-    // The legacy decoder expects only the short Qwen3-ASR instruction.  Keep a
-    // valid explicit language when it is present; otherwise use the proven
-    // Chinese default.  Never pass a rendered template or a `None` language
-    // through to the decoder.
     const size_t lang = req.prompt.find("language ");
     const size_t marker = lang == std::string::npos ? std::string::npos : req.prompt.find("<asr_text>", lang);
     if (lang != std::string::npos && marker != std::string::npos) {
@@ -76,6 +63,9 @@ bool smt_multi_asr_service::submit(const std::vector<uint8_t> & audio, const std
     } else {
         req.prompt = "language Chinese<asr_text>";
     }
-    req.n_predict = n_predict > 0 ? n_predict : 256;
     return orchestrator_->submit_and_wait(req, result);
 }
+
+llama_context * smt_multi_asr_service::context() const {
+    return orchestrator_ ? orchestrator_->context() : nullptr;
+}
diff --git a/tools/smt-mtmd/smt/multi-asr/multi-asr-service.h b/tools/smt-mtmd/smt/multi-asr/multi-asr-service.h
index e0eff7ee6e1..508e80f033f 100644
--- a/tools/smt-mtmd/smt/multi-asr/multi-asr-service.h
+++ b/tools/smt-mtmd/smt/multi-asr/multi-asr-service.h
@@ -5,9 +5,7 @@
 
 #include <memory>
 
-// Server-facing facade for the legacy, correctness-proven ASR pipeline.
-// The facade deliberately does not expose llama-server slots or media chunks:
-// requests are admitted FIFO and decoded by the private legacy context.
+// Thin HTTP-facing facade over the ASR-owned encoder/decoder scheduler.
 class smt_multi_asr_service {
   public:
     smt_multi_asr_service() = default;
@@ -19,6 +17,7 @@ class smt_multi_asr_service {
     void init(llama_model * model, const common_params & server_params);
     bool submit(const std::vector<uint8_t> & audio, const std::string & prompt,
                 int32_t n_predict, multi_asr_request & result);
+    llama_context * context() const;
 
   private:
     std::unique_ptr<multi_asr_orchestrator> orchestrator_;

From 8274512ee81fc216616a96d515b746e5bd4b66dd Mon Sep 17 00:00:00 2001
From: co-seven <linxi.cai@spacemit.com>
Date: Fri, 28 Aug 2026 06:06:36 +0000
Subject: [PATCH 2/2] chore: bump version to 0.1.9

---
 VERSION_NUMBER | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION_NUMBER b/VERSION_NUMBER
index 699c6c6d4e0..1a030947e83 100644
--- a/VERSION_NUMBER
+++ b/VERSION_NUMBER
@@ -1 +1 @@
-0.1.8
+0.1.9