feat(server): integrate isolated multi-ASR service

common/arg.cpp

@@ -2381,6 +2381,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.smt_config_dir = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_MTMD}).set_env("LLAMA_ARG_SMT_CONFIG_DIR"));
+    add_opt(common_arg(
+        {"--smt-multi-asr"},
+        "enable the isolated legacy multi-ASR FIFO service (default: disabled)",
+        [](common_params & params) {
+            params.smt_multi_asr = true;
+        }
+    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_SMT_MULTI_ASR"));
 #endif
     add_opt(common_arg(
         {"--image", "--audio", "--video"}, "FILE",

common/common.h

@@ -598,6 +598,7 @@ struct common_params {
 #if defined(LLAMA_SERVER_SMT_MTMD)
     std::string media_backend = "auto"; // multimodal backend: auto|mtmd|smt
     std::string smt_config_dir;         // SMT config dir (config.json + ONNX)
+    bool smt_multi_asr = false;         // opt in to the isolated legacy multi-ASR FIFO service
 #endif
     std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
     int image_min_tokens = -1;

tools/server/server-context.cpp

@@ -8,6 +8,9 @@
 #include "server-queue.h"
 #include "server-schema.h"
 #include "server-stream.h"
+#if defined(LLAMA_SERVER_SMT_MTMD)
+#include "multi-asr-service.h"
+#endif
 
 #include "build-info.h"
 #include "common.h"
@@ -894,6 +897,9 @@ struct server_context_impl {
     server_media_context * media = nullptr;
     mtmd_context * mctx = nullptr;
     const llama_vocab * vocab = nullptr;
+#if defined(LLAMA_SERVER_SMT_MTMD)
+    std::unique_ptr<smt_multi_asr_service> smt_asr_service;
+#endif
 
     server_queue    queue_tasks;
     server_response queue_results;
@@ -972,6 +978,11 @@ struct server_context_impl {
     int64_t t_last_load_progress_ms = 0;
 
     void destroy() {
+#if defined(LLAMA_SERVER_SMT_MTMD)
+        // The isolated ASR context borrows model_tgt; tear it down before the
+        // owning common_init_result releases the model.
+        smt_asr_service.reset();
+#endif
         spec.reset();
         spec_init.reset();
 
@@ -1050,15 +1061,35 @@ struct server_context_impl {
 #if defined(LLAMA_SERVER_SMT_MTMD)
         const bool has_smt_media = !params.smt_config_dir.empty() &&
                 (params.media_backend == "auto" || params.media_backend == "smt");
+        const bool use_multi_asr = has_smt_media && !has_mmproj && params.smt_multi_asr;
 #else
         const bool has_smt_media = false;
+        const bool use_multi_asr = false;
 #endif
         const bool has_draft = params.speculative.has_dft();
         const bool spec_mtp = std::find(params_base.speculative.types.begin(),
                                         params_base.speculative.types.end(),
                                         COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end();
         const bool has_spec = has_draft || spec_mtp;
 
+#if defined(LLAMA_SERVER_SMT_MTMD)
+        // Initialize the legacy ASR stack before loading the server's own
+        // llama model.  SpacemiT's backend keeps process-wide state, and the
+        // proven standalone implementation is sensitive to a second model
+        // being loaded ahead of it.
+        if (use_multi_asr && !smt_asr_service) {
+            try {
+                smt_asr_service = std::make_unique<smt_multi_asr_service>();
+                smt_asr_service->init(nullptr, params_base);
+                SRV_INF("%s", "initialized isolated SMT multi-ASR service\n");
+            } catch (const std::exception & e) {
+                SRV_ERR("failed to initialize isolated SMT multi-ASR service: %s\n", e.what());
+                smt_asr_service.reset();
+                return false;
+            }
+        }
+#endif
+
         if (callback_state) {
             std::vector<std::string> stages = {"text_model"};
             if (has_spec) {
@@ -1078,7 +1109,9 @@ struct server_context_impl {
         }
 
         std::unique_ptr<media_worker> media_worker_init;
-        if (has_mmproj || has_smt_media) {
+        // Default SMT audio keeps the original generic media path. Only the
+        // explicit multi-ASR mode uses the isolated legacy service.
+        if (has_mmproj || (has_smt_media && !use_multi_asr)) {
             std::string worker_backend = "mtmd";
 #if defined(LLAMA_SERVER_SMT_MTMD)
             if (params_base.media_backend == "smt" || (params_base.media_backend == "auto" && !has_mmproj && has_smt_media)) {
@@ -1248,7 +1281,7 @@ struct server_context_impl {
             load_progress_callback(1.0f, &load_progress_spec);
         }
 
-        if (has_mmproj || has_smt_media) {
+        if (has_mmproj || (has_smt_media && !use_multi_asr)) {
             if (callback_state) {
                 callback_state(SERVER_STATE_LOADING, {{"stage", has_mmproj ? "mmproj_model" : "media_model"}});
             }
@@ -1527,14 +1560,19 @@ struct server_context_impl {
             // IMPORTANT: chat_params is reused across sleeping / resuming states,
             //            never store llama_context/llama_model pointers in chat_params,
             //            as they may be invalidated after sleeping
+            bool allow_audio = media ? media->supports_audio() : false;
+#if defined(LLAMA_SERVER_SMT_MTMD)
+            allow_audio = allow_audio || smt_asr_service != nullptr;
+#endif
+
             chat_params = {
                 /* use_jinja             */ params_base.use_jinja,
                 /* prefill_assistant     */ params_base.prefill_assistant,
                 /* reasoning_format      */ params_base.reasoning_format,
                 /* chat_template_kwargs  */ params_base.default_template_kwargs,
                 /* tmpls                 */ std::move(chat_templates),
                 /* allow_image           */ media ? media->supports_vision() : false,
-                /* allow_audio           */ media ? media->supports_audio()  : false,
+                /* allow_audio           */ allow_audio,
                 /* allow_video           */ media ? media->supports_video()  : false,
                 /* enable_thinking       */ enable_thinking,
                 /* reasoning_budget      */ params_base.sampling.reasoning_budget_tokens,
@@ -4024,13 +4062,18 @@ server_context_meta server_context::get_meta() const {
 
     const char * ftype_name = llama_ftype_name(llama_model_ftype(impl->model_tgt));
 
+    bool has_mtmd = impl->media != nullptr && impl->media->supports_prompt_embeddings();
+#if defined(LLAMA_SERVER_SMT_MTMD)
+    has_mtmd = has_mtmd || impl->smt_asr_service != nullptr;
+#endif
+
     return server_context_meta {
         /* build_info             */ std::string(llama_build_info()),
         /* model_name             */ impl->model_name,
         /* model_aliases          */ impl->model_aliases,
         /* model_tags             */ impl->model_tags,
         /* model_path             */ impl->params_base.model.path,
-        /* has_mtmd               */ impl->media != nullptr && impl->media->supports_prompt_embeddings(),
+        /* has_mtmd               */ has_mtmd,
         /* has_inp_image          */ impl->chat_params.allow_image,
         /* has_inp_audio          */ impl->chat_params.allow_audio,
         /* has_inp_video          */ impl->chat_params.allow_video,
@@ -4116,6 +4159,55 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
     int32_t sse_ping_interval = params.sse_ping_interval;
 
     try {
+        // Audio-only SMT requests use the isolated legacy ASR orchestrator.
+        // This synchronous facade is intentionally ahead of task creation:
+        // no server slot, PEG parser, prompt cache, or shared KV sequence is
+        // touched by these requests.
+#if defined(LLAMA_SERVER_SMT_MTMD)
+        const std::string raw_prompt = data.contains("prompt") && data.at("prompt").is_string()
+                ? data.at("prompt").get<std::string>() : std::string();
+        // oaicompat_chat_params_parse() has already decoded input_audio into
+        // `files`; chat-template rendering may remove the media marker from
+        // the resulting prompt, so do not use the rendered text to decide
+        // whether this is an audio request.
+        const bool has_audio_media = !files.empty();
+        if (ctx_server.smt_asr_service && has_audio_media) {
+            multi_asr_request asr_result;
+            const int32_t n_predict = json_value(data, "max_tokens", 256);
+            const std::string user_prompt = raw_prompt;
+            const bool ok = ctx_server.smt_asr_service->submit(files.front(), user_prompt, n_predict, asr_result);
+            if (!ok) {
+                res->error(format_error_response(asr_result.error.empty() ? "ASR request failed" : asr_result.error,
+                                                 ERROR_TYPE_SERVER));
+                return res;
+            }
+            const json choice = {
+                {"index", 0},
+                {"message", {{"role", "assistant"}, {"content", asr_result.text}}},
+                {"finish_reason", "stop"},
+            };
+            res->ok({
+                {"id", completion_id},
+                {"object", "chat.completion"},
+                {"created", std::time(nullptr)},
+                {"model", meta->model_name},
+                {"choices", json::array({choice})},
+                {"usage", {{"prompt_tokens", asr_result.timings.n_audio_tokens},
+                            {"completion_tokens", asr_result.timings.n_out_tokens},
+                            {"total_tokens", asr_result.timings.n_audio_tokens + asr_result.timings.n_out_tokens}}},
+                {"multi_asr_timings", {
+                    {"queue_ms", asr_result.timings.queue_ms},
+                    {"encode_ms", asr_result.timings.encode_ms},
+                    {"prefill_ms", asr_result.timings.prefill_ms},
+                    {"decode_ms", asr_result.timings.decode_ms},
+                    {"total_ms", asr_result.timings.total_ms},
+                    {"n_audio_tokens", asr_result.timings.n_audio_tokens},
+                    {"n_out_tokens", asr_result.timings.n_out_tokens},
+                }},
+            });
+            return res;
+        }
+#endif
         std::vector<server_task> tasks;
 
         const auto & prompt = data.at("prompt");

tools/smt-mtmd/CMakeLists.txt

@@ -45,6 +45,16 @@ if(LLAMA_SERVER_SMT_MTMD)
     target_sources(${TARGET} PRIVATE
         smt/smt-audio-wrapper.cpp
         smt/smt-audio-wrapper.h
+        smt/multi-asr/multi-asr-common.cpp
+        smt/multi-asr/multi-asr-common.h
+        smt/multi-asr/multi-asr-encoder.cpp
+        smt/multi-asr/multi-asr-encoder.h
+        smt/multi-asr/multi-asr-decoder.cpp
+        smt/multi-asr/multi-asr-decoder.h
+        smt/multi-asr/multi-asr-orchestrator.cpp
+        smt/multi-asr/multi-asr-orchestrator.h
+        smt/multi-asr/multi-asr-service.cpp
+        smt/multi-asr/multi-asr-service.h
         smt/smt-media-common.cpp
         smt/smt-media-common.h
         smt/smt-profile.h
@@ -64,8 +74,10 @@ if(LLAMA_SERVER_SMT_MTMD)
     )
     target_include_directories(${TARGET} PUBLIC
         ${CMAKE_CURRENT_SOURCE_DIR}/smt
+        ${CMAKE_CURRENT_SOURCE_DIR}/smt/multi-asr
     )
     target_include_directories(${TARGET} PRIVATE
+        ${CMAKE_SOURCE_DIR}/common
         ${CMAKE_SOURCE_DIR}/tools/mtmd
         ${CMAKE_SOURCE_DIR}/vendor
         ${CMAKE_SOURCE_DIR}/ggml/src

tools/smt-mtmd/smt/multi-asr/multi-asr-common.cpp

@@ -0,0 +1,94 @@
+#include "multi-asr-common.h"
+
+#include <cctype>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <sstream>
+#include <string>
+
+// Parse one segment: either a single core "8" or a dash range "8-11".
+// Returns true on success and sets the corresponding mask bits.
+static bool parse_one_segment(const std::string & seg, bool (&mask)[GGML_MAX_N_THREADS]) {
+    if (seg.empty()) {
+        return false;
+    }
+
+    const size_t dash = seg.find('-');
+    if (dash == std::string::npos) {
+        // single cpu index
+        char *     endp = nullptr;
+        const long cpu  = std::strtol(seg.c_str(), &endp, 10);
+        if (endp == seg.c_str() || *endp != '\0' || cpu < 0 || cpu >= GGML_MAX_N_THREADS) {
+            return false;
+        }
+        mask[cpu] = true;
+        return true;
+    }
+
+    // dash range "lo-hi"
+    const std::string lo_s = seg.substr(0, dash);
+    const std::string hi_s = seg.substr(dash + 1);
+    char *            e1   = nullptr;
+    char *            e2   = nullptr;
+    const long        lo   = std::strtol(lo_s.c_str(), &e1, 10);
+    const long        hi   = std::strtol(hi_s.c_str(), &e2, 10);
+    if (e1 == lo_s.c_str() || *e1 != '\0' || e2 == hi_s.c_str() || *e2 != '\0') {
+        return false;
+    }
+    if (lo < 0 || hi < 0 || lo >= GGML_MAX_N_THREADS || hi >= GGML_MAX_N_THREADS || lo > hi) {
+        return false;
+    }
+    for (long c = lo; c <= hi; ++c) {
+        mask[c] = true;
+    }
+    return true;
+}
+
+int multi_asr_parse_cpu_range(const std::string & range, bool (&mask)[GGML_MAX_N_THREADS]) {
+    std::memset(mask, 0, sizeof(bool) * GGML_MAX_N_THREADS);
+    if (range.empty()) {
+        return 0;
+    }
+
+    // Split on commas; each part is a single cpu or a dash range.
+    std::stringstream ss(range);
+    std::string       part;
+    while (std::getline(ss, part, ',')) {
+        // trim ASCII whitespace
+        size_t b = 0;
+        size_t e = part.size();
+        while (b < e && std::isspace((unsigned char) part[b])) {
+            ++b;
+        }
+        while (e > b && std::isspace((unsigned char) part[e - 1])) {
+            --e;
+        }
+        const std::string seg = part.substr(b, e - b);
+        if (seg.empty()) {
+            continue;
+        }
+        if (!parse_one_segment(seg, mask)) {
+            return -1;
+        }
+    }
+
+    int count = 0;
+    for (int i = 0; i < GGML_MAX_N_THREADS; ++i) {
+        count += mask[i] ? 1 : 0;
+    }
+    return count;
+}
+
+std::string multi_asr_cpu_mask_to_string(const bool (&mask)[GGML_MAX_N_THREADS]) {
+    std::string out;
+    for (int i = 0; i < GGML_MAX_N_THREADS; ++i) {
+        if (mask[i]) {
+            if (!out.empty()) {
+                out += ",";
+            }
+            out += std::to_string(i);
+        }
+    }
+    return out;
+}