diff --git a/.ci/scripts/BUCK b/.ci/scripts/BUCK new file mode 100644 index 00000000000..ea724a7f29f --- /dev/null +++ b/.ci/scripts/BUCK @@ -0,0 +1,13 @@ +load("@fbsource//tools/build_defs:fb_native_wrapper.bzl", "fb_native") + +oncall("executorch") + +# CMake builds this CI-only tool; group its sources without a product target. +fb_native.filegroup( + name = "muse_glimmer_batching_smoke_sources", + srcs = [ + "muse_glimmer_batching_smoke.cpp", + "muse_glimmer_batching_workload.h", + "tests/muse_glimmer_batching_workload_test.cpp", + ], +) diff --git a/.ci/scripts/build_muse_glimmer_batching_smoke.sh b/.ci/scripts/build_muse_glimmer_batching_smoke.sh new file mode 100644 index 00000000000..e990842c122 --- /dev/null +++ b/.ci/scripts/build_muse_glimmer_batching_smoke.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Run after the normal batching build; reuse its backend and package settings. +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +BUILD_DIR="${1:-${ROOT}/cmake-out/examples/models/muse-glimmer-batching}" +cmake -S "${ROOT}/examples/models/muse-glimmer" -B "$BUILD_DIR" \ + -DMUSE_GLIMMER_BUILD_BATCHING=ON -DMUSE_GLIMMER_BUILD_CI_SMOKE=ON +cmake --build "$BUILD_DIR" --target muse_glimmer_batching_smoke --parallel diff --git a/.ci/scripts/export_model_artifact.sh b/.ci/scripts/export_model_artifact.sh index 223dd82951c..b23165c23e5 100755 --- a/.ci/scripts/export_model_artifact.sh +++ b/.ci/scripts/export_model_artifact.sh @@ -44,6 +44,7 @@ Arguments: - vr-streaming: Voxtral Realtime streaming mode - vr-offline: Voxtral Realtime offline mode - solo-text: Muse Glimmer solo text mode + - solo-text-batching: Muse Glimmer text on the CUDA batching runtime - dflash-image: Muse Glimmer DFlash vision mode Examples: @@ -95,7 +96,7 @@ if [ -n "$MODE" ]; then exit 1 fi ;; - solo-text|dflash-image) + solo-text|solo-text-batching|dflash-image) if [ "$HF_MODEL" != "meta-models/Muse-Glimmer-30B-GGUF" ]; then echo "Error: Mode '$MODE' can only be used with Muse Glimmer model" echo "Provided model: $HF_MODEL" @@ -104,7 +105,7 @@ if [ -n "$MODE" ]; then ;; *) echo "Error: Unsupported mode '$MODE'" - echo "Supported modes: vr-streaming, vr-offline, solo-text, dflash-image" + echo "Supported modes: vr-streaming, vr-offline, solo-text, solo-text-batching, dflash-image" exit 1 ;; esac @@ -288,8 +289,8 @@ if [ "$MODEL_NAME" = "muse_glimmer" ]; then echo "Error: Muse Glimmer requires quantization 'kquant-17gb' or 'kquant-dynamic'" exit 1 fi - if [ "$MODE" != "solo-text" ] && [ "$MODE" != "dflash-image" ]; then - echo "Error: Muse Glimmer requires mode 'solo-text' or 'dflash-image'" + if [ "$MODE" != "solo-text" ] && [ "$MODE" != "solo-text-batching" ] && [ "$MODE" != "dflash-image" ]; then + echo "Error: Muse Glimmer requires mode 'solo-text', 'solo-text-batching' or 'dflash-image'" exit 1 fi fi @@ -565,6 +566,18 @@ if [ "$MODEL_NAME" = "muse_glimmer" ]; then --backend cuda \ --output-dir "${OUTPUT_DIR}" ;; + solo-text-batching) + # The off-graph cache exports the batching methods CudaExecutor drives: + # Static forward_{N}, dynamic forward_others and the device samplers. + EXPORT_START_SECONDS=$SECONDS + TMPDIR="$INDUCTOR_TMPDIR" \ + TORCHINDUCTOR_CACHE_DIR="$INDUCTOR_CACHE" \ + python -m executorch.examples.models.muse_glimmer.export.export_solo \ + --gguf "$TARGET_GGUF_PATH" \ + --backend cuda \ + --use-offgraph-kv-cache \ + --output-dir "${OUTPUT_DIR}" + ;; dflash-image) DRAFT_GGUF_FILE="dflash-Muse-Glimmer-30B-Q4_K_M.gguf" MMPROJ_GGUF_FILE="mmproj-Muse-Glimmer-30B-Q4_K_M.gguf" @@ -581,7 +594,7 @@ if [ "$MODEL_NAME" = "muse_glimmer" ]; then --output-dir "${OUTPUT_DIR}" ;; *) - echo "Error: Muse Glimmer requires mode 'solo-text' or 'dflash-image'" + echo "Error: Muse Glimmer requires mode 'solo-text', 'solo-text-batching' or 'dflash-image'" exit 1 ;; esac diff --git a/.ci/scripts/muse_glimmer_batching_smoke.cpp b/.ci/scripts/muse_glimmer_batching_smoke.cpp new file mode 100644 index 00000000000..6b9c794c706 --- /dev/null +++ b/.ci/scripts/muse_glimmer_batching_smoke.cpp @@ -0,0 +1,101 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#include "muse_glimmer_batching_workload.h" + +#include +#include +#include + +#include +#include +#include +#include + +DECLARE_string(pte); +DECLARE_string(tokenizer); +DECLARE_string(data_path); +DECLARE_string(backend); +DECLARE_int32(max_sessions); +DECLARE_int32(max_session_tokens); +DECLARE_uint64(bos_id); + +DEFINE_string(mode, "requests", "Correctness workload: requests"); +DEFINE_int32(requests, 8, "Number of concurrent requests"); +DEFINE_int32(prompt_tokens, 512, "Prompt positions including BOS per request"); +DEFINE_int32(max_new_tokens, 128, "Exact completion budget per request"); +DEFINE_int32(stagger_ms, 0, "Delay between request submissions"); +DEFINE_int32(max_decode, 32, "Maximum decodes admitted per step"); +DEFINE_int32(timeout_s, 600, "Request workload deadline"); + +int main(int argc, char** argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + try { + namespace llm = ::executorch::extension::llm; + namespace ci = muse_glimmer_ci; + if (FLAGS_mode != "requests" || FLAGS_pte.empty() || + FLAGS_tokenizer.empty() || FLAGS_max_sessions <= 0 || + FLAGS_max_session_tokens < 0) { + throw std::invalid_argument( + "invalid model or correctness workload flags"); + } + const ci::RequestOptions options{ + FLAGS_requests, + FLAGS_prompt_tokens, + FLAGS_max_new_tokens, + FLAGS_stagger_ms, + FLAGS_max_decode, + FLAGS_timeout_s}; + // Reject impossible workloads before loading tokenizer or model weights. + ci::validate_requests( + options, FLAGS_max_sessions, FLAGS_max_session_tokens); + tokenizers::HFTokenizer tokenizer; + if (tokenizer.load(FLAGS_tokenizer) != tokenizers::Error::Ok) { + throw std::runtime_error("could not load the tokenizer"); + } + llm::MuseGlimmerBackendConfig config; + config.backend = FLAGS_backend; + config.model_path = FLAGS_pte; + config.data_path = FLAGS_data_path; + config.max_sessions = FLAGS_max_sessions; + config.max_session_tokens = FLAGS_max_session_tokens; + config.bos_id = FLAGS_bos_id; + auto backend = llm::create_muse_glimmer_backend(config); + if (!backend.ok()) { + throw std::runtime_error( + "could not create the executor: error " + + std::to_string(static_cast(backend.error()))); + } + ci::EvidenceExecutor executor(*backend->executor); + llm::serving::ServingRuntimeConfig serving; + serving.max_sessions = options.requests; + serving.max_requests = options.requests; + serving.max_pending_operations = options.requests; + serving.max_context_length = backend->preparation->max_context_length; + auto result = ci::run_requests( + options, + executor, + tokenizer, + serving, + [spec = backend->preparation]( + const llm::serving::PromptPreparationContext& context, + llm::serving::PromptInput input) { + return llm::prepare_muse_glimmer_input( + context, std::move(input), spec); + }); + const auto evidence = executor.evidence(); + ci::check_batching(evidence); + std::cout << "Batching correctness passed: " << result.accepted + << " requests completed with exact token budgets; " + << evidence.multi_session_steps + << " successful multi-session executor steps\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "batching smoke failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/.ci/scripts/muse_glimmer_batching_workload.h b/.ci/scripts/muse_glimmer_batching_workload.h new file mode 100644 index 00000000000..33f25418cb9 --- /dev/null +++ b/.ci/scripts/muse_glimmer_batching_workload.h @@ -0,0 +1,365 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Private CI workload control, shared by correctness and measurement tools. +namespace muse_glimmer_ci { +namespace batching = ::executorch::extension::llm::batching; +namespace serving = ::executorch::extension::llm::serving; + +struct RequestOptions { + std::int32_t requests = 8; + std::int32_t prompt_tokens = 512; + std::int32_t max_new_tokens = 128; + std::int32_t stagger_ms = 0; + std::int32_t max_decode = 32; + std::int32_t timeout_s = 600; +}; + +inline void check_session_capacity( + std::size_t required, + std::size_t available) { + if (available == 0 || required == 0 || required > available) { + throw std::invalid_argument("workload exceeds --max_sessions"); + } +} + +inline void validate_requests( + const RequestOptions& options, + std::size_t max_sessions, + std::size_t max_context_length = 0) { + if (options.requests <= 0 || options.prompt_tokens <= 0 || + options.max_new_tokens <= 0 || options.stagger_ms < 0 || + options.max_decode <= 0 || options.timeout_s <= 0) { + throw std::invalid_argument("invalid request workload options"); + } + check_session_capacity(options.requests, max_sessions); + const auto tokens = static_cast(options.prompt_tokens) + + static_cast(options.max_new_tokens); + if (max_context_length > 0 && tokens > max_context_length) { + throw std::invalid_argument("request token budget exceeds context"); + } +} + +inline std::vector prompt_ids( + std::size_t length, + std::size_t salt) { + std::vector ids(length); + for (std::size_t i = 0; i < length; ++i) { + ids[i] = 1000 + (i * 7919 + salt * 104729) % 99000; + } + return ids; +} + +struct BatchEvidence { + std::size_t max_sessions = 0; + std::size_t multi_session_steps = 0; + std::size_t mixed_steps = 0; + + void observe(const batching::BatchInput& batch) { + std::set sessions; + std::size_t decodes = 0; + for (const auto& input : batch.inputs) { + sessions.insert(input.sid); + decodes += input.size == 1 && input.produce_output && + std::holds_alternative(input.payload); + } + max_sessions = std::max(max_sessions, sessions.size()); + if (sessions.size() > 1) { + ++multi_session_steps; + if (decodes > 0 && batch.size() > decodes) { + ++mixed_steps; + } + } + } +}; + +// Counts distinct owners only after the forwarded executor call succeeds. +class EvidenceExecutor final : public batching::Executor { + public: + explicit EvidenceExecutor(batching::Executor& inner) : inner_(inner) {} + + bool initialize() override { + return inner_.initialize(); + } + std::size_t preferred_batch_tokens() const override { + return inner_.preferred_batch_tokens(); + } + std::optional open_session() override { + return inner_.open_session(); + } + void close_session(batching::SessionId session) override { + inner_.close_session(session); + } + std::optional clone( + batching::SessionId source, + batching::Position upto) override { + return inner_.clone(source, upto); + } + void set_sampling( + batching::SessionId session, + const batching::SamplingParams& params, + std::optional seed) override { + inner_.set_sampling(session, params, seed); + } + bool accepts(const batching::PreparedInput& input) const override { + return inner_.accepts(input); + } + bool execute(const batching::BatchInput& batch, batching::BatchOutput& out) + override { + if (!inner_.execute(batch, out)) { + return false; + } + std::lock_guard lock(mutex_); + evidence_.observe(batch); + return true; + } + BatchEvidence evidence() const { + std::lock_guard lock(mutex_); + return evidence_; + } + + private: + batching::Executor& inner_; + mutable std::mutex mutex_; + BatchEvidence evidence_; +}; + +inline void check_batching(const BatchEvidence& evidence) { + if (evidence.max_sessions <= 1 || evidence.multi_session_steps == 0) { + throw std::runtime_error("no successful multi-session executor step"); + } +} + +inline std::unique_ptr make_runtime( + batching::Executor& executor, + std::unique_ptr scheduler, + const tokenizers::Tokenizer& tokenizer, + serving::ServingRuntimeConfig config, + serving::ModelPreparer preparer) { + if (!scheduler) { + throw std::invalid_argument("unusable scheduler limits"); + } + return std::make_unique( + executor, std::move(scheduler), tokenizer, config, std::move(preparer)); +} + +struct Inflight { + std::mutex mutex; + std::condition_variable changed; + std::size_t running = 0; + std::size_t failed = 0; + std::string error; + + void finished(const serving::TerminalEvent& terminal) { + { + std::lock_guard lock(mutex); + --running; + if (terminal.finish_reason == serving::FinishReason::Failed) { + ++failed; + if (terminal.error && error.empty()) { + error = terminal.error->message; + } + } + } + changed.notify_all(); + } +}; + +inline std::optional submit_request( + serving::ServingRuntime& runtime, + Inflight& inflight, + std::size_t length, + std::size_t salt, + std::int32_t new_tokens, + std::function on_terminal = {}, + std::function on_text = {}) { + if (length == 0 || new_tokens <= 0) { + throw std::invalid_argument("invalid prompt or generation budget"); + } + serving::PromptInput prompt; + // Model preparation adds BOS, which is included in the requested length. + prompt.segments.emplace_back(prompt_ids(length - 1, salt)); + serving::GenerationOptions options; + options.max_new_tokens = new_tokens; + options.sampling.temperature = 0.0f; + options.seed = salt; + { + std::lock_guard lock(inflight.mutex); + ++inflight.running; + } + auto result = runtime.generate( + std::nullopt, + std::move(prompt), + options, + [&inflight, on_terminal, on_text](serving::GenerationEvent event) { + if (const auto* text = std::get_if(&event); + text && !text->text.empty() && on_text) { + on_text(); + } + if (const auto* terminal = + std::get_if(&event)) { + if (on_terminal) { + on_terminal(*terminal); + } + inflight.finished(*terminal); + } + }); + if (const auto* error = std::get_if(&result)) { + std::lock_guard lock(inflight.mutex); + --inflight.running; + ++inflight.failed; + inflight.error = error->message; + return std::nullopt; + } + return std::get(std::move(result)); +} + +struct RequestResult { + std::size_t accepted = 0; + std::vector> terminals; +}; + +inline void check_requests_complete( + const RequestOptions& options, + const RequestResult& result) { + if (result.accepted != static_cast(options.requests) || + result.terminals.size() != result.accepted) { + throw std::runtime_error("not all workload requests were accepted"); + } + const auto incomplete = + [&](const std::optional& terminal) { + return !terminal || + terminal->finish_reason != serving::FinishReason::Length || + terminal->stats.prompt_tokens != + static_cast(options.prompt_tokens) || + terminal->stats.completion_tokens != + static_cast(options.max_new_tokens); + }; + if (std::any_of( + result.terminals.begin(), result.terminals.end(), incomplete)) { + throw std::runtime_error("request failed or token budget incomplete"); + } +} + +// Submission runs on the caller; text/terminal hooks run on delivery. Captured +// state must outlive run_requests. Hooks must be bounded and must not throw. +struct RequestCallbacks { + std::function submitted; + std::function text; + std::function terminal; +}; + +inline RequestResult run_requests( + const RequestOptions& options, + batching::Executor& executor, + const tokenizers::Tokenizer& tokenizer, + serving::ServingRuntimeConfig config, + serving::ModelPreparer preparer, + const RequestCallbacks& callbacks = {}) { + validate_requests(options, config.max_sessions, config.max_context_length); + check_session_capacity(options.requests, config.max_requests); + const auto decodes = + static_cast(std::min(options.max_decode, options.requests)); + const auto budget = std::max(executor.preferred_batch_tokens(), decodes + 1); + auto scheduler = + batching::DecodeFirstScheduler::create(budget, decodes, budget - decodes); + config.default_stop_tokens = {}; + RequestResult result; + result.terminals.resize(options.requests); + Inflight inflight; + std::vector handles; + // Runtime destruction fences callbacks before their captured state dies. + auto runtime = make_runtime( + executor, std::move(scheduler), tokenizer, config, std::move(preparer)); + using Clock = std::chrono::steady_clock; + const auto deadline = Clock::now() + std::chrono::seconds(options.timeout_s); + bool timed_out = false; + for (std::size_t i = 0; i < result.terminals.size(); ++i) { + if (i > 0 && options.stagger_ms > 0) { + std::this_thread::sleep_until(std::min( + deadline, + Clock::now() + std::chrono::milliseconds(options.stagger_ms))); + } + if (Clock::now() >= deadline) { + timed_out = true; + break; + } + if (callbacks.submitted) { + callbacks.submitted(i); + } + auto handle = submit_request( + *runtime, + inflight, + options.prompt_tokens, + i, + options.max_new_tokens, + [&result, &callbacks, i](const serving::TerminalEvent& terminal) { + result.terminals[i] = terminal; + if (callbacks.terminal) { + callbacks.terminal(i, terminal); + } + }, + [&callbacks, i] { + if (callbacks.text) { + callbacks.text(i); + } + }); + if (!handle) { + break; + } + ++result.accepted; + handles.push_back(std::move(*handle)); + } + { + std::unique_lock lock(inflight.mutex); + timed_out = timed_out || Clock::now() >= deadline || + !inflight.changed.wait_until(lock, deadline, [&] { + return inflight.running == 0 || inflight.failed > 0; + }); + if (timed_out || inflight.failed > 0 || + result.accepted != result.terminals.size()) { + lock.unlock(); + for (const auto& handle : handles) { + handle.cancel(); + } + runtime->shutdown(); + } + } + for (const auto& handle : handles) { + handle.wait(); + } + runtime->shutdown(); + if (timed_out) { + throw std::runtime_error("request workload timed out"); + } + check_requests_complete(options, result); + return result; +} + +} // namespace muse_glimmer_ci diff --git a/.ci/scripts/test_model_e2e.sh b/.ci/scripts/test_model_e2e.sh index 427329ae961..c6a2131c004 100755 --- a/.ci/scripts/test_model_e2e.sh +++ b/.ci/scripts/test_model_e2e.sh @@ -46,6 +46,7 @@ Arguments: - vr-streaming: Voxtral Realtime streaming mode - vr-offline: Voxtral Realtime offline mode - solo-text: Muse Glimmer solo text mode + - solo-text-batching: Muse Glimmer text on the CUDA batching runtime - dflash-image: Muse Glimmer DFlash vision mode Environment: @@ -111,7 +112,7 @@ if [ -n "$MODE" ]; then exit 1 fi ;; - solo-text|dflash-image) + solo-text|solo-text-batching|dflash-image) if [ "$HF_MODEL" != "meta-models/Muse-Glimmer-30B-GGUF" ]; then echo "Error: Mode '$MODE' can only be used with Muse Glimmer model" echo "Provided model: $HF_MODEL" @@ -120,7 +121,7 @@ if [ -n "$MODE" ]; then ;; *) echo "Error: Unsupported mode '$MODE'" - echo "Supported modes: vr-streaming, vr-offline, solo-text, dflash-image" + echo "Supported modes: vr-streaming, vr-offline, solo-text, solo-text-batching, dflash-image" exit 1 ;; esac @@ -290,13 +291,20 @@ case "$HF_MODEL" in EXPECTED_OUTPUT="Paris" IMAGE_URL="" ;; + solo-text-batching) + # Built by `make muse-glimmer-batching-cuda` into its own directory. + RUNNER_PATH="muse-glimmer-batching" + RUNNER_TARGET="muse_glimmer_batching_runner" + EXPECTED_OUTPUT="Paris" + IMAGE_URL="" + ;; dflash-image) RUNNER_TARGET="dflash_runner" EXPECTED_OUTPUT="dog" IMAGE_URL="https://github.com/pytorch/hub/raw/master/images/dog.jpg" ;; *) - echo "Error: Muse Glimmer requires mode 'solo-text' or 'dflash-image'" + echo "Error: Muse Glimmer requires mode 'solo-text', 'solo-text-batching' or 'dflash-image'" exit 1 ;; esac @@ -453,6 +461,11 @@ EOF fi RUNNER_ARGS="$RUNNER_ARGS --tokenizer_path ${MODEL_DIR}/$TOKENIZER_FILE --prompt_file \"$PROMPT_FILE\" --max_new_tokens 512" RUNNER_ARGS="$RUNNER_ARGS --cuda_graph" + if [ "$MODE" = "solo-text-batching" ]; then + # The batching runner goes through ServingRuntime and takes its own flags. + printf '%s' '<|start|>user<|message|>What is the capital of France?<|eot|><|start|>assistant' > "$PROMPT_FILE" + RUNNER_ARGS="--pte ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer ${MODEL_DIR}/$TOKENIZER_FILE --prompt \"\$(cat $PROMPT_FILE)\" --max_new_tokens 128 --temperature 0 --max_session_tokens 4096" + fi if [ "$MODE" = "dflash-image" ]; then RUNNER_ARGS="$RUNNER_ARGS --image_path ${MODEL_DIR}/test_image.jpg" fi @@ -540,6 +553,121 @@ else fi echo "::endgroup::" +if [ "$DEVICE" = "cuda" ] && [ "$MODEL_NAME" = "muse_glimmer" ] && [ "$MODE" = "solo-text-batching" ]; then + echo "::group::Run $MODEL_NAME batching serving smoke" + # The CI-only correctness tool is not part of the normal serving build. + bash .ci/scripts/build_muse_glimmer_batching_smoke.sh + # Concurrent staggered requests through ServingRuntime on CudaExecutor. + cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_smoke \ + --pte "${MODEL_DIR}/model.pte" --data_path "${MODEL_DIR}/aoti_cuda_blob.ptd" \ + --tokenizer "${MODEL_DIR}/$TOKENIZER_FILE" --max_sessions 8 --max_session_tokens 2048 \ + --mode requests --requests 5 --prompt_tokens 600 --max_new_tokens 64 --stagger_ms 200 + + pip install -r examples/llm_server/python/requirements.txt + python -c "from huggingface_hub import hf_hub_download; [hf_hub_download('${HF_MODEL}', f, local_dir='${MODEL_DIR}') for f in ('tokenizer_config.json', 'chat_template.jinja')]" + PORT=$(python - <<'PY' +import socket + +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + print(s.getsockname()[1]) +PY +) + SERVER_LOG=$(mktemp) + python -u -m executorch.examples.models.muse_glimmer.serving.serve_batching \ + --backend cuda \ + --worker-bin cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_worker \ + --model-path "${MODEL_DIR}/model.pte" \ + --data-path "${MODEL_DIR}/aoti_cuda_blob.ptd" \ + --tokenizer-path "${MODEL_DIR}/$TOKENIZER_FILE" \ + --hf-tokenizer "${MODEL_DIR}" \ + --max-context 4096 \ + --prefix-cache-entries 2 \ + --host 127.0.0.1 \ + --port "$PORT" >"$SERVER_LOG" 2>&1 & + SERVER_PID=$! + cleanup_mg_server() { + if kill -0 "$SERVER_PID" 2>/dev/null; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + rm -f "$SERVER_LOG" + } + trap cleanup_mg_server EXIT + + if ! python - "$PORT" "$SERVER_LOG" <<'PY' +import json +import sys +import time +import urllib.request + +port, log_path = sys.argv[1], sys.argv[2] +base = f"http://127.0.0.1:{port}" + + +def request(path, payload=None): + data = None + headers = {} + if payload is not None: + data = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request(base + path, data=data, headers=headers) + with urllib.request.urlopen(req, timeout=300) as resp: + return resp.read().decode("utf-8") + + +last = None +for _ in range(600): + try: + request("/health") + break + except Exception as e: + last = e + time.sleep(1) +else: + print(open(log_path, encoding="utf-8", errors="replace").read()) + raise RuntimeError(f"server did not become healthy: {last}") + +body = { + "model": "muse_glimmer", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "max_tokens": 128, + "temperature": 0, +} +# The second identical request reuses the first one's prompt snapshot. +for attempt in range(2): + resp = json.loads(request("/v1/chat/completions", body)) + message = resp["choices"][0]["message"] + text = (message.get("content") or "") + (message.get("reasoning_content") or "") + if "Paris" not in text: + raise AssertionError(f"expected Paris in serving response, got: {message!r}") + details = resp["usage"].get("prompt_tokens_details") or {} + cached = details.get("cached_tokens", 0) + print(f"request {attempt}: prompt {resp['usage']['prompt_tokens']} cached {cached}") +if cached == 0: + raise AssertionError("the repeated request reused no prompt tokens") + +stream = request("/v1/chat/completions", dict(body, stream=True)) +pieces = [] +for line in stream.splitlines(): + if line.startswith("data: ") and line != "data: [DONE]": + delta = json.loads(line[6:])["choices"][0]["delta"] + pieces.append((delta.get("content") or "") + (delta.get("reasoning_content") or "")) +if len([p for p in pieces if p]) < 2 or "Paris" not in "".join(pieces): + raise AssertionError(f"expected Paris streamed in several chunks, got: {pieces!r}") +print("Muse Glimmer CUDA batching serving smoke passed") +PY + then + echo "Muse Glimmer batching serving smoke failed; server log:" + cat "$SERVER_LOG" + exit 1 + fi + + cleanup_mg_server + trap - EXIT + echo "::endgroup::" +fi + if [ "$DEVICE" = "cuda" ] && [ "$MODEL_NAME" = "qwen3_5_moe" ]; then echo "::group::Run $MODEL_NAME OpenAI serving smoke" pip install -r examples/llm_server/python/requirements.txt "transformers==5.0.0rc1" diff --git a/.ci/scripts/tests/muse_glimmer_batching_workload_test.cpp b/.ci/scripts/tests/muse_glimmer_batching_workload_test.cpp new file mode 100644 index 00000000000..94ba8ed639b --- /dev/null +++ b/.ci/scripts/tests/muse_glimmer_batching_workload_test.cpp @@ -0,0 +1,160 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#include "../muse_glimmer_batching_workload.h" + +#include +#include + +namespace ci = muse_glimmer_ci; +namespace batching = ci::batching; +namespace serving = ci::serving; + +namespace { +void require(bool condition) { + if (!condition) { + throw std::runtime_error("CI workload regression failed"); + } +} + +template +void rejected(F action) { + bool failed = false; + try { + action(); + } catch (const std::exception&) { + failed = true; + } + require(failed); +} + +class FakeExecutor final : public batching::Executor { + public: + bool succeeds = true; + std::optional open_session() override { + return 1; + } + void close_session(batching::SessionId) override {} + void set_sampling( + batching::SessionId, + const batching::SamplingParams&, + std::optional) override {} + bool execute(const batching::BatchInput&, batching::BatchOutput&) override { + return succeeds; + } +}; + +batching::Input input(batching::SessionId session, std::size_t size = 1) { + return { + session, + true, + 0, + size, + std::make_shared>(size, 1000), + 0}; +} + +void workload_limits() { + ci::RequestOptions options{5, 600, 64, 200, 32, 600}; + ci::validate_requests(options, 8, 2048); + rejected([&] { ci::validate_requests(options, 4, 2048); }); + rejected([&] { ci::validate_requests(options, 8, 663); }); + ci::validate_requests(options, 5, 664); + ci::validate_requests(options, 5); + rejected([] { ci::check_session_capacity(0, 5); }); + rejected([] { ci::check_session_capacity(1, 0); }); + for (auto member : + {&ci::RequestOptions::requests, + &ci::RequestOptions::prompt_tokens, + &ci::RequestOptions::max_new_tokens, + &ci::RequestOptions::max_decode, + &ci::RequestOptions::timeout_s}) { + auto invalid = options; + invalid.*member = 0; + rejected([&] { ci::validate_requests(invalid, 8); }); + } + options.stagger_ms = -1; + rejected([&] { ci::validate_requests(options, 8); }); + options = {1, std::numeric_limits::max(), 1, 0, 1, 1}; + rejected([&] { + ci::validate_requests(options, 1, std::numeric_limits::max()); + }); + const auto prompt = ci::prompt_ids(599, 3); + require(prompt.size() == 599 && prompt == ci::prompt_ids(599, 3)); + require(prompt != ci::prompt_ids(599, 4)); + require(ci::prompt_ids(0, 0).empty()); + for (const auto token : prompt) { + require(token >= 1000 && token < 100000); + } +} + +void successful_batch_owners() { + FakeExecutor inner; + ci::EvidenceExecutor executor(inner); + batching::BatchOutput out; + require(executor.execute({{input(1), input(1)}}, out)); + require(executor.execute({{input(2)}}, out)); + require(executor.evidence().max_sessions == 1); + rejected([&] { ci::check_batching(executor.evidence()); }); + inner.succeeds = false; + require(!executor.execute({{input(1), input(2)}}, out)); + require(executor.evidence().multi_session_steps == 0); + rejected([&] { ci::check_batching(executor.evidence()); }); + // cppcheck-suppress redundantAssignment + inner.succeeds = true; + require(executor.execute({{input(1), input(2)}}, out)); + ci::check_batching(executor.evidence()); + require(executor.evidence().multi_session_steps == 1); + require(executor.evidence().mixed_steps == 0); + require(executor.execute({{input(1), input(2, 4)}}, out)); + require(executor.evidence().mixed_steps == 1); +} + +void exact_completion() { + const ci::RequestOptions options{2, 600, 64, 0, 2, 1}; + serving::TerminalEvent terminal; + terminal.finish_reason = serving::FinishReason::Length; + terminal.stats.prompt_tokens = 600; + terminal.stats.completion_tokens = 64; + ci::RequestResult result{2, {terminal, terminal}}; + ci::check_requests_complete(options, result); + result.accepted = 1; + rejected([&] { ci::check_requests_complete(options, result); }); + result.accepted = 2; + result.terminals[1].reset(); + rejected([&] { ci::check_requests_complete(options, result); }); + for (const auto reason : + {serving::FinishReason::Failed, + serving::FinishReason::Stop, + serving::FinishReason::Cancelled}) { + result.terminals[1] = terminal; + result.terminals[1]->finish_reason = reason; + rejected([&] { ci::check_requests_complete(options, result); }); + } + for (const auto count : {63, 65}) { + result.terminals[1] = terminal; + result.terminals[1]->stats.completion_tokens = count; + rejected([&] { ci::check_requests_complete(options, result); }); + } + result.terminals[1] = terminal; + result.terminals[1]->stats.prompt_tokens = 599; + rejected([&] { ci::check_requests_complete(options, result); }); +} +} // namespace + +int main() { + try { + workload_limits(); + successful_batch_owners(); + exact_completion(); + std::cout << "CI batching workload tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/.ci/scripts/tests/test_muse_glimmer_batching_ci.py b/.ci/scripts/tests/test_muse_glimmer_batching_ci.py new file mode 100644 index 00000000000..5c8fcead860 --- /dev/null +++ b/.ci/scripts/tests/test_muse_glimmer_batching_ci.py @@ -0,0 +1,144 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import re +import subprocess +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPTS = ROOT / ".ci/scripts" +CORRECTNESS_PATHS = ( + ".ci/scripts/build_muse_glimmer_batching_smoke.sh", + ".ci/scripts/muse_glimmer_batching_smoke.cpp", + ".ci/scripts/muse_glimmer_batching_workload.h", + ".ci/scripts/tests/test_muse_glimmer_batching_ci.py", + ".ci/scripts/tests/test_muse_glimmer_batching_workload.py", + ".ci/scripts/tests/muse_glimmer_batching_workload_test.cpp", +) + + +class MuseGlimmerBatchingCITest(unittest.TestCase): + def test_e2e_builds_smoke_before_running_it(self): + script = (SCRIPTS / "test_model_e2e.sh").read_text() + build = script.index("bash .ci/scripts/build_muse_glimmer_batching_smoke.sh") + run = script.index( + "cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_smoke" + ) + self.assertLess(build, run) + # A separate measurement pass may follow this correctness command at tip. + lines = [] + for line in script[run:].splitlines(): + lines.append(line) + if not line.endswith("\\"): + break + smoke = "\n".join(lines) + for flag in ("--requests 5", "--prompt_tokens 600", "--max_new_tokens 64"): + self.assertIn(flag, smoke) + self.assertIn("--stagger_ms 200", smoke) + self.assertNotIn("bench", smoke) + self.assertNotIn("report_json", smoke) + self.assertNotIn("tokens_per_s", smoke) + + def test_workflow_selects_correctness_tests_and_smoke(self): + workflow = (ROOT / ".github/workflows/cuda.yml").read_text() + unit_job = workflow.split(" unittest-cuda-muse-glimmer-batching:", 1)[1].split( + " unittest-cuda-quantized-gemm:", 1 + )[0] + self.assertIn("bash .ci/scripts/build_muse_glimmer_batching_smoke.sh", unit_job) + self.assertIn( + 'test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_smoke"', unit_job + ) + self.assertIn("-p 'test_muse_glimmer_batching_ci.py'", unit_job) + self.assertIn("-p 'test_muse_glimmer_batching_workload.py'", unit_job) + self.assertNotIn("test_muse_glimmer_batching_*.py", unit_job) + filters = workflow.split(" pull_request:\n", 1)[1].split( + " workflow_dispatch:", 1 + )[0] + e2e_job = workflow.split(" test-muse-glimmer-cuda-e2e:", 1)[1] + for path in CORRECTNESS_PATHS: + self.assertIn(f" - {path}\n", filters) + self.assertIn(f"'{path}'", unit_job) + self.assertIn(f"'{path}'", e2e_job) + self.assertNotIn(".ci/scripts/*muse_glimmer_batching*", filters) + self.assertNotIn(".ci/scripts/tests/*muse_glimmer_batching*", filters) + for prefix in ( + ".ci/scripts/muse_glimmer_batching", + ".ci/scripts/build_muse_glimmer_batching", + ".ci/scripts/tests/test_muse_glimmer_batching", + ".ci/scripts/tests/muse_glimmer_batching", + ): + self.assertNotIn(f"'{prefix}'", unit_job + e2e_job) + + def test_smoke_uses_shared_runtime_and_has_no_measurement_features(self): + source = (SCRIPTS / "muse_glimmer_batching_smoke.cpp").read_text() + helper = (SCRIPTS / "muse_glimmer_batching_workload.h").read_text() + self.assertIn("create_muse_glimmer_backend(config)", source) + self.assertIn("ci::run_requests(", source) + self.assertIn("ci::check_batching(evidence)", source) + self.assertIn("std::make_unique", helper) + self.assertIn("runtime.generate(", helper) + self.assertIn("config.default_stop_tokens = {};", helper) + self.assertIn("runtime->shutdown();", helper) + self.assertIn("handle.wait();", helper) + self.assertIn("check_requests_complete(options, result)", helper) + for forbidden in ( + "bench_utils", + "SweepShape", + "TimedExecutor", + "RequestTiming", + "report_json", + "nlohmann", + "tokens_per_s", + "ttft", + "tpot", + "prefill_ms", + "decode_ms", + "total_ms", + "VRAM", + ): + self.assertNotIn(forbidden, source + helper) + self.assertLess( + source.index("ci::validate_requests("), source.index("tokenizer.load(") + ) + + def test_smoke_cmake_is_opt_in_and_reuses_runtime(self): + project = ROOT if (ROOT / "examples").is_dir() else ROOT.parent + cmake = (project / "examples/models/muse-glimmer/CMakeLists.txt").read_text() + self.assertRegex( + cmake, + r'option\(MUSE_GLIMMER_BUILD_CI_SMOKE\s+"[^"]+"\s+OFF\s*\)', + ) + self.assertIn( + "if(MUSE_GLIMMER_BUILD_CI_SMOKE AND NOT MUSE_GLIMMER_BUILD_BATCHING)", cmake + ) + smoke = cmake.split(" if(MUSE_GLIMMER_BUILD_CI_SMOKE)\n", 1)[1].split( + "\n endif()", 1 + )[0] + self.assertIn( + "muse_glimmer_batching_smoke PRIVATE muse_glimmer_batching_runtime", smoke + ) + self.assertIn("LINKER:--export-dynamic", smoke) + self.assertIn( + "executorch_target_copy_mlx_metallib(muse_glimmer_batching_smoke)", smoke + ) + self.assertNotIn("BENCHMARK", smoke) + self.assertNotIn("batching_bench", smoke) + + def test_owned_shell_and_unit_workflow_syntax(self): + for script in ("test_model_e2e.sh", "build_muse_glimmer_batching_smoke.sh"): + result = subprocess.run(["bash", "-n", str(SCRIPTS / script)]) + self.assertEqual(result.returncode, 0) + workflow = (ROOT / ".github/workflows/cuda.yml").read_text() + unit_job = workflow.split(" unittest-cuda-muse-glimmer-batching:", 1)[1].split( + " unittest-cuda-quantized-gemm:", 1 + )[0] + body = unit_job.split(" script: |\n", 1)[1] + shell = "\n".join(line[8:] for line in body.splitlines()) + shell = re.sub(r"\$\{\{[^}]+\}\}", "ci-value", shell) + result = subprocess.run(["bash", "-n"], input=shell, text=True) + self.assertEqual(result.returncode, 0) diff --git a/.ci/scripts/tests/test_muse_glimmer_batching_workload.py b/.ci/scripts/tests/test_muse_glimmer_batching_workload.py new file mode 100644 index 00000000000..3951b37a9c8 --- /dev/null +++ b/.ci/scripts/tests/test_muse_glimmer_batching_workload.py @@ -0,0 +1,106 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SCRIPTS = Path(__file__).resolve().parents[1] + + +class MuseGlimmerBatchingWorkloadTest(unittest.TestCase): + def test_workload_without_a_model(self): + compiler = shutil.which("c++") + self.assertIsNotNone(compiler, "CI workload tests require a C++ compiler") + root = SCRIPTS.parents[1] + project = root if (root / "examples").is_dir() else root.parent + with tempfile.TemporaryDirectory() as directory: + temporary = Path(directory) + binary = temporary / "workload_test" + # Only the unused multimodal/tensor boundary is stubbed. Compile the + # real serving declarations, executor and private workload helper. + multimodal = ( + temporary / "executorch/extension/llm/runner/multimodal_input.h" + ) + multimodal.parent.mkdir(parents=True) + multimodal.write_text( + "#pragma once\n#include \n#include \n" + "namespace executorch::extension::llm {\n" + "struct MultimodalInput {\n" + " explicit MultimodalInput(std::vector) {}\n" + "};\n}\n" + ) + result = subprocess.run( + [ + compiler, + "-std=c++17", + "-Wall", + "-Wextra", + "-Werror", + "-Wno-deprecated-declarations", + "-pthread", + "-I", + str(temporary), + "-I", + str(project.parent), + str(SCRIPTS / "tests/muse_glimmer_batching_workload_test.cpp"), + "-o", + str(binary), + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + result = subprocess.run([str(binary)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("tests passed", result.stdout) + + def test_build_enables_smoke_and_propagates_failures(self): + for failure in ("none", "configure", "build"): + with self.subTest( + failure=failure + ), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + log = root / "commands" + cmake = root / "cmake" + cmake.write_text( + '#!/bin/bash\nprintf "%s\\n" "$*" >> "$COMMAND_LOG"\n' + 'if [[ "$1" == "--build" ]]; then\n' + ' [[ "$FAIL_AT" != "build" ]]\n' + 'else\n [[ "$FAIL_AT" != "configure" ]]\nfi\n' + ) + cmake.chmod(0o755) + build_dir = root / "build with spaces" + result = subprocess.run( + [ + "bash", + str(SCRIPTS / "build_muse_glimmer_batching_smoke.sh"), + str(build_dir), + ], + env={ + **os.environ, + "PATH": f"{root}:{os.environ['PATH']}", + "COMMAND_LOG": str(log), + "FAIL_AT": failure, + }, + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode == 0, failure == "none", result.stderr + ) + commands = log.read_text().splitlines() + self.assertIn("-DMUSE_GLIMMER_BUILD_CI_SMOKE=ON", commands[0]) + self.assertIn("-DMUSE_GLIMMER_BUILD_BATCHING=ON", commands[0]) + self.assertIn(str(build_dir), commands[0]) + self.assertNotIn("BENCHMARK", "\n".join(commands)) + self.assertEqual(len(commands), 1 if failure == "configure" else 2) + if len(commands) == 2: + self.assertIn("--target muse_glimmer_batching_smoke", commands[1]) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 63709510639..49cfa87559a 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -34,6 +34,12 @@ on: - .ci/scripts/cuda_benchmark.py - .ci/scripts/cuda_benchmark_from_e2e.sh - .ci/scripts/cuda_perf_prompts/** + - .ci/scripts/build_muse_glimmer_batching_smoke.sh + - .ci/scripts/muse_glimmer_batching_smoke.cpp + - .ci/scripts/muse_glimmer_batching_workload.h + - .ci/scripts/tests/test_muse_glimmer_batching_ci.py + - .ci/scripts/tests/test_muse_glimmer_batching_workload.py + - .ci/scripts/tests/muse_glimmer_batching_workload_test.cpp - examples/models/muse-glimmer/** - extension/pybindings/** - runtime/__init__.py @@ -556,6 +562,64 @@ jobs: pip install gguf python -m pytest examples/models/gemma4_31b/tests/ --ignore=examples/models/gemma4_31b/tests/test_mlx_pipeline.py -v -o "addopts=" + unittest-cuda-muse-glimmer-batching: + name: unittest-cuda-muse-glimmer-batching + needs: [changed-files, run-decision] + if: | + contains(needs.changed-files.outputs.changed-files, 'backends/cuda') || + contains(needs.changed-files.outputs.changed-files, 'backends/aoti') || + contains(needs.changed-files.outputs.changed-files, 'extension/llm/batching') || + contains(needs.changed-files.outputs.changed-files, 'extension/llm/serving') || + contains(needs.changed-files.outputs.changed-files, 'examples/llm_server/cpp') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/build_muse_glimmer_batching_smoke.sh') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/muse_glimmer_batching_smoke.cpp') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/muse_glimmer_batching_workload.h') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/test_muse_glimmer_batching_ci.py') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/test_muse_glimmer_batching_workload.py') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/muse_glimmer_batching_workload_test.cpp') || + contains(needs.changed-files.outputs.changed-files, 'examples/models/muse-glimmer') || + contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda.yml') || + needs.run-decision.outputs.is-full-run == 'true' + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read + with: + timeout: 120 + runner: mt-l-x86aavx2-11-41-a10g + gpu-arch-type: cuda + gpu-arch-version: "13.0" + submodules: recursive + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + script: | + set -eux + bash ./install_executorch.sh + conda install -y -c conda-forge 'libstdcxx-ng>=12' + export LD_LIBRARY_PATH=/opt/conda/lib:$LD_LIBRARY_PATH + + python -m unittest discover -s .ci/scripts/tests -p 'test_muse_glimmer_batching_ci.py' -v + python -m unittest discover -s .ci/scripts/tests -p 'test_muse_glimmer_batching_workload.py' -v + + # The normal serving binaries and the opt-in correctness tool share + # CudaExecutor; MG preparation tests also run on Linux. + make muse-glimmer-batching-cuda + BATCHING_BIN_DIR="${PWD}/cmake-out/examples/models/muse-glimmer-batching" + cmake -S "${PWD}/examples/models/muse-glimmer" -B "${BATCHING_BIN_DIR}" \ + -DMUSE_GLIMMER_BUILD_BATCHING_TESTS=ON + bash .ci/scripts/build_muse_glimmer_batching_smoke.sh "${BATCHING_BIN_DIR}" + cmake --build "${BATCHING_BIN_DIR}" --target muse_glimmer_batching_test --parallel + ctest --test-dir "${BATCHING_BIN_DIR}" \ + -R '^muse_glimmer_batching$' --output-on-failure --no-tests=error + test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_runner" + test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_worker" + test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_smoke" + + # The batching export on a tiny model, and the CLI's backend dispatch. + pip install gguf + python -m pytest -v -o "addopts=" \ + examples/models/muse-glimmer/tests/test_cuda_batching_pipeline.py \ + examples/models/muse-glimmer/tests/test_checkpoint_context.py + unittest-cuda-quantized-gemm: name: unittest-cuda-quantized-gemm (${{ matrix.shard }}) needs: [changed-files, run-decision] @@ -1044,6 +1108,12 @@ jobs: name: test-muse-glimmer-cuda-e2e-${{ matrix.variant }}-${{ matrix.mode }} needs: [changed-files, run-decision] if: | + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/build_muse_glimmer_batching_smoke.sh') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/muse_glimmer_batching_smoke.cpp') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/muse_glimmer_batching_workload.h') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/test_muse_glimmer_batching_ci.py') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/test_muse_glimmer_batching_workload.py') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/muse_glimmer_batching_workload_test.cpp') || contains(needs.changed-files.outputs.changed-files, 'backends/cuda') || contains(needs.changed-files.outputs.changed-files, 'backends/aoti') || contains(needs.changed-files.outputs.changed-files, 'extension/cuda') || @@ -1063,6 +1133,8 @@ jobs: include: - variant: kquant-17gb mode: solo-text + - variant: kquant-17gb + mode: solo-text-batching - variant: kquant-17gb mode: dflash-image - variant: kquant-dynamic diff --git a/.lintrunner.toml b/.lintrunner.toml index 89e5801db63..fab27e7a7c2 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -447,6 +447,7 @@ exclude_patterns = [ # Zephyr sample application can use std::vector in the sample code 'zephyr/samples/**', 'scripts/**', + '**/.ci/**', 'third-party/**', 'util/**', ] diff --git a/examples/models/muse-glimmer/CMakeLists.txt b/examples/models/muse-glimmer/CMakeLists.txt index 03b8f574b4d..70e24f3d61c 100644 --- a/examples/models/muse-glimmer/CMakeLists.txt +++ b/examples/models/muse-glimmer/CMakeLists.txt @@ -197,6 +197,12 @@ option(MUSE_GLIMMER_BUILD_BATCHING_TESTS "Build preparation/materialization tests without the batching executor" OFF ) +option(MUSE_GLIMMER_BUILD_CI_SMOKE + "Build the CI-only batching correctness workload" OFF +) +if(MUSE_GLIMMER_BUILD_CI_SMOKE AND NOT MUSE_GLIMMER_BUILD_BATCHING) + message(FATAL_ERROR "CI smoke requires MUSE_GLIMMER_BUILD_BATCHING=ON") +endif() if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS) if(EXECUTORCH_BUILD_CUDA) set(_batching_backend cuda) @@ -319,6 +325,24 @@ if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS) executorch_target_copy_mlx_metallib(muse_glimmer_batching_runner) executorch_target_copy_mlx_metallib(muse_glimmer_batching_worker) endif() + if(MUSE_GLIMMER_BUILD_CI_SMOKE) + add_executable( + muse_glimmer_batching_smoke + ${EXECUTORCH_ROOT}/.ci/scripts/muse_glimmer_batching_smoke.cpp + ) + target_link_libraries( + muse_glimmer_batching_smoke PRIVATE muse_glimmer_batching_runtime + ) + if(_batching_backend STREQUAL "cuda") + if(NOT APPLE AND NOT MSVC) + target_link_options( + muse_glimmer_batching_smoke PRIVATE "LINKER:--export-dynamic" + ) + endif() + else() + executorch_target_copy_mlx_metallib(muse_glimmer_batching_smoke) + endif() + endif() endif() if(MUSE_GLIMMER_BUILD_BATCHING_TESTS)