From efa5c018863ddc73509dfc4e9509696e7e2ef732 Mon Sep 17 00:00:00 2001 From: gasoonjia Date: Mon, 19 Oct 2026 13:07:58 -0700 Subject: [PATCH] [executorch][ci] Test Muse Glimmer batching on CUDA in OSS CI Pull Request resolved: https://github.com/pytorch/executorch/pull/23818 Build and test Muse Glimmer CUDA batching in OSS CI through the shared serving APIs. The unit job builds the normal runner/worker and explicitly enables the default-OFF MUSE_GLIMMER_BUILD_CI_SMOKE target. The e2e batching mode builds and runs this correctness-only workload with five staggered requests. Require all requests accepted, exact prompt/completion budgets, length completion and successful distinct-session executor steps so serial execution cannot pass as batching coverage. Keep private reusable request control, validation, prompts and batch-owner evidence separate from performance reporting. Smoke reuses ServingRuntime, Runner, DecodeFirstScheduler and the backend executor, disables stop tokens for the exact budget, and has no measurement output. CPU helper regressions cover capacity/overflow validation, successful distinct owners, failed executor calls, exact completion and configure/build failure propagation. Keep the server launcher smoke and existing non-batching variants. Workflow filters and gates explicitly select correctness/helper sources and tests. ghstack-source-id: 451855098 @exported-using-ghexport Differential Revision: [D124871963](https://our.internmc.facebook.com/intern/diff/D124871963/) --- .ci/scripts/BUCK | 13 + .../build_muse_glimmer_batching_smoke.sh | 14 + .ci/scripts/export_model_artifact.sh | 23 +- .ci/scripts/muse_glimmer_batching_smoke.cpp | 101 +++++ .ci/scripts/muse_glimmer_batching_workload.h | 365 ++++++++++++++++++ .ci/scripts/test_model_e2e.sh | 134 ++++++- .../muse_glimmer_batching_workload_test.cpp | 160 ++++++++ .../tests/test_muse_glimmer_batching_ci.py | 144 +++++++ .../test_muse_glimmer_batching_workload.py | 106 +++++ .github/workflows/cuda.yml | 72 ++++ .lintrunner.toml | 1 + examples/models/muse-glimmer/CMakeLists.txt | 24 ++ 12 files changed, 1149 insertions(+), 8 deletions(-) create mode 100644 .ci/scripts/BUCK create mode 100644 .ci/scripts/build_muse_glimmer_batching_smoke.sh create mode 100644 .ci/scripts/muse_glimmer_batching_smoke.cpp create mode 100644 .ci/scripts/muse_glimmer_batching_workload.h create mode 100644 .ci/scripts/tests/muse_glimmer_batching_workload_test.cpp create mode 100644 .ci/scripts/tests/test_muse_glimmer_batching_ci.py create mode 100644 .ci/scripts/tests/test_muse_glimmer_batching_workload.py diff --git a/.ci/scripts/BUCK b/.ci/scripts/BUCK new file mode 100644 index 00000000000..ea724a7f29f --- /dev/null +++ b/.ci/scripts/BUCK @@ -0,0 +1,13 @@ +load("@fbsource//tools/build_defs:fb_native_wrapper.bzl", "fb_native") + +oncall("executorch") + +# CMake builds this CI-only tool; group its sources without a product target. +fb_native.filegroup( + name = "muse_glimmer_batching_smoke_sources", + srcs = [ + "muse_glimmer_batching_smoke.cpp", + "muse_glimmer_batching_workload.h", + "tests/muse_glimmer_batching_workload_test.cpp", + ], +) diff --git a/.ci/scripts/build_muse_glimmer_batching_smoke.sh b/.ci/scripts/build_muse_glimmer_batching_smoke.sh new file mode 100644 index 00000000000..e990842c122 --- /dev/null +++ b/.ci/scripts/build_muse_glimmer_batching_smoke.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Run after the normal batching build; reuse its backend and package settings. +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +BUILD_DIR="${1:-${ROOT}/cmake-out/examples/models/muse-glimmer-batching}" +cmake -S "${ROOT}/examples/models/muse-glimmer" -B "$BUILD_DIR" \ + -DMUSE_GLIMMER_BUILD_BATCHING=ON -DMUSE_GLIMMER_BUILD_CI_SMOKE=ON +cmake --build "$BUILD_DIR" --target muse_glimmer_batching_smoke --parallel diff --git a/.ci/scripts/export_model_artifact.sh b/.ci/scripts/export_model_artifact.sh index 223dd82951c..b23165c23e5 100755 --- a/.ci/scripts/export_model_artifact.sh +++ b/.ci/scripts/export_model_artifact.sh @@ -44,6 +44,7 @@ Arguments: - vr-streaming: Voxtral Realtime streaming mode - vr-offline: Voxtral Realtime offline mode - solo-text: Muse Glimmer solo text mode + - solo-text-batching: Muse Glimmer text on the CUDA batching runtime - dflash-image: Muse Glimmer DFlash vision mode Examples: @@ -95,7 +96,7 @@ if [ -n "$MODE" ]; then exit 1 fi ;; - solo-text|dflash-image) + solo-text|solo-text-batching|dflash-image) if [ "$HF_MODEL" != "meta-models/Muse-Glimmer-30B-GGUF" ]; then echo "Error: Mode '$MODE' can only be used with Muse Glimmer model" echo "Provided model: $HF_MODEL" @@ -104,7 +105,7 @@ if [ -n "$MODE" ]; then ;; *) echo "Error: Unsupported mode '$MODE'" - echo "Supported modes: vr-streaming, vr-offline, solo-text, dflash-image" + echo "Supported modes: vr-streaming, vr-offline, solo-text, solo-text-batching, dflash-image" exit 1 ;; esac @@ -288,8 +289,8 @@ if [ "$MODEL_NAME" = "muse_glimmer" ]; then echo "Error: Muse Glimmer requires quantization 'kquant-17gb' or 'kquant-dynamic'" exit 1 fi - if [ "$MODE" != "solo-text" ] && [ "$MODE" != "dflash-image" ]; then - echo "Error: Muse Glimmer requires mode 'solo-text' or 'dflash-image'" + if [ "$MODE" != "solo-text" ] && [ "$MODE" != "solo-text-batching" ] && [ "$MODE" != "dflash-image" ]; then + echo "Error: Muse Glimmer requires mode 'solo-text', 'solo-text-batching' or 'dflash-image'" exit 1 fi fi @@ -565,6 +566,18 @@ if [ "$MODEL_NAME" = "muse_glimmer" ]; then --backend cuda \ --output-dir "${OUTPUT_DIR}" ;; + solo-text-batching) + # The off-graph cache exports the batching methods CudaExecutor drives: + # Static forward_{N}, dynamic forward_others and the device samplers. + EXPORT_START_SECONDS=$SECONDS + TMPDIR="$INDUCTOR_TMPDIR" \ + TORCHINDUCTOR_CACHE_DIR="$INDUCTOR_CACHE" \ + python -m executorch.examples.models.muse_glimmer.export.export_solo \ + --gguf "$TARGET_GGUF_PATH" \ + --backend cuda \ + --use-offgraph-kv-cache \ + --output-dir "${OUTPUT_DIR}" + ;; dflash-image) DRAFT_GGUF_FILE="dflash-Muse-Glimmer-30B-Q4_K_M.gguf" MMPROJ_GGUF_FILE="mmproj-Muse-Glimmer-30B-Q4_K_M.gguf" @@ -581,7 +594,7 @@ if [ "$MODEL_NAME" = "muse_glimmer" ]; then --output-dir "${OUTPUT_DIR}" ;; *) - echo "Error: Muse Glimmer requires mode 'solo-text' or 'dflash-image'" + echo "Error: Muse Glimmer requires mode 'solo-text', 'solo-text-batching' or 'dflash-image'" exit 1 ;; esac diff --git a/.ci/scripts/muse_glimmer_batching_smoke.cpp b/.ci/scripts/muse_glimmer_batching_smoke.cpp new file mode 100644 index 00000000000..6b9c794c706 --- /dev/null +++ b/.ci/scripts/muse_glimmer_batching_smoke.cpp @@ -0,0 +1,101 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#include "muse_glimmer_batching_workload.h" + +#include +#include +#include + +#include +#include +#include +#include + +DECLARE_string(pte); +DECLARE_string(tokenizer); +DECLARE_string(data_path); +DECLARE_string(backend); +DECLARE_int32(max_sessions); +DECLARE_int32(max_session_tokens); +DECLARE_uint64(bos_id); + +DEFINE_string(mode, "requests", "Correctness workload: requests"); +DEFINE_int32(requests, 8, "Number of concurrent requests"); +DEFINE_int32(prompt_tokens, 512, "Prompt positions including BOS per request"); +DEFINE_int32(max_new_tokens, 128, "Exact completion budget per request"); +DEFINE_int32(stagger_ms, 0, "Delay between request submissions"); +DEFINE_int32(max_decode, 32, "Maximum decodes admitted per step"); +DEFINE_int32(timeout_s, 600, "Request workload deadline"); + +int main(int argc, char** argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + try { + namespace llm = ::executorch::extension::llm; + namespace ci = muse_glimmer_ci; + if (FLAGS_mode != "requests" || FLAGS_pte.empty() || + FLAGS_tokenizer.empty() || FLAGS_max_sessions <= 0 || + FLAGS_max_session_tokens < 0) { + throw std::invalid_argument( + "invalid model or correctness workload flags"); + } + const ci::RequestOptions options{ + FLAGS_requests, + FLAGS_prompt_tokens, + FLAGS_max_new_tokens, + FLAGS_stagger_ms, + FLAGS_max_decode, + FLAGS_timeout_s}; + // Reject impossible workloads before loading tokenizer or model weights. + ci::validate_requests( + options, FLAGS_max_sessions, FLAGS_max_session_tokens); + tokenizers::HFTokenizer tokenizer; + if (tokenizer.load(FLAGS_tokenizer) != tokenizers::Error::Ok) { + throw std::runtime_error("could not load the tokenizer"); + } + llm::MuseGlimmerBackendConfig config; + config.backend = FLAGS_backend; + config.model_path = FLAGS_pte; + config.data_path = FLAGS_data_path; + config.max_sessions = FLAGS_max_sessions; + config.max_session_tokens = FLAGS_max_session_tokens; + config.bos_id = FLAGS_bos_id; + auto backend = llm::create_muse_glimmer_backend(config); + if (!backend.ok()) { + throw std::runtime_error( + "could not create the executor: error " + + std::to_string(static_cast(backend.error()))); + } + ci::EvidenceExecutor executor(*backend->executor); + llm::serving::ServingRuntimeConfig serving; + serving.max_sessions = options.requests; + serving.max_requests = options.requests; + serving.max_pending_operations = options.requests; + serving.max_context_length = backend->preparation->max_context_length; + auto result = ci::run_requests( + options, + executor, + tokenizer, + serving, + [spec = backend->preparation]( + const llm::serving::PromptPreparationContext& context, + llm::serving::PromptInput input) { + return llm::prepare_muse_glimmer_input( + context, std::move(input), spec); + }); + const auto evidence = executor.evidence(); + ci::check_batching(evidence); + std::cout << "Batching correctness passed: " << result.accepted + << " requests completed with exact token budgets; " + << evidence.multi_session_steps + << " successful multi-session executor steps\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "batching smoke failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/.ci/scripts/muse_glimmer_batching_workload.h b/.ci/scripts/muse_glimmer_batching_workload.h new file mode 100644 index 00000000000..33f25418cb9 --- /dev/null +++ b/.ci/scripts/muse_glimmer_batching_workload.h @@ -0,0 +1,365 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Private CI workload control, shared by correctness and measurement tools. +namespace muse_glimmer_ci { +namespace batching = ::executorch::extension::llm::batching; +namespace serving = ::executorch::extension::llm::serving; + +struct RequestOptions { + std::int32_t requests = 8; + std::int32_t prompt_tokens = 512; + std::int32_t max_new_tokens = 128; + std::int32_t stagger_ms = 0; + std::int32_t max_decode = 32; + std::int32_t timeout_s = 600; +}; + +inline void check_session_capacity( + std::size_t required, + std::size_t available) { + if (available == 0 || required == 0 || required > available) { + throw std::invalid_argument("workload exceeds --max_sessions"); + } +} + +inline void validate_requests( + const RequestOptions& options, + std::size_t max_sessions, + std::size_t max_context_length = 0) { + if (options.requests <= 0 || options.prompt_tokens <= 0 || + options.max_new_tokens <= 0 || options.stagger_ms < 0 || + options.max_decode <= 0 || options.timeout_s <= 0) { + throw std::invalid_argument("invalid request workload options"); + } + check_session_capacity(options.requests, max_sessions); + const auto tokens = static_cast(options.prompt_tokens) + + static_cast(options.max_new_tokens); + if (max_context_length > 0 && tokens > max_context_length) { + throw std::invalid_argument("request token budget exceeds context"); + } +} + +inline std::vector prompt_ids( + std::size_t length, + std::size_t salt) { + std::vector ids(length); + for (std::size_t i = 0; i < length; ++i) { + ids[i] = 1000 + (i * 7919 + salt * 104729) % 99000; + } + return ids; +} + +struct BatchEvidence { + std::size_t max_sessions = 0; + std::size_t multi_session_steps = 0; + std::size_t mixed_steps = 0; + + void observe(const batching::BatchInput& batch) { + std::set sessions; + std::size_t decodes = 0; + for (const auto& input : batch.inputs) { + sessions.insert(input.sid); + decodes += input.size == 1 && input.produce_output && + std::holds_alternative(input.payload); + } + max_sessions = std::max(max_sessions, sessions.size()); + if (sessions.size() > 1) { + ++multi_session_steps; + if (decodes > 0 && batch.size() > decodes) { + ++mixed_steps; + } + } + } +}; + +// Counts distinct owners only after the forwarded executor call succeeds. +class EvidenceExecutor final : public batching::Executor { + public: + explicit EvidenceExecutor(batching::Executor& inner) : inner_(inner) {} + + bool initialize() override { + return inner_.initialize(); + } + std::size_t preferred_batch_tokens() const override { + return inner_.preferred_batch_tokens(); + } + std::optional open_session() override { + return inner_.open_session(); + } + void close_session(batching::SessionId session) override { + inner_.close_session(session); + } + std::optional clone( + batching::SessionId source, + batching::Position upto) override { + return inner_.clone(source, upto); + } + void set_sampling( + batching::SessionId session, + const batching::SamplingParams& params, + std::optional seed) override { + inner_.set_sampling(session, params, seed); + } + bool accepts(const batching::PreparedInput& input) const override { + return inner_.accepts(input); + } + bool execute(const batching::BatchInput& batch, batching::BatchOutput& out) + override { + if (!inner_.execute(batch, out)) { + return false; + } + std::lock_guard lock(mutex_); + evidence_.observe(batch); + return true; + } + BatchEvidence evidence() const { + std::lock_guard lock(mutex_); + return evidence_; + } + + private: + batching::Executor& inner_; + mutable std::mutex mutex_; + BatchEvidence evidence_; +}; + +inline void check_batching(const BatchEvidence& evidence) { + if (evidence.max_sessions <= 1 || evidence.multi_session_steps == 0) { + throw std::runtime_error("no successful multi-session executor step"); + } +} + +inline std::unique_ptr make_runtime( + batching::Executor& executor, + std::unique_ptr scheduler, + const tokenizers::Tokenizer& tokenizer, + serving::ServingRuntimeConfig config, + serving::ModelPreparer preparer) { + if (!scheduler) { + throw std::invalid_argument("unusable scheduler limits"); + } + return std::make_unique( + executor, std::move(scheduler), tokenizer, config, std::move(preparer)); +} + +struct Inflight { + std::mutex mutex; + std::condition_variable changed; + std::size_t running = 0; + std::size_t failed = 0; + std::string error; + + void finished(const serving::TerminalEvent& terminal) { + { + std::lock_guard lock(mutex); + --running; + if (terminal.finish_reason == serving::FinishReason::Failed) { + ++failed; + if (terminal.error && error.empty()) { + error = terminal.error->message; + } + } + } + changed.notify_all(); + } +}; + +inline std::optional submit_request( + serving::ServingRuntime& runtime, + Inflight& inflight, + std::size_t length, + std::size_t salt, + std::int32_t new_tokens, + std::function on_terminal = {}, + std::function on_text = {}) { + if (length == 0 || new_tokens <= 0) { + throw std::invalid_argument("invalid prompt or generation budget"); + } + serving::PromptInput prompt; + // Model preparation adds BOS, which is included in the requested length. + prompt.segments.emplace_back(prompt_ids(length - 1, salt)); + serving::GenerationOptions options; + options.max_new_tokens = new_tokens; + options.sampling.temperature = 0.0f; + options.seed = salt; + { + std::lock_guard lock(inflight.mutex); + ++inflight.running; + } + auto result = runtime.generate( + std::nullopt, + std::move(prompt), + options, + [&inflight, on_terminal, on_text](serving::GenerationEvent event) { + if (const auto* text = std::get_if(&event); + text && !text->text.empty() && on_text) { + on_text(); + } + if (const auto* terminal = + std::get_if(&event)) { + if (on_terminal) { + on_terminal(*terminal); + } + inflight.finished(*terminal); + } + }); + if (const auto* error = std::get_if(&result)) { + std::lock_guard lock(inflight.mutex); + --inflight.running; + ++inflight.failed; + inflight.error = error->message; + return std::nullopt; + } + return std::get(std::move(result)); +} + +struct RequestResult { + std::size_t accepted = 0; + std::vector> terminals; +}; + +inline void check_requests_complete( + const RequestOptions& options, + const RequestResult& result) { + if (result.accepted != static_cast(options.requests) || + result.terminals.size() != result.accepted) { + throw std::runtime_error("not all workload requests were accepted"); + } + const auto incomplete = + [&](const std::optional& terminal) { + return !terminal || + terminal->finish_reason != serving::FinishReason::Length || + terminal->stats.prompt_tokens != + static_cast(options.prompt_tokens) || + terminal->stats.completion_tokens != + static_cast(options.max_new_tokens); + }; + if (std::any_of( + result.terminals.begin(), result.terminals.end(), incomplete)) { + throw std::runtime_error("request failed or token budget incomplete"); + } +} + +// Submission runs on the caller; text/terminal hooks run on delivery. Captured +// state must outlive run_requests. Hooks must be bounded and must not throw. +struct RequestCallbacks { + std::function submitted; + std::function text; + std::function terminal; +}; + +inline RequestResult run_requests( + const RequestOptions& options, + batching::Executor& executor, + const tokenizers::Tokenizer& tokenizer, + serving::ServingRuntimeConfig config, + serving::ModelPreparer preparer, + const RequestCallbacks& callbacks = {}) { + validate_requests(options, config.max_sessions, config.max_context_length); + check_session_capacity(options.requests, config.max_requests); + const auto decodes = + static_cast(std::min(options.max_decode, options.requests)); + const auto budget = std::max(executor.preferred_batch_tokens(), decodes + 1); + auto scheduler = + batching::DecodeFirstScheduler::create(budget, decodes, budget - decodes); + config.default_stop_tokens = {}; + RequestResult result; + result.terminals.resize(options.requests); + Inflight inflight; + std::vector handles; + // Runtime destruction fences callbacks before their captured state dies. + auto runtime = make_runtime( + executor, std::move(scheduler), tokenizer, config, std::move(preparer)); + using Clock = std::chrono::steady_clock; + const auto deadline = Clock::now() + std::chrono::seconds(options.timeout_s); + bool timed_out = false; + for (std::size_t i = 0; i < result.terminals.size(); ++i) { + if (i > 0 && options.stagger_ms > 0) { + std::this_thread::sleep_until(std::min( + deadline, + Clock::now() + std::chrono::milliseconds(options.stagger_ms))); + } + if (Clock::now() >= deadline) { + timed_out = true; + break; + } + if (callbacks.submitted) { + callbacks.submitted(i); + } + auto handle = submit_request( + *runtime, + inflight, + options.prompt_tokens, + i, + options.max_new_tokens, + [&result, &callbacks, i](const serving::TerminalEvent& terminal) { + result.terminals[i] = terminal; + if (callbacks.terminal) { + callbacks.terminal(i, terminal); + } + }, + [&callbacks, i] { + if (callbacks.text) { + callbacks.text(i); + } + }); + if (!handle) { + break; + } + ++result.accepted; + handles.push_back(std::move(*handle)); + } + { + std::unique_lock lock(inflight.mutex); + timed_out = timed_out || Clock::now() >= deadline || + !inflight.changed.wait_until(lock, deadline, [&] { + return inflight.running == 0 || inflight.failed > 0; + }); + if (timed_out || inflight.failed > 0 || + result.accepted != result.terminals.size()) { + lock.unlock(); + for (const auto& handle : handles) { + handle.cancel(); + } + runtime->shutdown(); + } + } + for (const auto& handle : handles) { + handle.wait(); + } + runtime->shutdown(); + if (timed_out) { + throw std::runtime_error("request workload timed out"); + } + check_requests_complete(options, result); + return result; +} + +} // namespace muse_glimmer_ci diff --git a/.ci/scripts/test_model_e2e.sh b/.ci/scripts/test_model_e2e.sh index 427329ae961..c6a2131c004 100755 --- a/.ci/scripts/test_model_e2e.sh +++ b/.ci/scripts/test_model_e2e.sh @@ -46,6 +46,7 @@ Arguments: - vr-streaming: Voxtral Realtime streaming mode - vr-offline: Voxtral Realtime offline mode - solo-text: Muse Glimmer solo text mode + - solo-text-batching: Muse Glimmer text on the CUDA batching runtime - dflash-image: Muse Glimmer DFlash vision mode Environment: @@ -111,7 +112,7 @@ if [ -n "$MODE" ]; then exit 1 fi ;; - solo-text|dflash-image) + solo-text|solo-text-batching|dflash-image) if [ "$HF_MODEL" != "meta-models/Muse-Glimmer-30B-GGUF" ]; then echo "Error: Mode '$MODE' can only be used with Muse Glimmer model" echo "Provided model: $HF_MODEL" @@ -120,7 +121,7 @@ if [ -n "$MODE" ]; then ;; *) echo "Error: Unsupported mode '$MODE'" - echo "Supported modes: vr-streaming, vr-offline, solo-text, dflash-image" + echo "Supported modes: vr-streaming, vr-offline, solo-text, solo-text-batching, dflash-image" exit 1 ;; esac @@ -290,13 +291,20 @@ case "$HF_MODEL" in EXPECTED_OUTPUT="Paris" IMAGE_URL="" ;; + solo-text-batching) + # Built by `make muse-glimmer-batching-cuda` into its own directory. + RUNNER_PATH="muse-glimmer-batching" + RUNNER_TARGET="muse_glimmer_batching_runner" + EXPECTED_OUTPUT="Paris" + IMAGE_URL="" + ;; dflash-image) RUNNER_TARGET="dflash_runner" EXPECTED_OUTPUT="dog" IMAGE_URL="https://github.com/pytorch/hub/raw/master/images/dog.jpg" ;; *) - echo "Error: Muse Glimmer requires mode 'solo-text' or 'dflash-image'" + echo "Error: Muse Glimmer requires mode 'solo-text', 'solo-text-batching' or 'dflash-image'" exit 1 ;; esac @@ -453,6 +461,11 @@ EOF fi RUNNER_ARGS="$RUNNER_ARGS --tokenizer_path ${MODEL_DIR}/$TOKENIZER_FILE --prompt_file \"$PROMPT_FILE\" --max_new_tokens 512" RUNNER_ARGS="$RUNNER_ARGS --cuda_graph" + if [ "$MODE" = "solo-text-batching" ]; then + # The batching runner goes through ServingRuntime and takes its own flags. + printf '%s' '<|start|>user<|message|>What is the capital of France?<|eot|><|start|>assistant' > "$PROMPT_FILE" + RUNNER_ARGS="--pte ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer ${MODEL_DIR}/$TOKENIZER_FILE --prompt \"\$(cat $PROMPT_FILE)\" --max_new_tokens 128 --temperature 0 --max_session_tokens 4096" + fi if [ "$MODE" = "dflash-image" ]; then RUNNER_ARGS="$RUNNER_ARGS --image_path ${MODEL_DIR}/test_image.jpg" fi @@ -540,6 +553,121 @@ else fi echo "::endgroup::" +if [ "$DEVICE" = "cuda" ] && [ "$MODEL_NAME" = "muse_glimmer" ] && [ "$MODE" = "solo-text-batching" ]; then + echo "::group::Run $MODEL_NAME batching serving smoke" + # The CI-only correctness tool is not part of the normal serving build. + bash .ci/scripts/build_muse_glimmer_batching_smoke.sh + # Concurrent staggered requests through ServingRuntime on CudaExecutor. + cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_smoke \ + --pte "${MODEL_DIR}/model.pte" --data_path "${MODEL_DIR}/aoti_cuda_blob.ptd" \ + --tokenizer "${MODEL_DIR}/$TOKENIZER_FILE" --max_sessions 8 --max_session_tokens 2048 \ + --mode requests --requests 5 --prompt_tokens 600 --max_new_tokens 64 --stagger_ms 200 + + pip install -r examples/llm_server/python/requirements.txt + python -c "from huggingface_hub import hf_hub_download; [hf_hub_download('${HF_MODEL}', f, local_dir='${MODEL_DIR}') for f in ('tokenizer_config.json', 'chat_template.jinja')]" + PORT=$(python - <<'PY' +import socket + +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + print(s.getsockname()[1]) +PY +) + SERVER_LOG=$(mktemp) + python -u -m executorch.examples.models.muse_glimmer.serving.serve_batching \ + --backend cuda \ + --worker-bin cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_worker \ + --model-path "${MODEL_DIR}/model.pte" \ + --data-path "${MODEL_DIR}/aoti_cuda_blob.ptd" \ + --tokenizer-path "${MODEL_DIR}/$TOKENIZER_FILE" \ + --hf-tokenizer "${MODEL_DIR}" \ + --max-context 4096 \ + --prefix-cache-entries 2 \ + --host 127.0.0.1 \ + --port "$PORT" >"$SERVER_LOG" 2>&1 & + SERVER_PID=$! + cleanup_mg_server() { + if kill -0 "$SERVER_PID" 2>/dev/null; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + rm -f "$SERVER_LOG" + } + trap cleanup_mg_server EXIT + + if ! python - "$PORT" "$SERVER_LOG" <<'PY' +import json +import sys +import time +import urllib.request + +port, log_path = sys.argv[1], sys.argv[2] +base = f"http://127.0.0.1:{port}" + + +def request(path, payload=None): + data = None + headers = {} + if payload is not None: + data = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request(base + path, data=data, headers=headers) + with urllib.request.urlopen(req, timeout=300) as resp: + return resp.read().decode("utf-8") + + +last = None +for _ in range(600): + try: + request("/health") + break + except Exception as e: + last = e + time.sleep(1) +else: + print(open(log_path, encoding="utf-8", errors="replace").read()) + raise RuntimeError(f"server did not become healthy: {last}") + +body = { + "model": "muse_glimmer", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "max_tokens": 128, + "temperature": 0, +} +# The second identical request reuses the first one's prompt snapshot. +for attempt in range(2): + resp = json.loads(request("/v1/chat/completions", body)) + message = resp["choices"][0]["message"] + text = (message.get("content") or "") + (message.get("reasoning_content") or "") + if "Paris" not in text: + raise AssertionError(f"expected Paris in serving response, got: {message!r}") + details = resp["usage"].get("prompt_tokens_details") or {} + cached = details.get("cached_tokens", 0) + print(f"request {attempt}: prompt {resp['usage']['prompt_tokens']} cached {cached}") +if cached == 0: + raise AssertionError("the repeated request reused no prompt tokens") + +stream = request("/v1/chat/completions", dict(body, stream=True)) +pieces = [] +for line in stream.splitlines(): + if line.startswith("data: ") and line != "data: [DONE]": + delta = json.loads(line[6:])["choices"][0]["delta"] + pieces.append((delta.get("content") or "") + (delta.get("reasoning_content") or "")) +if len([p for p in pieces if p]) < 2 or "Paris" not in "".join(pieces): + raise AssertionError(f"expected Paris streamed in several chunks, got: {pieces!r}") +print("Muse Glimmer CUDA batching serving smoke passed") +PY + then + echo "Muse Glimmer batching serving smoke failed; server log:" + cat "$SERVER_LOG" + exit 1 + fi + + cleanup_mg_server + trap - EXIT + echo "::endgroup::" +fi + if [ "$DEVICE" = "cuda" ] && [ "$MODEL_NAME" = "qwen3_5_moe" ]; then echo "::group::Run $MODEL_NAME OpenAI serving smoke" pip install -r examples/llm_server/python/requirements.txt "transformers==5.0.0rc1" diff --git a/.ci/scripts/tests/muse_glimmer_batching_workload_test.cpp b/.ci/scripts/tests/muse_glimmer_batching_workload_test.cpp new file mode 100644 index 00000000000..94ba8ed639b --- /dev/null +++ b/.ci/scripts/tests/muse_glimmer_batching_workload_test.cpp @@ -0,0 +1,160 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#include "../muse_glimmer_batching_workload.h" + +#include +#include + +namespace ci = muse_glimmer_ci; +namespace batching = ci::batching; +namespace serving = ci::serving; + +namespace { +void require(bool condition) { + if (!condition) { + throw std::runtime_error("CI workload regression failed"); + } +} + +template +void rejected(F action) { + bool failed = false; + try { + action(); + } catch (const std::exception&) { + failed = true; + } + require(failed); +} + +class FakeExecutor final : public batching::Executor { + public: + bool succeeds = true; + std::optional open_session() override { + return 1; + } + void close_session(batching::SessionId) override {} + void set_sampling( + batching::SessionId, + const batching::SamplingParams&, + std::optional) override {} + bool execute(const batching::BatchInput&, batching::BatchOutput&) override { + return succeeds; + } +}; + +batching::Input input(batching::SessionId session, std::size_t size = 1) { + return { + session, + true, + 0, + size, + std::make_shared>(size, 1000), + 0}; +} + +void workload_limits() { + ci::RequestOptions options{5, 600, 64, 200, 32, 600}; + ci::validate_requests(options, 8, 2048); + rejected([&] { ci::validate_requests(options, 4, 2048); }); + rejected([&] { ci::validate_requests(options, 8, 663); }); + ci::validate_requests(options, 5, 664); + ci::validate_requests(options, 5); + rejected([] { ci::check_session_capacity(0, 5); }); + rejected([] { ci::check_session_capacity(1, 0); }); + for (auto member : + {&ci::RequestOptions::requests, + &ci::RequestOptions::prompt_tokens, + &ci::RequestOptions::max_new_tokens, + &ci::RequestOptions::max_decode, + &ci::RequestOptions::timeout_s}) { + auto invalid = options; + invalid.*member = 0; + rejected([&] { ci::validate_requests(invalid, 8); }); + } + options.stagger_ms = -1; + rejected([&] { ci::validate_requests(options, 8); }); + options = {1, std::numeric_limits::max(), 1, 0, 1, 1}; + rejected([&] { + ci::validate_requests(options, 1, std::numeric_limits::max()); + }); + const auto prompt = ci::prompt_ids(599, 3); + require(prompt.size() == 599 && prompt == ci::prompt_ids(599, 3)); + require(prompt != ci::prompt_ids(599, 4)); + require(ci::prompt_ids(0, 0).empty()); + for (const auto token : prompt) { + require(token >= 1000 && token < 100000); + } +} + +void successful_batch_owners() { + FakeExecutor inner; + ci::EvidenceExecutor executor(inner); + batching::BatchOutput out; + require(executor.execute({{input(1), input(1)}}, out)); + require(executor.execute({{input(2)}}, out)); + require(executor.evidence().max_sessions == 1); + rejected([&] { ci::check_batching(executor.evidence()); }); + inner.succeeds = false; + require(!executor.execute({{input(1), input(2)}}, out)); + require(executor.evidence().multi_session_steps == 0); + rejected([&] { ci::check_batching(executor.evidence()); }); + // cppcheck-suppress redundantAssignment + inner.succeeds = true; + require(executor.execute({{input(1), input(2)}}, out)); + ci::check_batching(executor.evidence()); + require(executor.evidence().multi_session_steps == 1); + require(executor.evidence().mixed_steps == 0); + require(executor.execute({{input(1), input(2, 4)}}, out)); + require(executor.evidence().mixed_steps == 1); +} + +void exact_completion() { + const ci::RequestOptions options{2, 600, 64, 0, 2, 1}; + serving::TerminalEvent terminal; + terminal.finish_reason = serving::FinishReason::Length; + terminal.stats.prompt_tokens = 600; + terminal.stats.completion_tokens = 64; + ci::RequestResult result{2, {terminal, terminal}}; + ci::check_requests_complete(options, result); + result.accepted = 1; + rejected([&] { ci::check_requests_complete(options, result); }); + result.accepted = 2; + result.terminals[1].reset(); + rejected([&] { ci::check_requests_complete(options, result); }); + for (const auto reason : + {serving::FinishReason::Failed, + serving::FinishReason::Stop, + serving::FinishReason::Cancelled}) { + result.terminals[1] = terminal; + result.terminals[1]->finish_reason = reason; + rejected([&] { ci::check_requests_complete(options, result); }); + } + for (const auto count : {63, 65}) { + result.terminals[1] = terminal; + result.terminals[1]->stats.completion_tokens = count; + rejected([&] { ci::check_requests_complete(options, result); }); + } + result.terminals[1] = terminal; + result.terminals[1]->stats.prompt_tokens = 599; + rejected([&] { ci::check_requests_complete(options, result); }); +} +} // namespace + +int main() { + try { + workload_limits(); + successful_batch_owners(); + exact_completion(); + std::cout << "CI batching workload tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/.ci/scripts/tests/test_muse_glimmer_batching_ci.py b/.ci/scripts/tests/test_muse_glimmer_batching_ci.py new file mode 100644 index 00000000000..5c8fcead860 --- /dev/null +++ b/.ci/scripts/tests/test_muse_glimmer_batching_ci.py @@ -0,0 +1,144 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import re +import subprocess +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPTS = ROOT / ".ci/scripts" +CORRECTNESS_PATHS = ( + ".ci/scripts/build_muse_glimmer_batching_smoke.sh", + ".ci/scripts/muse_glimmer_batching_smoke.cpp", + ".ci/scripts/muse_glimmer_batching_workload.h", + ".ci/scripts/tests/test_muse_glimmer_batching_ci.py", + ".ci/scripts/tests/test_muse_glimmer_batching_workload.py", + ".ci/scripts/tests/muse_glimmer_batching_workload_test.cpp", +) + + +class MuseGlimmerBatchingCITest(unittest.TestCase): + def test_e2e_builds_smoke_before_running_it(self): + script = (SCRIPTS / "test_model_e2e.sh").read_text() + build = script.index("bash .ci/scripts/build_muse_glimmer_batching_smoke.sh") + run = script.index( + "cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_smoke" + ) + self.assertLess(build, run) + # A separate measurement pass may follow this correctness command at tip. + lines = [] + for line in script[run:].splitlines(): + lines.append(line) + if not line.endswith("\\"): + break + smoke = "\n".join(lines) + for flag in ("--requests 5", "--prompt_tokens 600", "--max_new_tokens 64"): + self.assertIn(flag, smoke) + self.assertIn("--stagger_ms 200", smoke) + self.assertNotIn("bench", smoke) + self.assertNotIn("report_json", smoke) + self.assertNotIn("tokens_per_s", smoke) + + def test_workflow_selects_correctness_tests_and_smoke(self): + workflow = (ROOT / ".github/workflows/cuda.yml").read_text() + unit_job = workflow.split(" unittest-cuda-muse-glimmer-batching:", 1)[1].split( + " unittest-cuda-quantized-gemm:", 1 + )[0] + self.assertIn("bash .ci/scripts/build_muse_glimmer_batching_smoke.sh", unit_job) + self.assertIn( + 'test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_smoke"', unit_job + ) + self.assertIn("-p 'test_muse_glimmer_batching_ci.py'", unit_job) + self.assertIn("-p 'test_muse_glimmer_batching_workload.py'", unit_job) + self.assertNotIn("test_muse_glimmer_batching_*.py", unit_job) + filters = workflow.split(" pull_request:\n", 1)[1].split( + " workflow_dispatch:", 1 + )[0] + e2e_job = workflow.split(" test-muse-glimmer-cuda-e2e:", 1)[1] + for path in CORRECTNESS_PATHS: + self.assertIn(f" - {path}\n", filters) + self.assertIn(f"'{path}'", unit_job) + self.assertIn(f"'{path}'", e2e_job) + self.assertNotIn(".ci/scripts/*muse_glimmer_batching*", filters) + self.assertNotIn(".ci/scripts/tests/*muse_glimmer_batching*", filters) + for prefix in ( + ".ci/scripts/muse_glimmer_batching", + ".ci/scripts/build_muse_glimmer_batching", + ".ci/scripts/tests/test_muse_glimmer_batching", + ".ci/scripts/tests/muse_glimmer_batching", + ): + self.assertNotIn(f"'{prefix}'", unit_job + e2e_job) + + def test_smoke_uses_shared_runtime_and_has_no_measurement_features(self): + source = (SCRIPTS / "muse_glimmer_batching_smoke.cpp").read_text() + helper = (SCRIPTS / "muse_glimmer_batching_workload.h").read_text() + self.assertIn("create_muse_glimmer_backend(config)", source) + self.assertIn("ci::run_requests(", source) + self.assertIn("ci::check_batching(evidence)", source) + self.assertIn("std::make_unique", helper) + self.assertIn("runtime.generate(", helper) + self.assertIn("config.default_stop_tokens = {};", helper) + self.assertIn("runtime->shutdown();", helper) + self.assertIn("handle.wait();", helper) + self.assertIn("check_requests_complete(options, result)", helper) + for forbidden in ( + "bench_utils", + "SweepShape", + "TimedExecutor", + "RequestTiming", + "report_json", + "nlohmann", + "tokens_per_s", + "ttft", + "tpot", + "prefill_ms", + "decode_ms", + "total_ms", + "VRAM", + ): + self.assertNotIn(forbidden, source + helper) + self.assertLess( + source.index("ci::validate_requests("), source.index("tokenizer.load(") + ) + + def test_smoke_cmake_is_opt_in_and_reuses_runtime(self): + project = ROOT if (ROOT / "examples").is_dir() else ROOT.parent + cmake = (project / "examples/models/muse-glimmer/CMakeLists.txt").read_text() + self.assertRegex( + cmake, + r'option\(MUSE_GLIMMER_BUILD_CI_SMOKE\s+"[^"]+"\s+OFF\s*\)', + ) + self.assertIn( + "if(MUSE_GLIMMER_BUILD_CI_SMOKE AND NOT MUSE_GLIMMER_BUILD_BATCHING)", cmake + ) + smoke = cmake.split(" if(MUSE_GLIMMER_BUILD_CI_SMOKE)\n", 1)[1].split( + "\n endif()", 1 + )[0] + self.assertIn( + "muse_glimmer_batching_smoke PRIVATE muse_glimmer_batching_runtime", smoke + ) + self.assertIn("LINKER:--export-dynamic", smoke) + self.assertIn( + "executorch_target_copy_mlx_metallib(muse_glimmer_batching_smoke)", smoke + ) + self.assertNotIn("BENCHMARK", smoke) + self.assertNotIn("batching_bench", smoke) + + def test_owned_shell_and_unit_workflow_syntax(self): + for script in ("test_model_e2e.sh", "build_muse_glimmer_batching_smoke.sh"): + result = subprocess.run(["bash", "-n", str(SCRIPTS / script)]) + self.assertEqual(result.returncode, 0) + workflow = (ROOT / ".github/workflows/cuda.yml").read_text() + unit_job = workflow.split(" unittest-cuda-muse-glimmer-batching:", 1)[1].split( + " unittest-cuda-quantized-gemm:", 1 + )[0] + body = unit_job.split(" script: |\n", 1)[1] + shell = "\n".join(line[8:] for line in body.splitlines()) + shell = re.sub(r"\$\{\{[^}]+\}\}", "ci-value", shell) + result = subprocess.run(["bash", "-n"], input=shell, text=True) + self.assertEqual(result.returncode, 0) diff --git a/.ci/scripts/tests/test_muse_glimmer_batching_workload.py b/.ci/scripts/tests/test_muse_glimmer_batching_workload.py new file mode 100644 index 00000000000..3951b37a9c8 --- /dev/null +++ b/.ci/scripts/tests/test_muse_glimmer_batching_workload.py @@ -0,0 +1,106 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SCRIPTS = Path(__file__).resolve().parents[1] + + +class MuseGlimmerBatchingWorkloadTest(unittest.TestCase): + def test_workload_without_a_model(self): + compiler = shutil.which("c++") + self.assertIsNotNone(compiler, "CI workload tests require a C++ compiler") + root = SCRIPTS.parents[1] + project = root if (root / "examples").is_dir() else root.parent + with tempfile.TemporaryDirectory() as directory: + temporary = Path(directory) + binary = temporary / "workload_test" + # Only the unused multimodal/tensor boundary is stubbed. Compile the + # real serving declarations, executor and private workload helper. + multimodal = ( + temporary / "executorch/extension/llm/runner/multimodal_input.h" + ) + multimodal.parent.mkdir(parents=True) + multimodal.write_text( + "#pragma once\n#include \n#include \n" + "namespace executorch::extension::llm {\n" + "struct MultimodalInput {\n" + " explicit MultimodalInput(std::vector) {}\n" + "};\n}\n" + ) + result = subprocess.run( + [ + compiler, + "-std=c++17", + "-Wall", + "-Wextra", + "-Werror", + "-Wno-deprecated-declarations", + "-pthread", + "-I", + str(temporary), + "-I", + str(project.parent), + str(SCRIPTS / "tests/muse_glimmer_batching_workload_test.cpp"), + "-o", + str(binary), + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + result = subprocess.run([str(binary)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("tests passed", result.stdout) + + def test_build_enables_smoke_and_propagates_failures(self): + for failure in ("none", "configure", "build"): + with self.subTest( + failure=failure + ), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + log = root / "commands" + cmake = root / "cmake" + cmake.write_text( + '#!/bin/bash\nprintf "%s\\n" "$*" >> "$COMMAND_LOG"\n' + 'if [[ "$1" == "--build" ]]; then\n' + ' [[ "$FAIL_AT" != "build" ]]\n' + 'else\n [[ "$FAIL_AT" != "configure" ]]\nfi\n' + ) + cmake.chmod(0o755) + build_dir = root / "build with spaces" + result = subprocess.run( + [ + "bash", + str(SCRIPTS / "build_muse_glimmer_batching_smoke.sh"), + str(build_dir), + ], + env={ + **os.environ, + "PATH": f"{root}:{os.environ['PATH']}", + "COMMAND_LOG": str(log), + "FAIL_AT": failure, + }, + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode == 0, failure == "none", result.stderr + ) + commands = log.read_text().splitlines() + self.assertIn("-DMUSE_GLIMMER_BUILD_CI_SMOKE=ON", commands[0]) + self.assertIn("-DMUSE_GLIMMER_BUILD_BATCHING=ON", commands[0]) + self.assertIn(str(build_dir), commands[0]) + self.assertNotIn("BENCHMARK", "\n".join(commands)) + self.assertEqual(len(commands), 1 if failure == "configure" else 2) + if len(commands) == 2: + self.assertIn("--target muse_glimmer_batching_smoke", commands[1]) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 63709510639..49cfa87559a 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -34,6 +34,12 @@ on: - .ci/scripts/cuda_benchmark.py - .ci/scripts/cuda_benchmark_from_e2e.sh - .ci/scripts/cuda_perf_prompts/** + - .ci/scripts/build_muse_glimmer_batching_smoke.sh + - .ci/scripts/muse_glimmer_batching_smoke.cpp + - .ci/scripts/muse_glimmer_batching_workload.h + - .ci/scripts/tests/test_muse_glimmer_batching_ci.py + - .ci/scripts/tests/test_muse_glimmer_batching_workload.py + - .ci/scripts/tests/muse_glimmer_batching_workload_test.cpp - examples/models/muse-glimmer/** - extension/pybindings/** - runtime/__init__.py @@ -556,6 +562,64 @@ jobs: pip install gguf python -m pytest examples/models/gemma4_31b/tests/ --ignore=examples/models/gemma4_31b/tests/test_mlx_pipeline.py -v -o "addopts=" + unittest-cuda-muse-glimmer-batching: + name: unittest-cuda-muse-glimmer-batching + needs: [changed-files, run-decision] + if: | + contains(needs.changed-files.outputs.changed-files, 'backends/cuda') || + contains(needs.changed-files.outputs.changed-files, 'backends/aoti') || + contains(needs.changed-files.outputs.changed-files, 'extension/llm/batching') || + contains(needs.changed-files.outputs.changed-files, 'extension/llm/serving') || + contains(needs.changed-files.outputs.changed-files, 'examples/llm_server/cpp') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/build_muse_glimmer_batching_smoke.sh') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/muse_glimmer_batching_smoke.cpp') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/muse_glimmer_batching_workload.h') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/test_muse_glimmer_batching_ci.py') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/test_muse_glimmer_batching_workload.py') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/muse_glimmer_batching_workload_test.cpp') || + contains(needs.changed-files.outputs.changed-files, 'examples/models/muse-glimmer') || + contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda.yml') || + needs.run-decision.outputs.is-full-run == 'true' + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read + with: + timeout: 120 + runner: mt-l-x86aavx2-11-41-a10g + gpu-arch-type: cuda + gpu-arch-version: "13.0" + submodules: recursive + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + script: | + set -eux + bash ./install_executorch.sh + conda install -y -c conda-forge 'libstdcxx-ng>=12' + export LD_LIBRARY_PATH=/opt/conda/lib:$LD_LIBRARY_PATH + + python -m unittest discover -s .ci/scripts/tests -p 'test_muse_glimmer_batching_ci.py' -v + python -m unittest discover -s .ci/scripts/tests -p 'test_muse_glimmer_batching_workload.py' -v + + # The normal serving binaries and the opt-in correctness tool share + # CudaExecutor; MG preparation tests also run on Linux. + make muse-glimmer-batching-cuda + BATCHING_BIN_DIR="${PWD}/cmake-out/examples/models/muse-glimmer-batching" + cmake -S "${PWD}/examples/models/muse-glimmer" -B "${BATCHING_BIN_DIR}" \ + -DMUSE_GLIMMER_BUILD_BATCHING_TESTS=ON + bash .ci/scripts/build_muse_glimmer_batching_smoke.sh "${BATCHING_BIN_DIR}" + cmake --build "${BATCHING_BIN_DIR}" --target muse_glimmer_batching_test --parallel + ctest --test-dir "${BATCHING_BIN_DIR}" \ + -R '^muse_glimmer_batching$' --output-on-failure --no-tests=error + test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_runner" + test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_worker" + test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_smoke" + + # The batching export on a tiny model, and the CLI's backend dispatch. + pip install gguf + python -m pytest -v -o "addopts=" \ + examples/models/muse-glimmer/tests/test_cuda_batching_pipeline.py \ + examples/models/muse-glimmer/tests/test_checkpoint_context.py + unittest-cuda-quantized-gemm: name: unittest-cuda-quantized-gemm (${{ matrix.shard }}) needs: [changed-files, run-decision] @@ -1044,6 +1108,12 @@ jobs: name: test-muse-glimmer-cuda-e2e-${{ matrix.variant }}-${{ matrix.mode }} needs: [changed-files, run-decision] if: | + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/build_muse_glimmer_batching_smoke.sh') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/muse_glimmer_batching_smoke.cpp') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/muse_glimmer_batching_workload.h') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/test_muse_glimmer_batching_ci.py') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/test_muse_glimmer_batching_workload.py') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/tests/muse_glimmer_batching_workload_test.cpp') || contains(needs.changed-files.outputs.changed-files, 'backends/cuda') || contains(needs.changed-files.outputs.changed-files, 'backends/aoti') || contains(needs.changed-files.outputs.changed-files, 'extension/cuda') || @@ -1063,6 +1133,8 @@ jobs: include: - variant: kquant-17gb mode: solo-text + - variant: kquant-17gb + mode: solo-text-batching - variant: kquant-17gb mode: dflash-image - variant: kquant-dynamic diff --git a/.lintrunner.toml b/.lintrunner.toml index 89e5801db63..fab27e7a7c2 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -447,6 +447,7 @@ exclude_patterns = [ # Zephyr sample application can use std::vector in the sample code 'zephyr/samples/**', 'scripts/**', + '**/.ci/**', 'third-party/**', 'util/**', ] diff --git a/examples/models/muse-glimmer/CMakeLists.txt b/examples/models/muse-glimmer/CMakeLists.txt index 03b8f574b4d..70e24f3d61c 100644 --- a/examples/models/muse-glimmer/CMakeLists.txt +++ b/examples/models/muse-glimmer/CMakeLists.txt @@ -197,6 +197,12 @@ option(MUSE_GLIMMER_BUILD_BATCHING_TESTS "Build preparation/materialization tests without the batching executor" OFF ) +option(MUSE_GLIMMER_BUILD_CI_SMOKE + "Build the CI-only batching correctness workload" OFF +) +if(MUSE_GLIMMER_BUILD_CI_SMOKE AND NOT MUSE_GLIMMER_BUILD_BATCHING) + message(FATAL_ERROR "CI smoke requires MUSE_GLIMMER_BUILD_BATCHING=ON") +endif() if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS) if(EXECUTORCH_BUILD_CUDA) set(_batching_backend cuda) @@ -319,6 +325,24 @@ if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS) executorch_target_copy_mlx_metallib(muse_glimmer_batching_runner) executorch_target_copy_mlx_metallib(muse_glimmer_batching_worker) endif() + if(MUSE_GLIMMER_BUILD_CI_SMOKE) + add_executable( + muse_glimmer_batching_smoke + ${EXECUTORCH_ROOT}/.ci/scripts/muse_glimmer_batching_smoke.cpp + ) + target_link_libraries( + muse_glimmer_batching_smoke PRIVATE muse_glimmer_batching_runtime + ) + if(_batching_backend STREQUAL "cuda") + if(NOT APPLE AND NOT MSVC) + target_link_options( + muse_glimmer_batching_smoke PRIVATE "LINKER:--export-dynamic" + ) + endif() + else() + executorch_target_copy_mlx_metallib(muse_glimmer_batching_smoke) + endif() + endif() endif() if(MUSE_GLIMMER_BUILD_BATCHING_TESTS)