diff --git a/README.md b/README.md index 64f999206..c660d56af 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,7 @@ curl -s http://127.0.0.1:8216/v1/chat/completions \ | Environment variables | [Environment reference](server/docs/ENVIRONMENT.md) | | Server internals | [Architecture](server/docs/ARCHITECTURE.md) | | Client integration and qualification | [Harness guide](harness/README.md) | +| Server engine components | [Engine components](server/docs/ENGINE_COMPONENTS.md) | Benchmarks stay with each implementation: [DFlash](server/RESULTS.md), [PFlash](optimizations/pflash/), [Spark](optimizations/spark/), [KVFlash](optimizations/kvflash/), and [Megakernel](optimizations/megakernel/). diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 0b237c94a..decf12a9b 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -541,12 +541,14 @@ add_library(dflash_common STATIC src/common/backend_precision.cpp src/common/daemon_loop.cpp src/common/gguf_inspect.cpp + src/common/backend_plan.cpp src/common/backend_factory.cpp src/common/feature_gate.cpp src/placement/placement_config.cpp src/common/layer_split_utils.cpp src/common/ddtree.cpp src/common/peer_access.cpp + src/engine/luce_engine.cpp # ── Server components (tokenizer, chat template) ── src/server/tokenizer.cpp src/server/chat_template.cpp @@ -1657,6 +1659,8 @@ if(DFLASH27B_TESTS) set(_server_unit_sources test/test_unit_main.cpp test/test_server_unit.cpp + test/test_generation.cpp + test/test_luce_engine.cpp test/test_anchor_params.cpp test/test_derived_scalars.cpp test/test_adaptive_keep_ratio.cpp @@ -1690,6 +1694,7 @@ if(DFLASH27B_TESTS) endif() add_executable(test_server_unit ${_server_unit_sources}) target_sources(test_server_unit PRIVATE + src/engine/generation.cpp src/server/http_server.cpp src/server/scheduler.cpp src/server/model_card.cpp @@ -1717,6 +1722,9 @@ if(DFLASH27B_TESTS) target_link_libraries(test_server_unit PRIVATE CURL::libcurl) endif() target_link_libraries(test_server_unit PRIVATE dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET}) + if(UNIX) + target_link_libraries(test_server_unit PRIVATE pthread) + endif() if(DFLASH27B_GPU_BACKEND STREQUAL "cuda") find_package(CUDAToolkit REQUIRED) target_link_libraries(test_server_unit PRIVATE CUDA::cudart) @@ -1816,16 +1824,18 @@ if(DFLASH27B_TESTS) list(APPEND _raw_unit_test_targets test_model_smoke) endif() - # Feature/architecture gate tests. check_feature_compatibility(), - # collect_feature_warnings() and the capability table are pure functions, - # so this target deliberately compiles only feature_gate.cpp and - # placement_config.cpp — no dflash_common, no ggml, no GPU toolkit. That - # keeps a gate rule testable in seconds instead of behind a full backend - # build, which is the whole reason these tests do not live in + # Backend planning and feature/architecture gate tests. The plan builder, + # check_feature_compatibility(), collect_feature_warnings(), and the + # capability table are pure policy, so this target compiles only their + # sources and placement_config.cpp. It needs no dflash_common, ggml, or GPU + # toolkit. This keeps policy rules testable in seconds instead of behind a + # full backend build, which is why these tests do not live in # test_server_unit. if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_feature_gate.cpp") add_executable(test_feature_gate test/test_feature_gate.cpp) target_sources(test_feature_gate PRIVATE + test/test_backend_plan.cpp + src/common/backend_plan.cpp src/common/feature_gate.cpp src/placement/placement_config.cpp) target_include_directories(test_feature_gate PRIVATE diff --git a/server/docs/ENGINE_COMPONENTS.md b/server/docs/ENGINE_COMPONENTS.md new file mode 100644 index 000000000..c87a9581b --- /dev/null +++ b/server/docs/ENGINE_COMPONENTS.md @@ -0,0 +1,251 @@ +# Server engine components + +This document describes the server engine structure implemented in the current +code. It is a reference, not a roadmap. + +## Ownership overview + +```text +server_main + ├─ builds BackendPlan + ├─ creates ModelBackend + ├─ transfers ModelBackend ownership to LuceEngine + └─ constructs HttpServer, which borrows LuceEngine + +LuceEngine + ├─ owns ModelBackend + ├─ owns the serving thread + └─ selects one serving loop + ├─ serial: HttpServer::worker_loop() + └─ concurrent: HttpServer::scheduler_loop(SeqEngine &) + +HttpServer + ├─ owns sockets and HTTP protocol state + ├─ owns the current ServerJob queue + ├─ prepares model-ready GenerateRequest values + └─ formats tokens and terminal results for clients +``` + +The backend outlives `HttpServer` because `server_main` constructs +`LuceEngine` before the server. `HttpServer` stops and joins the serving loop +before its transport and cache state is destroyed. + +## Components + +### `BackendPlan` + +`BackendPlan` is the validated, normalized input to backend construction. Its +groups describe the effective model, placement, cache, speculation, runtime, +and architecture-specific values selected during startup. + +The backend factory accepts a plan, projects its values into the selected +architecture config, and returns one owned `ModelBackend`. Architecture +configs own persistent strings; they do not retain pointers into the plan. + +Files: + +- `server/src/common/backend_plan.cpp` +- `server/src/common/backend_plan_internal.h` +- `server/src/common/backend_factory.{h,cpp}` + +### `ModelBackend` + +`ModelBackend` is the common model-resource interface implemented by each +architecture adapter. A concrete backend owns weights, caches, snapshots, +model-specific execution state, and any architecture-specific helpers. + +Its complete-request generation operations remain the execution mechanism for +serial serving. A backend that supports continuous batching also exposes a +borrowed `SeqEngine`; that object is owned by the backend and remains valid for +the backend lifetime. + +File: `server/src/common/model_backend.h` + +### `LuceEngine` + +`LuceEngine` is the runtime owner. It owns exactly one `ModelBackend` and at +most one serving thread. + +Its current public lifecycle is: + +```cpp +explicit LuceEngine(std::unique_ptr backend); + +ModelBackend & backend() noexcept; +bool start_serving(ServingLoops loops, bool allow_concurrent); +void stop_serving(); +``` + +`start_serving()` selects the concurrent loop only when both conditions hold: + +- the caller permits concurrent local serving; and +- the backend exposes a `SeqEngine`. + +Otherwise it starts the serial loop. The selected callback is owned by the +worker thread. `stop_serving()` requests shutdown and joins that thread before +returning. Destruction also calls `stop_serving()` and then destroys the owned +backend, whose concrete destructor performs backend shutdown. + +Files: `server/src/engine/luce_engine.{h,cpp}` + +### `HttpServer` + +`HttpServer` owns the HTTP-facing state: + +- listen and client sockets; +- request parsing and response formatting; +- SSE state and client-disconnect detection; +- tokenizer-dependent request preparation; +- prefix-cache policy and server status; +- the current intrusive `ServerJob` queue. + +It borrows `LuceEngine` and, through it, a `ModelBackend`. The references are +valid for the complete `HttpServer` lifetime. + +At startup the server supplies its established serial and concurrent loops to +`LuceEngine`. Upstream PFlash forwarding disables concurrent local serving, +so the serial worker remains selected for that configuration. + +Files: + +- `server/src/server/http_server.{h,cpp}` +- `server/src/server/scheduler.cpp` + +### `GenerateRequest` and `GenerateResult` + +`GenerateRequest` is the model-ready input shared by backend execution paths. +It owns every token sequence that may be retained during generation: + +- prompt tokens; +- speculative hint tokens; +- stall-detection sequences; +- thinking-budget close tokens. + +`GenerateResult` owns the completed token vector, timings, typed failure, and +generation metadata. Neither type contains HTTP or JSON state. + +File: `server/src/common/generation_types.h` + +### Generation channel + +The generation channel provides transport-independent request and result +lifetime types: + +```text +GenerationQueue::submit(GenerateRequest) + -> Generation consumer handle + +GenerationQueue::next() + -> GenerateRequest + GenerationSource + +GenerationSource + -> TokenBatch + -> coalesced GenerationProgress + -> one GenerateCompleted result +``` + +`Generation` and `GenerationSource` are move-only. Dropping an unfinished +consumer cancels that generation. Losing its producer completes the consumer +with a typed failure. The queue bounds both waiting requests and buffered +tokens; shutdown rejects new requests and completes every live channel. + +These channel types are implemented and tested. The live HTTP path still uses +`ServerJob` and the two existing serving loops, so the channel is not yet the +HTTP submission path. + +Files: + +- `server/src/engine/generation.{h,cpp}` +- `server/test/test_generation.cpp` + +### `SeqEngine` + +`SeqEngine` is the model-side capability used by concurrent serving. It owns +slot allocation, per-sequence model state, KV capacity, batched prefill and +decode execution, sampling, and retirement. + +`HttpServer::scheduler_loop()` currently owns admission order, fair prefill +selection, cancellation, response construction, and non-blocking delivery. +All `SeqEngine` calls come from the single serving thread owned by +`LuceEngine`. + +File: `server/src/common/concurrency/seq_engine.h` + +## Runtime flow + +### Startup + +```text +CLI arguments + -> BackendPlan::build() + -> create_backend(plan) + -> LuceEngine(std::move(backend)) + -> HttpServer(engine, tokenizer, config) + -> LuceEngine::start_serving(...) +``` + +Backend validation and normalization happen before construction. Reporting, +tokenizer setup, and backend construction read the same effective plan values. + +### Serial request + +```text +client thread + -> enqueue ServerJob +serving thread + -> worker_loop() + -> prepare prompt, cache state, GenerateRequest, and callbacks + -> ModelBackend::generate() or restore_and_generate() + -> stream or format GenerateResult + -> complete ServerJob +client thread + -> close request connection +``` + +Only the serving thread mutates model execution state. + +### Concurrent request + +```text +client thread + -> enqueue ServerJob +serving thread + -> scheduler_loop(SeqEngine &) + -> admit requests into model slots + -> select bounded prefill work and all live decode rows + -> SeqEngine::step() + -> buffer and flush client output without blocking other slots + -> retire completed, failed, or cancelled slots +``` + +Admission order and slot identity remain stable until retirement. A slow or +disconnected client cannot block model progress for other live slots. + +### Shutdown + +```text +HttpServer::shutdown() + -> set stopping flag and wake the request queue + -> LuceEngine::stop_serving() + -> invoke request_stop callback + -> join serving thread + -> close SSE clients + -> drain remaining ServerJob values + -> release server-owned cache and transport state +``` + +`stop_serving()` is idempotent and serializes stop/restart through the full +worker join. `HttpServer` also calls it when `run()` exits. `LuceEngine` +destroys the backend only after the serving thread has joined; the concrete +backend destructor owns its single shutdown call. + +## Current boundary + +The ownership boundary is active: `LuceEngine` owns the backend and execution +thread, while `HttpServer` owns transport. Request coordination has not fully +crossed that boundary yet: `ServingLoops` and the `ServerJob` queue connect the +runtime owner to the existing HTTP worker and scheduler. + +The generation channel is therefore an available engine component, not yet a +live server entry point. This distinction is reflected in the types and tests +rather than hidden behind a second generation path. diff --git a/server/src/bailingmoe3/bailingmoe3_backend.cpp b/server/src/bailingmoe3/bailingmoe3_backend.cpp index feb7f0367..1085966a3 100644 --- a/server/src/bailingmoe3/bailingmoe3_backend.cpp +++ b/server/src/bailingmoe3/bailingmoe3_backend.cpp @@ -1,13 +1,14 @@ #include "bailingmoe3_backend.h" #include +#include namespace dflash::common { namespace { -Qwen35Config make_qwen_runtime_config(const BailingMoe3Config & cfg) { +Qwen35Config make_qwen_runtime_config(BailingMoe3Config cfg) { Qwen35Config runtime; - runtime.target_path = cfg.model_path; + runtime.target_path = std::move(cfg.model_path); runtime.device = cfg.device; runtime.stream_fd = cfg.stream_fd; // The Ling baseline uses the ordinary contiguous F16/Q4 KV cache and the @@ -22,8 +23,8 @@ Qwen35Config make_qwen_runtime_config(const BailingMoe3Config & cfg) { } // namespace -BailingMoe3Backend::BailingMoe3Backend(const BailingMoe3Config & cfg) - : Qwen35Backend(make_qwen_runtime_config(cfg)) {} +BailingMoe3Backend::BailingMoe3Backend(BailingMoe3Config cfg) + : Qwen35Backend(make_qwen_runtime_config(std::move(cfg))) {} bool BailingMoe3Backend::load_target_model(ggml_backend_t backend, TargetWeights & out) { diff --git a/server/src/bailingmoe3/bailingmoe3_backend.h b/server/src/bailingmoe3/bailingmoe3_backend.h index 61e775753..f2953570e 100644 --- a/server/src/bailingmoe3/bailingmoe3_backend.h +++ b/server/src/bailingmoe3/bailingmoe3_backend.h @@ -2,20 +2,22 @@ #include "qwen35_backend.h" +#include + namespace dflash::common { // Configuration intentionally exposes only the features the first native // Ling backend implements. Speculative decode and expert offload can be added // after the autoregressive path has a logits-equivalent baseline. struct BailingMoe3Config { - const char * model_path = nullptr; + std::string model_path; DevicePlacement device; int stream_fd = -1; }; class BailingMoe3Backend final : public Qwen35Backend { public: - explicit BailingMoe3Backend(const BailingMoe3Config & cfg); + explicit BailingMoe3Backend(BailingMoe3Config cfg); void print_ready_banner() const override; bool supports_dflash_spec_decode() const override { return false; } diff --git a/server/src/common/backend_args.h b/server/src/common/backend_args.h index 611dd8d77..1067ab3e2 100644 --- a/server/src/common/backend_args.h +++ b/server/src/common/backend_args.h @@ -1,11 +1,13 @@ // Raw backend construction arguments. // // This contains only caller-requested configuration. Runtime facts derived -// from the model or compiled binary belong in ResolvedBackendPlan instead. +// from the model or compiled binary belong in BackendPlan instead. #pragma once #include +#include +#include #include "placement/draft_residency.h" #include "placement/placement_config.h" @@ -15,35 +17,40 @@ namespace dflash::common { -// Server-owned features that participate in backend admission even though -// they are not consumed by ModelBackend construction itself. Keep these -// separate from BackendArgs so the factory API remains usable by callers that -// do not run the HTTP server. -struct BackendFeatureConfig { +enum class KvFlashRequest { + Off, + Auto, + Fixed, +}; + +// Server-owned facts that participate in backend admission without becoming +// backend construction arguments. This is the only projection the HTTP server +// may pass into backend preparation. +struct BackendAdmissionContext { bool pflash_enabled = false; bool pflash_drafter_configured = false; DraftResidencyPolicy draft_residency = DraftResidencyPolicy::Auto; - // MoE-only server features. Recorded here so the gate can report them as - // inert on a dense architecture; both are applied via env vars at parse - // time rather than through BackendArgs. - bool routing_stats_requested = false; // --freq / --collect-routing - bool adaptive_experts_requested = false; // --adaptive-experts + // Automatic sizing remains backend-owned because only the initialized + // backend has the VRAM budget. Fixed pools can participate in admission. + KvFlashRequest kvflash = KvFlashRequest::Off; - // A fixed KVFlash pool requested through DFLASH_KVFLASH. "auto" is - // resolved later by the backend because only it has the VRAM budget needed - // to know whether a pool will actually be active. - bool kvflash_enabled = false; + bool kvflash_requested() const { + return kvflash != KvFlashRequest::Off; + } + bool fixed_kvflash_requested() const { + return kvflash == KvFlashRequest::Fixed; + } }; -// A superset of all per-architecture config fields. The factory reads only -// those relevant to the resolved architecture; unused fields are ignored. +// A superset of all per-architecture config fields. Preparation projects only +// the effective fields into BackendPlan's concern-specific snapshots. struct BackendArgs { // Required - const char * model_path = nullptr; // target .gguf + std::string model_path; // target .gguf // Optional: speculative decode draft model (qwen35 only) - const char * draft_path = nullptr; + std::optional draft_path; // Device placement DevicePlacement device; @@ -82,13 +89,22 @@ struct BackendArgs { bool fast_rollback = true; bool seq_verify = false; bool specla_mode = false; + int specla_top_k = 4; + bool specla_top_k_explicit = false; bool ddtree_mode = false; int ddtree_budget = 22; float ddtree_temp = 1.0f; bool ddtree_chain_seed = true; float ddtree_tau = std::numeric_limits::infinity(); + bool ddtree_tau_explicit = false; int verify_width = 0; // chain spec verify width; 0 = adaptive bool use_feature_mirror = false; + + // MoE backend requests. The server currently realizes these through + // environment variables, but admission still treats them as explicit + // operator input rather than server-owned context. + bool routing_stats_requested = false; + bool adaptive_experts_requested = false; }; } // namespace dflash::common diff --git a/server/src/common/backend_factory.cpp b/server/src/common/backend_factory.cpp index d462be899..1feff37f7 100644 --- a/server/src/common/backend_factory.cpp +++ b/server/src/common/backend_factory.cpp @@ -1,8 +1,10 @@ // Backend factory implementation. #include "backend_factory.h" -#include "feature_gate.h" +#include "backend_plan_internal.h" #include "gguf_inspect.h" +#include "model_capabilities.h" +#include "platform_env.h" #include "qwen35_backend.h" #include "qwen35moe_backend.h" @@ -19,7 +21,9 @@ #include #include +#include #include +#include namespace dflash::common { @@ -111,152 +115,69 @@ DFLASH_CHECK_ARCH_OPTION("qwen35", Qwen35Config, Qwen35LayerSplitAdapterConfig, #undef DFLASH_CHECK_ARCH #undef DFLASH_CHECK_ARCH_OPTION -PlacementBackend resolve_target_backend( - const BackendArgs & args, - PlacementBackend compiled_backend) { - return args.device.backend == PlacementBackend::Auto - ? compiled_backend - : args.device.backend; -} - -} // namespace - -std::string detect_arch(const char * model_path) { - auto info = inspect_gguf_model_info(model_path); - return info.arch; -} - -BackendPreparation prepare_backend( - const BackendArgs & args, - const BackendFeatureConfig & features) { - BackendPreparation preparation; - if (!args.model_path) { - preparation.error = BackendPreparationError::InvalidRequest; - preparation.message = "model_path is null"; - return preparation; - } - - preparation.plan.model_path_ = args.model_path; - preparation.plan.features_ = features; - preparation.plan.compiled_backend_ = compiled_placement_backend(); - preparation.plan.target_backend_ = resolve_target_backend( - args, preparation.plan.compiled_backend_); - preparation.plan.model_ = inspect_gguf_model_info(args.model_path); - - if (preparation.plan.arch().empty()) { - preparation.error = BackendPreparationError::ModelInspection; - preparation.message = - "failed to detect architecture from " + - preparation.plan.model_path(); - return preparation; - } - - // A model this binary cannot construct is a property of the model, not of - // the requested feature set — same category (and same exit status) as an - // unreadable GGUF. Checking it here means the arch-dependent rules below - // only ever run against an architecture the capability table describes. - if (!arch_is_supported(preparation.plan.arch())) { - preparation.error = BackendPreparationError::ModelInspection; - preparation.message = - "unsupported model architecture '" + preparation.plan.arch() + - "' in " + preparation.plan.model_path(); - return preparation; - } - - preparation.message = check_feature_compatibility( - args, - preparation.plan.features(), - preparation.plan.arch(), - preparation.plan.target_backend(), - preparation.plan.compiled_backend()); - if (!preparation.message.empty()) { - preparation.error = BackendPreparationError::FeatureCompatibility; - return preparation; - } - - preparation.warnings = collect_feature_warnings( - args, preparation.plan.features(), preparation.plan.arch()); - return preparation; -} - -std::unique_ptr create_backend(const BackendArgs & args) { - const BackendPreparation preparation = prepare_backend(args); - if (!preparation.ok()) { - std::fprintf(stderr, "[backend_factory] %s\n", - preparation.message.c_str()); - return nullptr; - } - for (const std::string & warning : preparation.warnings) { - std::fprintf(stderr, "[backend_factory] warning: %s\n", - warning.c_str()); - } - return create_backend(args, preparation.plan); -} - -std::unique_ptr create_backend( - const BackendArgs & args, - const ResolvedBackendPlan & plan) { - if (!args.model_path) { - std::fprintf(stderr, "[backend_factory] model_path is null\n"); - return nullptr; - } - if (plan.model_path() != args.model_path) { - std::fprintf(stderr, - "[backend_factory] resolved plan does not match model_path %s\n", - args.model_path); - return nullptr; - } - if (plan.compiled_backend() != compiled_placement_backend() || - plan.target_backend() != - resolve_target_backend(args, plan.compiled_backend())) { - std::fprintf(stderr, - "[backend_factory] resolved plan does not match target placement\n"); - return nullptr; - } - +// Every config retained by a backend or adapter owns its path storage. +// Borrowed C strings are confined to immediate loader and C API calls. +static_assert(std::is_same_v< + decltype(Qwen35Config{}.target_path), std::string>); +static_assert(std::is_same_v< + decltype(Qwen35Config{}.draft_path), std::optional>); +static_assert(std::is_same_v< + decltype(Qwen35LayerSplitAdapterConfig{}.target_path), std::string>); +static_assert(std::is_same_v< + decltype(Qwen35LayerSplitAdapterConfig{}.draft_path), + std::optional>); +static_assert(std::is_same_v< + decltype(BailingMoe3Config{}.model_path), std::string>); +static_assert(std::is_same_v< + decltype(LagunaBackendArgs{}.target_path), std::string>); +static_assert(std::is_same_v< + decltype(LagunaBackendArgs{}.draft_path), std::string>); +static_assert(std::is_same_v< + decltype(LagunaLayerSplitAdapterConfig{}.target_path), std::string>); +static_assert(std::is_same_v< + decltype(Qwen3BackendConfig{}.model_path), std::string>); +static_assert(std::is_same_v< + decltype(Gemma4BackendConfig{}.model_path), std::string>); +static_assert(std::is_same_v< + decltype(Gemma4BackendConfig{}.draft_path), + std::optional>); +static_assert(std::is_same_v< + decltype(Gemma4LayerSplitAdapterConfig{}.target_path), std::string>); +static_assert(std::is_same_v< + decltype(DeepSeek4BackendConfig{}.model_path), std::string>); +static_assert(std::is_same_v< + decltype(DeepSeek4LayerSplitAdapterConfig{}.target_path), std::string>); + +std::unique_ptr construct_backend( + const BackendPlan & plan) { + const BackendPlan::Model & model = plan.model(); + const BackendPlan::Placement & placement = plan.placement(); + const BackendPlan::Cache & cache = plan.cache(); + const BackendPlan::Speculation & speculation = plan.speculation(); + const BackendPlan::Execution & execution = plan.execution(); + const BackendPlan::DeepSeek4 & deepseek4 = plan.deepseek4(); const std::string & arch = plan.arch(); - if (arch.empty()) { - std::fprintf(stderr, - "[backend_factory] failed to detect architecture from %s\n", - args.model_path); - return nullptr; - } - - std::fprintf(stderr, "[backend_factory] detected arch=%s\n", arch.c_str()); - - // Recheck at the construction boundary in case raw arguments changed - // after preparation. No entry point can dispatch an incoherent request. - const std::string incompatible = check_feature_compatibility( - args, - plan.features(), - arch, - plan.target_backend(), - plan.compiled_backend()); - if (!incompatible.empty()) { - std::fprintf(stderr, "[backend_factory] %s\n", incompatible.c_str()); - return nullptr; - } - if (arch == "qwen35") { - if (args.device.is_layer_split()) { + if (placement.target.is_layer_split()) { Qwen35LayerSplitAdapterConfig cfg; - cfg.target_path = args.model_path; - cfg.draft_path = args.draft_path; - cfg.device = args.device; - cfg.draft_gpu = args.draft_device.gpu; - cfg.remote_draft = args.remote_draft; - cfg.remote_target_shard = args.remote_target_shard; - cfg.fa_window = args.fa_window; - cfg.kq_stride_pad = args.kq_stride_pad; - cfg.draft_swa_window = args.draft_swa_window; - cfg.draft_ctx_max = args.draft_ctx_max; - cfg.chunk = args.chunk; - cfg.max_verify_tokens = args.ddtree_mode - ? std::max(DFLASH27B_DRAFT_BLOCK_SIZE, args.ddtree_budget + 1) + cfg.target_path = model.path; + cfg.draft_path = speculation.draft_path; + cfg.device = placement.target; + cfg.draft_gpu = placement.draft.gpu; + cfg.remote_draft = placement.remote_draft; + cfg.remote_target_shard = placement.remote_target_shard; + cfg.fa_window = cache.fa_window; + cfg.kq_stride_pad = cache.kq_stride_pad; + cfg.draft_swa_window = cache.draft_swa_window; + cfg.draft_ctx_max = cache.draft_ctx_max; + cfg.chunk = execution.chunk; + cfg.max_verify_tokens = speculation.ddtree_mode + ? std::max(DFLASH27B_DRAFT_BLOCK_SIZE, speculation.ddtree_budget + 1) : DFLASH27B_DRAFT_BLOCK_SIZE; - cfg.run_dflash = args.draft_path != nullptr; + cfg.run_dflash = speculation.draft_path.has_value(); - auto adapter = std::make_unique(cfg); + auto adapter = std::make_unique( + std::move(cfg)); auto backend = std::make_unique(std::move(adapter)); if (!backend->init()) { std::fprintf(stderr, "[backend_factory] LayerSplitBackend(qwen35) init failed\n"); @@ -266,30 +187,30 @@ std::unique_ptr create_backend( } Qwen35Config cfg; - cfg.target_path = args.model_path; - cfg.draft_path = args.draft_path; - cfg.device = args.device; - cfg.draft_gpu = args.draft_device.gpu; - cfg.remote_draft = args.remote_draft; - cfg.stream_fd = args.stream_fd; - cfg.fa_window = args.fa_window; - cfg.paged_attention = args.paged_attention; - cfg.max_concurrency = args.max_concurrency; - cfg.kv_pool_tokens = args.kv_pool_tokens; - cfg.kq_stride_pad = args.kq_stride_pad; - cfg.draft_block_size = args.draft_block_size; - cfg.draft_swa_window = args.draft_swa_window; - cfg.draft_ctx_max = args.draft_ctx_max; - cfg.fast_rollback = args.fast_rollback; - cfg.seq_verify = args.seq_verify; - cfg.ddtree_mode = args.ddtree_mode; - cfg.ddtree_budget = args.ddtree_budget; - cfg.ddtree_temp = args.ddtree_temp; - cfg.ddtree_chain_seed = args.ddtree_chain_seed; - cfg.ddtree_tau = args.ddtree_tau; - cfg.use_feature_mirror = args.use_feature_mirror; - - auto backend = std::make_unique(cfg); + cfg.target_path = model.path; + cfg.draft_path = speculation.draft_path; + cfg.device = placement.target; + cfg.draft_gpu = placement.draft.gpu; + cfg.remote_draft = placement.remote_draft; + cfg.stream_fd = execution.stream_fd; + cfg.fa_window = cache.fa_window; + cfg.paged_attention = cache.paged_attention; + cfg.max_concurrency = cache.max_concurrency; + cfg.kv_pool_tokens = cache.kv_pool_tokens; + cfg.kq_stride_pad = cache.kq_stride_pad; + cfg.draft_block_size = speculation.draft_block_size; + cfg.draft_swa_window = cache.draft_swa_window; + cfg.draft_ctx_max = cache.draft_ctx_max; + cfg.fast_rollback = speculation.fast_rollback; + cfg.seq_verify = speculation.seq_verify; + cfg.ddtree_mode = speculation.ddtree_mode; + cfg.ddtree_budget = speculation.ddtree_budget; + cfg.ddtree_temp = speculation.ddtree_temp; + cfg.ddtree_chain_seed = speculation.ddtree_chain_seed; + cfg.ddtree_tau = speculation.ddtree_tau; + cfg.use_feature_mirror = speculation.use_feature_mirror; + + auto backend = std::make_unique(std::move(cfg)); if (!backend->init()) { std::fprintf(stderr, "[backend_factory] Qwen35Backend init failed\n"); return nullptr; @@ -298,25 +219,25 @@ std::unique_ptr create_backend( } else if (arch == "qwen35moe") { Qwen35Config cfg; - cfg.target_path = args.model_path; - cfg.draft_path = args.draft_path; - cfg.device = args.device; - cfg.draft_gpu = args.draft_device.gpu; - cfg.stream_fd = args.stream_fd; - cfg.fa_window = args.fa_window; - cfg.kq_stride_pad = args.kq_stride_pad; - cfg.draft_swa_window = args.draft_swa_window; - cfg.draft_ctx_max = args.draft_ctx_max; - cfg.fast_rollback = args.fast_rollback; - cfg.seq_verify = args.seq_verify; - cfg.ddtree_mode = args.ddtree_mode; - cfg.ddtree_budget = args.ddtree_budget; - cfg.ddtree_temp = args.ddtree_temp; - cfg.ddtree_chain_seed = args.ddtree_chain_seed; - cfg.ddtree_tau = args.ddtree_tau; - cfg.use_feature_mirror = args.use_feature_mirror; - - auto backend = std::make_unique(cfg); + cfg.target_path = model.path; + cfg.draft_path = speculation.draft_path; + cfg.device = placement.target; + cfg.draft_gpu = placement.draft.gpu; + cfg.stream_fd = execution.stream_fd; + cfg.fa_window = cache.fa_window; + cfg.kq_stride_pad = cache.kq_stride_pad; + cfg.draft_swa_window = cache.draft_swa_window; + cfg.draft_ctx_max = cache.draft_ctx_max; + cfg.fast_rollback = speculation.fast_rollback; + cfg.seq_verify = speculation.seq_verify; + cfg.ddtree_mode = speculation.ddtree_mode; + cfg.ddtree_budget = speculation.ddtree_budget; + cfg.ddtree_temp = speculation.ddtree_temp; + cfg.ddtree_chain_seed = speculation.ddtree_chain_seed; + cfg.ddtree_tau = speculation.ddtree_tau; + cfg.use_feature_mirror = speculation.use_feature_mirror; + + auto backend = std::make_unique(std::move(cfg)); if (!backend->init()) { std::fprintf(stderr, "[backend_factory] Qwen35MoeBackend init failed\n"); return nullptr; @@ -325,11 +246,11 @@ std::unique_ptr create_backend( } else if (arch == "bailingmoe3") { BailingMoe3Config cfg; - cfg.model_path = args.model_path; - cfg.device = args.device; - cfg.stream_fd = args.stream_fd; + cfg.model_path = model.path; + cfg.device = placement.target; + cfg.stream_fd = execution.stream_fd; - auto backend = std::make_unique(cfg); + auto backend = std::make_unique(std::move(cfg)); if (!backend->init()) { std::fprintf(stderr, "[backend_factory] BailingMoe3Backend init failed\n"); return nullptr; @@ -337,14 +258,15 @@ std::unique_ptr create_backend( return backend; } else if (arch == "laguna") { - if (args.device.is_layer_split()) { + if (placement.target.is_layer_split()) { LagunaLayerSplitAdapterConfig cfg; - cfg.target_path = args.model_path; - cfg.device = args.device; - cfg.remote_target_shard = args.remote_target_shard; - cfg.chunk = args.chunk; + cfg.target_path = model.path; + cfg.device = placement.target; + cfg.remote_target_shard = placement.remote_target_shard; + cfg.chunk = execution.chunk; - auto adapter = std::make_unique(cfg); + auto adapter = std::make_unique( + std::move(cfg)); auto backend = std::make_unique(std::move(adapter)); if (!backend->init()) { std::fprintf(stderr, "[backend_factory] LayerSplitBackend(laguna) init failed\n"); @@ -354,20 +276,20 @@ std::unique_ptr create_backend( } LagunaBackendArgs lcfg; - lcfg.target_path = args.model_path; - lcfg.draft_path = args.draft_path ? args.draft_path : ""; - lcfg.draft_gpu = args.draft_device.gpu; - lcfg.draft_ctx_max = args.draft_ctx_max; - lcfg.ddtree_mode = args.ddtree_mode; - lcfg.ddtree_budget = args.ddtree_budget; - lcfg.ddtree_temp = args.ddtree_temp; - lcfg.verify_width = args.verify_width; - lcfg.device = args.device; - lcfg.max_ctx = args.device.max_ctx; - lcfg.chunk = args.chunk; + lcfg.target_path = model.path; + lcfg.draft_path = speculation.draft_path.value_or(""); + lcfg.draft_gpu = placement.draft.gpu; + lcfg.draft_ctx_max = cache.draft_ctx_max; + lcfg.ddtree_mode = speculation.ddtree_mode; + lcfg.ddtree_budget = speculation.ddtree_budget; + lcfg.ddtree_temp = speculation.ddtree_temp; + lcfg.verify_width = speculation.verify_width; + lcfg.device = placement.target; + lcfg.max_ctx = placement.target.max_ctx; + lcfg.chunk = execution.chunk; // kv_type defaults to Q8_0 in LagunaBackendArgs - auto backend = std::make_unique(lcfg); + auto backend = std::make_unique(std::move(lcfg)); if (!backend->init()) { std::fprintf(stderr, "[backend_factory] LagunaBackend init failed\n"); return nullptr; @@ -376,12 +298,12 @@ std::unique_ptr create_backend( } else if (arch == "qwen3") { Qwen3BackendConfig qcfg; - qcfg.model_path = args.model_path; - qcfg.device = args.device; - qcfg.stream_fd = args.stream_fd; - qcfg.chunk = args.chunk; + qcfg.model_path = model.path; + qcfg.device = placement.target; + qcfg.stream_fd = execution.stream_fd; + qcfg.chunk = execution.chunk; - auto backend = std::make_unique(qcfg); + auto backend = std::make_unique(std::move(qcfg)); if (!backend->init()) { std::fprintf(stderr, "[backend_factory] Qwen3Backend init failed\n"); return nullptr; @@ -389,15 +311,16 @@ std::unique_ptr create_backend( return backend; } else if (arch == "gemma4") { - if (args.device.is_layer_split()) { + if (placement.target.is_layer_split()) { Gemma4LayerSplitAdapterConfig cfg; - cfg.target_path = args.model_path; - cfg.device = args.device; - cfg.remote_target_shard = args.remote_target_shard; - cfg.chunk = args.chunk; - cfg.fa_window = args.fa_window; - - auto adapter = std::make_unique(cfg); + cfg.target_path = model.path; + cfg.device = placement.target; + cfg.remote_target_shard = placement.remote_target_shard; + cfg.chunk = execution.chunk; + cfg.fa_window = cache.fa_window; + + auto adapter = std::make_unique( + std::move(cfg)); auto backend = std::make_unique(std::move(adapter)); if (!backend->init()) { std::fprintf(stderr, "[backend_factory] LayerSplitBackend(gemma4) init failed\n"); @@ -407,19 +330,19 @@ std::unique_ptr create_backend( } Gemma4BackendConfig gcfg; - gcfg.model_path = args.model_path; - gcfg.draft_path = args.draft_path; - gcfg.draft_gpu = args.draft_device.gpu; - gcfg.draft_ctx_max = args.draft_ctx_max; - gcfg.device = args.device; - gcfg.stream_fd = args.stream_fd; - gcfg.chunk = args.chunk; + gcfg.model_path = model.path; + gcfg.draft_path = speculation.draft_path; + gcfg.draft_gpu = placement.draft.gpu; + gcfg.draft_ctx_max = cache.draft_ctx_max; + gcfg.device = placement.target; + gcfg.stream_fd = execution.stream_fd; + gcfg.chunk = execution.chunk; // Gemma4Backend reads this into its cache (gemma4_backend.cpp) exactly // as the layer-split adapter does; leaving it unset silently dropped // --fa-window on single-device gemma4. - gcfg.fa_window = args.fa_window; + gcfg.fa_window = cache.fa_window; - auto backend = std::make_unique(gcfg); + auto backend = std::make_unique(std::move(gcfg)); if (!backend->init()) { std::fprintf(stderr, "[backend_factory] Gemma4Backend init failed\n"); return nullptr; @@ -433,20 +356,20 @@ std::unique_ptr create_backend( // A single local device uses the monolithic backend. Reserve the // layer-split adapter for explicit multi-device placement or remote // target shards. - if (!args.device.is_layer_split() && - !args.remote_target_shard.enabled()) { + if (!placement.target.is_layer_split() && + !placement.remote_target_shard.enabled()) { DeepSeek4BackendConfig cfg; - cfg.model_path = args.model_path; - cfg.device = args.device; - cfg.stream_fd = args.stream_fd; - cfg.max_ctx = args.device.max_ctx; - cfg.chunk = args.chunk; - cfg.expert_top_k = args.ds4_expert_top_k; - cfg.fused_decode = args.ds4_fused_decode; - cfg.fused_verify_f16_kv = args.ds4_fused_verify_f16_kv; - cfg.prefill_mode = args.ds4_prefill_mode; - - auto backend = std::make_unique(cfg); + cfg.model_path = model.path; + cfg.device = placement.target; + cfg.stream_fd = execution.stream_fd; + cfg.max_ctx = placement.target.max_ctx; + cfg.chunk = execution.chunk; + cfg.expert_top_k = deepseek4.expert_top_k; + cfg.fused_decode = deepseek4.fused_decode; + cfg.fused_verify_f16_kv = deepseek4.fused_verify_f16_kv; + cfg.prefill_mode = deepseek4.prefill_mode; + + auto backend = std::make_unique(std::move(cfg)); if (!backend->init()) { std::fprintf(stderr, "[backend_factory] DeepSeek4Backend init failed\n"); return nullptr; @@ -456,12 +379,13 @@ std::unique_ptr create_backend( // Explicit local splits and CUDA/HIP remote splits use the adapter. DeepSeek4LayerSplitAdapterConfig cfg; - cfg.target_path = args.model_path; - cfg.device = args.device; - cfg.remote_target_shard = args.remote_target_shard; - cfg.chunk = args.chunk; + cfg.target_path = model.path; + cfg.device = placement.target; + cfg.remote_target_shard = placement.remote_target_shard; + cfg.chunk = execution.chunk; - auto adapter = std::make_unique(cfg); + auto adapter = std::make_unique( + std::move(cfg)); auto backend = std::make_unique(std::move(adapter)); if (!backend->init()) { std::fprintf(stderr, "[backend_factory] LayerSplitBackend(deepseek4) init failed\n"); @@ -476,4 +400,49 @@ std::unique_ptr create_backend( } } +} // namespace + +BackendPreparation prepare_backend( + BackendArgs args, + BackendAdmissionContext admission) { + if (args.model_path.empty()) { + return BackendPreparationFailure{ + BackendPreparationError::InvalidRequest, + "model_path is empty", + {}}; + } + + GgufModelInfo model = inspect_gguf_model_info(args.model_path.c_str()); + return detail::BackendPlanBuilder::resolve( + std::move(args), + std::move(admission), + std::move(model), + compiled_placement_backend()); +} + +std::unique_ptr create_backend(const BackendPlan & plan) { + switch (plan.specla_environment_) { + case BackendPlan::SpeclaEnvironmentAction::Preserve: + break; + case BackendPlan::SpeclaEnvironmentAction::Enable: + set_environment_variable("DFLASH_SPECLA", "1", true); + if (plan.speculation_.specla_top_k_explicit) { + const std::string top_k = + std::to_string(plan.speculation_.specla_top_k); + set_environment_variable( + "DFLASH_SPECLA_TOPK", top_k.c_str(), true); + } + break; + case BackendPlan::SpeclaEnvironmentAction::Disable: + unset_environment_variable("DFLASH_SPECLA"); + break; + } + + std::fprintf( + stderr, + "[backend_factory] detected arch=%s\n", + plan.arch().c_str()); + return construct_backend(plan); +} + } // namespace dflash::common diff --git a/server/src/common/backend_factory.h b/server/src/common/backend_factory.h index bb3d71164..790d9668f 100644 --- a/server/src/common/backend_factory.h +++ b/server/src/common/backend_factory.h @@ -1,91 +1,147 @@ -// Backend factory — arch-detecting ModelBackend construction. +// Backend planning and arch-detecting ModelBackend construction. // -// Given a GGUF model path and placement options, inspects the file's -// `general.architecture` key and constructs the appropriate ModelBackend -// subclass (Qwen35Backend, LagunaBackend, Qwen3Backend, Gemma4Backend). -// -// This decouples backend creation from the daemon binary's argv parsing -// and allows both the daemon (test_dflash) and the new native server to -// share the same construction logic. +// BackendArgs is mutable input. prepare_backend() consumes it, resolves model +// and placement facts, normalizes backend policy, and returns an immutable +// BackendPlan. create_backend() accepts only that plan. #pragma once #include "backend_args.h" #include "gguf_inspect.h" -#include "model_capabilities.h" -#include "model_backend.h" +#include #include +#include #include +#include #include namespace dflash::common { -struct BackendPreparation; +namespace detail { +class BackendPlanBuilder; +} + +class BackendPlan; +struct ModelBackend; + +// The sole construction entry point. Architecture configs own any path data +// retained by the returned backend, so the plan may have an independent +// lifetime after construction. +std::unique_ptr create_backend(const BackendPlan & plan); -// Runtime facts resolved once from BackendArgs and shared by server admission -// and backend construction. Fields are private so callers cannot substitute -// an architecture that disagrees with model_path. -class ResolvedBackendPlan { +// The grouped field carriers are public so consumers can name their read-only +// views. Only the private plan builder can populate the enclosing plan. +class BackendPlan final { public: - const GgufModelInfo & model() const { return model_; } - const std::string & arch() const { return model_.arch; } - const std::string & model_path() const { return model_path_; } - PlacementBackend target_backend() const { return target_backend_; } - PlacementBackend compiled_backend() const { return compiled_backend_; } - const BackendFeatureConfig & features() const { return features_; } + struct Model { + std::string path; + GgufModelInfo metadata; + }; + + struct Placement { + DevicePlacement target; + DevicePlacement draft; + RemoteDraftConfig remote_draft; + RemoteTargetShardConfig remote_target_shard; + }; + + struct Cache { + int fa_window = 0; + bool paged_attention = false; + int max_concurrency = 1; + long long kv_pool_tokens = 0; + int kq_stride_pad = 32; + int draft_swa_window = 0; + int draft_ctx_max = 4096; + }; + + struct Speculation { + std::optional draft_path; + int draft_block_size = 0; + bool fast_rollback = true; + bool seq_verify = false; + bool specla_mode = false; + int specla_top_k = 4; + bool specla_top_k_explicit = false; + bool ddtree_mode = false; + int ddtree_budget = 22; + float ddtree_temp = 1.0f; + bool ddtree_chain_seed = true; + float ddtree_tau = std::numeric_limits::infinity(); + int verify_width = 0; + bool use_feature_mirror = false; + }; + + struct Execution { + int stream_fd = -1; + int chunk = 512; + }; + + struct DeepSeek4 { + PrefillAttentionMode prefill_mode = PrefillAttentionMode::Exact; + int expert_top_k = 0; + bool fused_decode = false; + bool fused_verify_f16_kv = false; + }; + + BackendPlan(BackendPlan &&) noexcept = default; + BackendPlan & operator=(BackendPlan &&) = delete; + BackendPlan(const BackendPlan &) = delete; + BackendPlan & operator=(const BackendPlan &) = delete; + + const Model & model() const { return model_; } + const Placement & placement() const { return placement_; } + const Cache & cache() const { return cache_; } + const Speculation & speculation() const { return speculation_; } + const Execution & execution() const { return execution_; } + const DeepSeek4 & deepseek4() const { return deepseek4_; } + const std::string & arch() const { return model_.metadata.arch; } + const std::vector & warnings() const { return warnings_; } private: - std::string model_path_; - GgufModelInfo model_; - BackendFeatureConfig features_; - PlacementBackend target_backend_ = PlacementBackend::Auto; - PlacementBackend compiled_backend_ = PlacementBackend::Auto; - - friend BackendPreparation prepare_backend( - const BackendArgs & args, - const BackendFeatureConfig & features); + enum class SpeclaEnvironmentAction { + Preserve, + Enable, + Disable, + }; + + BackendPlan() = default; + + Model model_; + Placement placement_; + Cache cache_; + Speculation speculation_; + Execution execution_; + DeepSeek4 deepseek4_; + std::vector warnings_; + SpeclaEnvironmentAction specla_environment_ = + SpeclaEnvironmentAction::Preserve; + + friend class detail::BackendPlanBuilder; + friend std::unique_ptr create_backend( + const BackendPlan & plan); }; enum class BackendPreparationError { - None, InvalidRequest, ModelInspection, FeatureCompatibility, }; -struct BackendPreparation { - ResolvedBackendPlan plan; - BackendPreparationError error = BackendPreparationError::None; +struct BackendPreparationFailure { + BackendPreparationError error; std::string message; - - // Accepted-but-inert options for the resolved architecture and placement. - // Populated only when ok(); the caller decides how to surface them. std::vector warnings; - - bool ok() const { return error == BackendPreparationError::None; } }; -// Resolve model metadata and compiled placement, then apply all cross-feature -// compatibility policy. This is the server's fail-fast factory entry point: -// server_main forwards the raw request and only handles the categorized result. +using BackendPreparation = + std::variant; + +// Consumes the mutable request. Successful preparation performs one GGUF +// inspection and returns the only value accepted by backend construction. BackendPreparation prepare_backend( - const BackendArgs & args, - const BackendFeatureConfig & features = {}); - -// ─── Factory function ─────────────────────────────────────────────────── -// Inspects model_path GGUF metadata, constructs the correct backend, and -// calls init(). Returns nullptr on failure (diagnostic printed to stderr). -std::unique_ptr create_backend(const BackendArgs & args); - -// Uses facts already resolved from the same BackendArgs. The factory verifies -// that the plan belongs to args.model_path before dispatching. -std::unique_ptr create_backend( - const BackendArgs & args, - const ResolvedBackendPlan & plan); - -// Returns the detected architecture string without creating a backend. -// Useful for early dispatch (e.g. printing which backend will be used). -std::string detect_arch(const char * model_path); + BackendArgs args, + BackendAdmissionContext admission = {}); } // namespace dflash::common diff --git a/server/src/common/backend_plan.cpp b/server/src/common/backend_plan.cpp new file mode 100644 index 000000000..5e23b22a6 --- /dev/null +++ b/server/src/common/backend_plan.cpp @@ -0,0 +1,176 @@ +#include "backend_plan_internal.h" + +#include "feature_gate.h" +#include "model_capabilities.h" + +#include +#include + +namespace dflash::common::detail { + +namespace { + +BackendPreparationFailure failure( + BackendPreparationError error, + std::string message, + std::vector warnings = {}) { + return {error, std::move(message), std::move(warnings)}; +} + +PlacementBackend resolve_target_backend( + const BackendArgs & args, + PlacementBackend compiled_backend) { + return args.device.backend == PlacementBackend::Auto + ? compiled_backend + : args.device.backend; +} + +} // namespace + +BackendPreparation BackendPlanBuilder::resolve( + BackendArgs args, + BackendAdmissionContext admission, + GgufModelInfo model, + PlacementBackend compiled_backend) { + if (args.model_path.empty()) { + return failure( + BackendPreparationError::InvalidRequest, + "model_path is empty"); + } + + if (model.arch.empty()) { + return failure( + BackendPreparationError::ModelInspection, + "failed to detect architecture from " + args.model_path); + } + + if (!arch_is_supported(model.arch)) { + return failure( + BackendPreparationError::ModelInspection, + "unsupported model architecture '" + model.arch + "' in " + + args.model_path); + } + + const PlacementBackend target_backend = + resolve_target_backend(args, compiled_backend); + + if (args.specla_mode && !args.ddtree_tau_explicit) { + args.ddtree_tau = 6.0f; + } + + // Preserve the original admission order. Warnings describe what the + // operator requested, before model-specific SpecLA normalization changes + // the effective construction request. + std::string incompatible = check_feature_compatibility( + args, admission, model.arch, target_backend, compiled_backend); + if (!incompatible.empty()) { + return failure( + BackendPreparationError::FeatureCompatibility, + std::move(incompatible)); + } + + std::vector warnings = + collect_feature_warnings(args, model.arch); + + BackendPlan::SpeclaEnvironmentAction specla_environment = + BackendPlan::SpeclaEnvironmentAction::Preserve; + + if (args.specla_mode) { + const bool supported = + model.arch == "qwen35" && !args.device.is_multi_device(); + if (supported) { + if (!args.draft_path.has_value()) { + return failure( + BackendPreparationError::FeatureCompatibility, + "Qwen3.6 SpecLA requires --draft ", + std::move(warnings)); + } + + args.ddtree_mode = true; + if (admission.kvflash_requested()) { + warnings.push_back( + "--specla is unavailable with KVFlash; using ordinary " + "DDTree verification"); + args.specla_mode = false; + if (!args.ddtree_tau_explicit) { + args.ddtree_tau = std::numeric_limits::infinity(); + } + specla_environment = + BackendPlan::SpeclaEnvironmentAction::Disable; + } else { + specla_environment = + BackendPlan::SpeclaEnvironmentAction::Enable; + } + } else { + warnings.push_back( + "--specla is unavailable for architecture '" + model.arch + + "' with placement " + placement_device_name(args.device) + + "; using the architecture's normal decode path"); + args.specla_mode = false; + if (!args.ddtree_tau_explicit) { + args.ddtree_tau = std::numeric_limits::infinity(); + } + specla_environment = + BackendPlan::SpeclaEnvironmentAction::Disable; + } + } + + // Validate the exact snapshot construction will consume. This replaces + // the old factory recheck against a second mutable BackendArgs object. + incompatible = check_feature_compatibility( + args, admission, model.arch, target_backend, compiled_backend); + if (!incompatible.empty()) { + return failure( + BackendPreparationError::FeatureCompatibility, + std::move(incompatible), + std::move(warnings)); + } + + BackendPlan plan; + plan.model_.path = std::move(args.model_path); + plan.model_.metadata = std::move(model); + + plan.placement_.target = std::move(args.device); + plan.placement_.draft = std::move(args.draft_device); + plan.placement_.remote_draft = std::move(args.remote_draft); + plan.placement_.remote_target_shard = + std::move(args.remote_target_shard); + + plan.cache_.fa_window = args.fa_window; + plan.cache_.paged_attention = args.paged_attention; + plan.cache_.max_concurrency = args.max_concurrency; + plan.cache_.kv_pool_tokens = args.kv_pool_tokens; + plan.cache_.kq_stride_pad = args.kq_stride_pad; + plan.cache_.draft_swa_window = args.draft_swa_window; + plan.cache_.draft_ctx_max = args.draft_ctx_max; + + plan.speculation_.draft_path = std::move(args.draft_path); + plan.speculation_.draft_block_size = args.draft_block_size; + plan.speculation_.fast_rollback = args.fast_rollback; + plan.speculation_.seq_verify = args.seq_verify; + plan.speculation_.specla_mode = args.specla_mode; + plan.speculation_.specla_top_k = args.specla_top_k; + plan.speculation_.specla_top_k_explicit = + args.specla_top_k_explicit; + plan.speculation_.ddtree_mode = args.ddtree_mode; + plan.speculation_.ddtree_budget = args.ddtree_budget; + plan.speculation_.ddtree_temp = args.ddtree_temp; + plan.speculation_.ddtree_chain_seed = args.ddtree_chain_seed; + plan.speculation_.ddtree_tau = args.ddtree_tau; + plan.speculation_.verify_width = args.verify_width; + plan.speculation_.use_feature_mirror = args.use_feature_mirror; + + plan.execution_.stream_fd = args.stream_fd; + plan.execution_.chunk = args.chunk; + plan.deepseek4_.prefill_mode = args.ds4_prefill_mode; + plan.deepseek4_.expert_top_k = args.ds4_expert_top_k; + plan.deepseek4_.fused_decode = args.ds4_fused_decode; + plan.deepseek4_.fused_verify_f16_kv = + args.ds4_fused_verify_f16_kv; + + plan.warnings_ = std::move(warnings); + plan.specla_environment_ = specla_environment; + return plan; +} + +} // namespace dflash::common::detail diff --git a/server/src/common/backend_plan_internal.h b/server/src/common/backend_plan_internal.h new file mode 100644 index 000000000..98b4a47f7 --- /dev/null +++ b/server/src/common/backend_plan_internal.h @@ -0,0 +1,18 @@ +#pragma once + +#include "backend_factory.h" + +namespace dflash::common::detail { + +// Pure planning seam used by prepare_backend() after GGUF inspection and by +// model-free tests with supplied model facts. +class BackendPlanBuilder { +public: + static BackendPreparation resolve( + BackendArgs args, + BackendAdmissionContext admission, + GgufModelInfo model, + PlacementBackend compiled_backend); +}; + +} // namespace dflash::common::detail diff --git a/server/src/common/feature_gate.cpp b/server/src/common/feature_gate.cpp index cae310910..a791efedf 100644 --- a/server/src/common/feature_gate.cpp +++ b/server/src/common/feature_gate.cpp @@ -11,7 +11,7 @@ namespace dflash::common { std::string check_feature_compatibility( const BackendArgs & args, - const BackendFeatureConfig & features, + const BackendAdmissionContext & admission, const std::string & arch, PlacementBackend target_backend, PlacementBackend compiled_backend) @@ -32,7 +32,7 @@ std::string check_feature_compatibility( ? target_backend : args.draft_device.backend; const bool draft_placement_used = - features.pflash_enabled || args.draft_path != nullptr; + admission.pflash_enabled || args.draft_path.has_value(); const bool mixed_draft_placement = draft_placement_used && target_backend != draft_backend; @@ -48,8 +48,8 @@ std::string check_feature_compatibility( } // ── PFlash enablement × drafter model - if (features.pflash_enabled && - !features.pflash_drafter_configured) { + if (admission.pflash_enabled && + !admission.pflash_drafter_configured) { return "--prefill-compression requires --prefill-drafter"; } @@ -98,7 +98,7 @@ std::string check_feature_compatibility( if (args.remote_target_shard.enabled()) { return "tensor parallelism is incompatible with --target-shard-ipc-bin"; } - if (features.pflash_enabled) { + if (admission.pflash_enabled) { return "tensor parallelism does not yet support prefill compression"; } } @@ -150,14 +150,14 @@ std::string check_feature_compatibility( } // ── remote draft execution × architecture - if (args.remote_draft.enabled() && args.draft_path && + if (args.remote_draft.enabled() && args.draft_path.has_value() && !arch_supports_remote_draft(arch)) { return "model architecture '" + arch + "' does not support remote draft execution"; } // ── mixed-backend PFlash × architecture - if (features.pflash_enabled && mixed_draft_placement && + if (admission.pflash_enabled && mixed_draft_placement && !arch_supports_pflash_compression(arch)) { return "model architecture '" + arch + "' does not support PFlash compression"; @@ -166,7 +166,7 @@ std::string check_feature_compatibility( // A block-size override changes the local draft graph itself. Remote // drafters own that shape in the IPC process and cannot be resized here. if (args.draft_block_size != 0) { - if (args.draft_path == nullptr) { + if (!args.draft_path.has_value()) { return "--draft-block-size requires --draft"; } if (args.remote_draft.enabled()) { @@ -176,7 +176,7 @@ std::string check_feature_compatibility( const bool concurrent_local_chain = arch == "qwen35" && args.paged_attention && - args.max_concurrency > 1 && args.draft_path != nullptr && + args.max_concurrency > 1 && args.draft_path.has_value() && !args.ddtree_mode && !args.remote_draft.enabled() && !args.device.is_layer_split() && !args.device.is_tensor_parallel() && @@ -186,7 +186,7 @@ std::string check_feature_compatibility( args.fa_window == 0; if (concurrent_local_chain && - features.draft_residency == DraftResidencyPolicy::RequestScoped) { + admission.draft_residency == DraftResidencyPolicy::RequestScoped) { return "concurrent DFlash2 does not support " "--draft-residency=request-scoped"; } @@ -208,7 +208,7 @@ std::string check_feature_compatibility( args.remote_target_shard.enabled()) { return "--paged-attention requires one local target device"; } - if ((args.draft_path != nullptr || args.remote_draft.enabled()) && + if ((args.draft_path.has_value() || args.remote_draft.enabled()) && !concurrent_local_chain) { return "--paged-attention requires autoregressive decode without a " "draft, or concurrent local same-device DFlash2 chains"; @@ -219,11 +219,11 @@ std::string check_feature_compatibility( if (args.fa_window != 0) { return "--paged-attention requires full attention (--fa-window 0)"; } - if (features.pflash_enabled) { + if (admission.pflash_enabled) { return "--paged-attention cannot be combined with PFlash prefill " "compression"; } - if (features.kvflash_enabled) { + if (admission.fixed_kvflash_requested()) { return "--paged-attention cannot be combined with KVFlash"; } // The pool rounds max_ctx up to a whole number of blocks, so the top @@ -357,7 +357,6 @@ void warn_inert(std::vector & out, std::vector collect_feature_warnings( const BackendArgs & args, - const BackendFeatureConfig & features, const std::string & arch) { std::vector out; @@ -365,7 +364,7 @@ std::vector collect_feature_warnings( // Each entry pairs a requested option with the capability predicate for // the field create_backend() would have to forward for it to take effect. - warn_inert(out, args.draft_path != nullptr, + warn_inert(out, args.draft_path.has_value(), arch_supports_decode_draft(arch, split), arch_supports_decode_draft(arch, false), split, arch, "--draft", "speculative decode"); @@ -395,13 +394,13 @@ std::vector collect_feature_warnings( arch_supports_draft_swa(arch, false), split, arch, "--draft-swa", "draft sliding-window attention"); - // MoE-only server features. These drive the DFLASH_QWEN35MOE_* / + // MoE-only backend requests. These drive the DFLASH_QWEN35MOE_* / // DFLASH_LAGUNA_* env vars, which a dense backend never reads. - if (features.routing_stats_requested && !arch_has_expert_offload(arch)) { + if (args.routing_stats_requested && !arch_has_expert_offload(arch)) { out.push_back("--freq/--collect-routing ignored: architecture '" + arch + "' has no expert routing to record"); } - if (features.adaptive_experts_requested && !arch_has_expert_offload(arch)) { + if (args.adaptive_experts_requested && !arch_has_expert_offload(arch)) { out.push_back("--adaptive-experts ignored: architecture '" + arch + "' has no expert-count gating"); } diff --git a/server/src/common/feature_gate.h b/server/src/common/feature_gate.h index 13cbfe5af..312139b1e 100644 --- a/server/src/common/feature_gate.h +++ b/server/src/common/feature_gate.h @@ -41,18 +41,18 @@ namespace dflash::common { // Returns an empty string when the requested feature set is coherent, or a // description of the first violated rule. // -// `features` carries launch features owned above the backend factory. `arch`, +// `admission` carries launch facts owned above the backend factory. `arch`, // `target_backend`, and `compiled_backend` are resolved facts: the architecture // read from the GGUF, the requested target with PlacementBackend::Auto mapped // to the compiled default, and the binary's compiled placement. Passing these // in keeps the gate a pure function that unit tests can drive without a model // file or GPU. // -// prepare_backend() owns the public admission decision, and create_backend() -// checks the same function as a safety net before dispatch. +// prepare_backend() owns the public admission decision. Construction trusts +// the immutable BackendPlan produced by that boundary. std::string check_feature_compatibility( const BackendArgs & args, - const BackendFeatureConfig & features, + const BackendAdmissionContext & admission, const std::string & arch, PlacementBackend target_backend, PlacementBackend compiled_backend); @@ -69,7 +69,6 @@ std::string check_feature_compatibility( // configuration has no useful warnings to report. std::vector collect_feature_warnings( const BackendArgs & args, - const BackendFeatureConfig & features, const std::string & arch); } // namespace dflash::common diff --git a/server/src/common/generation_types.h b/server/src/common/generation_types.h new file mode 100644 index 000000000..893a88f69 --- /dev/null +++ b/server/src/common/generation_types.h @@ -0,0 +1,134 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "sampler.h" + +namespace dflash::common { + +// Called once for each token committed by transitional whole-request +// executors. Returning false requests cancellation. +using TokenCallback = std::function; + +// Pre-commit token substitution for thinking budgets. When the remaining +// generation budget reaches hard_limit_remaining, close_token_ids replaces +// the next sampled tokens in order so reusable KV state sees the replacement. +struct BudgetHook { + std::vector close_token_ids; + int hard_limit_remaining = 0; +}; + +struct GenerateRequest { + std::vector prompt; + int n_gen = 0; + SamplerCfg sampler; + bool do_sample = false; + bool stream = false; + // Existing physical snapshot coordinates during the cache migration. + int snap_pos = -1; + int snap_slot = -1; + // Transitional callback used below the future Generation channel adapter. + TokenCallback on_token; + // Model-ready optional sequences are owned because the engine may retain + // the request after its caller returns. + std::vector hint_tokens; + std::vector stall_tool_prefix_tokens; + std::vector stall_action_suffix_tokens; + std::vector stall_skip_tokens; + BudgetHook budget_hook; + // Set only for the single autoregressive retry after empty spec output. + bool force_ar_decode = false; +}; + +// Backend-independent failure categories. generate_error_code() is their +// stable wire representation once a value is published. +enum class GenerateErrorCode { + Incomplete, + AdapterUnavailable, + ContextOverflow, + SamplingUnsupported, + PrefillFailed, + DecodeSeedMissing, + DecodeFailed, + InvalidSnapshotSlot, + ModelParked, + Cancelled, + OutputBackpressure, + Overloaded, + ShuttingDown, + BackendSpecific, +}; + +constexpr std::string_view generate_error_code(GenerateErrorCode error) { + switch (error) { + case GenerateErrorCode::Incomplete: return "incomplete"; + case GenerateErrorCode::AdapterUnavailable: return "adapter_unavailable"; + case GenerateErrorCode::ContextOverflow: return "context_overflow"; + case GenerateErrorCode::SamplingUnsupported: return "sampling_unsupported"; + case GenerateErrorCode::PrefillFailed: return "prefill_failed"; + case GenerateErrorCode::DecodeSeedMissing: return "decode_seed_missing"; + case GenerateErrorCode::DecodeFailed: return "decode_failed"; + case GenerateErrorCode::InvalidSnapshotSlot: return "invalid_snapshot_slot"; + case GenerateErrorCode::ModelParked: return "model_parked"; + case GenerateErrorCode::Cancelled: return "cancelled"; + case GenerateErrorCode::OutputBackpressure: return "output_backpressure"; + case GenerateErrorCode::Overloaded: return "overloaded"; + case GenerateErrorCode::ShuttingDown: return "shutting_down"; + case GenerateErrorCode::BackendSpecific: return "backend_specific"; + } + return "unknown_error"; +} + +struct GenerateError { + GenerateErrorCode code = GenerateErrorCode::Incomplete; + std::string detail; +}; + +struct GenerateResult { + // A producer must explicitly call succeed() before returning success. + std::optional error = GenerateError{}; + std::vector tokens; + double prefill_s = 0.0; + double decode_s = 0.0; + // Prompt tokens confirmed by the backend's physical snapshot restore. + int restored_prefix_tokens = 0; + // Distinguishes engine-injected thinking closure from a natural close. + bool budget_forced_close = false; + // True when the post-close watchdog stopped a repetition loop. + bool degenerate_decode_close = false; + // accepted_draft_tokens / total_draft_positions; zero without spec decode. + float accept_rate = 0.0f; + // Separates a zero accept rate from an autoregressive execution. + bool spec_decode_ran = false; + // The attempt emitted only tokens suppressed by the response layer and is + // eligible for the same one-time retry as an empty token vector. + bool empty_visible_output = false; + + bool ok() const { + return !error.has_value(); + } + + std::string_view error_code() const { + return error ? generate_error_code(error->code) : std::string_view{}; + } + + std::string_view error_detail() const { + return error ? std::string_view(error->detail) : std::string_view{}; + } + + void succeed() { + error.reset(); + } + + void fail(GenerateErrorCode code, std::string detail = {}) { + error = GenerateError{code, std::move(detail)}; + } +}; + +} // namespace dflash::common diff --git a/server/src/common/model_backend.h b/server/src/common/model_backend.h index 221e54ab5..e6992856d 100644 --- a/server/src/common/model_backend.h +++ b/server/src/common/model_backend.h @@ -22,6 +22,7 @@ #include "ggml.h" #include "ggml-backend.h" +#include "generation_types.h" #include "sampler.h" #include "concurrency/seq_engine.h" #include "placement/draft_residency.h" @@ -78,10 +79,6 @@ constexpr bool park_target_includes_draft_model(ParkTarget target) { target == ParkTarget::DraftModel; } -// Token callback for streaming generation. Called once per committed token. -// Return true to continue generation, false to abort. -using TokenCallback = std::function; - // Return true when an in-flight request should stop. Backends poll this at // their existing prefill/decode cancellation boundaries so cancellation does // not depend on filling the socket's send buffer first. @@ -132,166 +129,6 @@ struct DaemonIO { DaemonIO with_token_callback(const TokenCallback & cb) const; }; -// ─── Generate request/result ──────────────────────────────────────────── - -// Thinking-budget force-close hook. Mirrors antirez/ds4 ds4_eval.c's -// hard_limit_reply_budget semantics: when the budget remaining (n_gen -// minus tokens committed so far) falls to hard_limit_remaining, the -// next sampled tokens get overridden with close_token_ids in order, -// giving the model the remaining budget to write a visible answer -// after the injected close-tag sequence. -// -// Single vs multi-token close: -// Qwen3.6: is one added_token (id 248069). close_token_ids -// has size 1. One override + budget_close_injected=true. -// DeepSeek/laguna: tokenizes to 3 ordinary tokens -// ([1718, 37947, 32] for DS-V3). close_token_ids has -// size 3. Three consecutive overrides, then resume. -// -// This is "Level 2" of our thinking-budget migration: in-process -// mid-stream force-close, KV-continuous. Beats Level 1's phase-2 -// reprompt because the model never sees a fresh prefill — its KV -// state continues naturally after the injected close. -// -// Current implementation: AR-decode only. When budget_hook is set, -// backends MAY route generation through their AR path (skipping spec -// decode) — the perf trade-off is acceptable since this only kicks in -// for thinking-enabled requests. Spec-decode integration is a follow-up. -struct BudgetHook { - // Multi-token close sequence injected when `(n_gen - committed)` - // drops to `hard_limit_remaining`. For Qwen3.x this is the - // canonical "Considering the limited time..." summarize-and-stop - // lead-in (tokenized at server startup); for non-qwen arches it's - // a single close-tag token. Empty = hook disabled. - std::vector close_token_ids; - int hard_limit_remaining = 0; -}; - -struct GenerateRequest { - std::vector prompt; - int n_gen = 0; - SamplerCfg sampler; - bool do_sample = false; - bool stream = false; // emit tokens to stream_fd - // Optional inline-snap: snapshot at this position after prefill. - int snap_pos = -1; - int snap_slot = -1; - // Optional token callback for streaming. When set, backends call this - // for each committed token. If it returns false, generation aborts - // immediately. This is the primary mechanism for client-disconnect - // cancellation in the native HTTP server. - TokenCallback on_token; - // Tool call hint tokens: pre-tokenized structural tokens that are - // predictable with ~100% confidence (XML tags, function name, param names). - // When non-null, the spec decode loop uses these as draft overrides, - // bypassing draft model computation for covered positions. - const std::vector * hint_tokens = nullptr; - // Optional env-gated dflash stall recovery: when spec decode is about to - // emit early EOS after an action preamble, inject a bare tool-call XML - // prefix and continue in AR with KV state intact. - const std::vector * stall_tool_prefix_tokens = nullptr; - const std::vector * stall_action_suffix_tokens = nullptr; - const std::vector * stall_skip_tokens = nullptr; - // Optional thinking-budget hook — see BudgetHook docs above. - BudgetHook budget_hook; - // Common retry knob. Upper layers set this after a speculative decode - // path returns success but emits no tokens, so each backend can route the - // retry through its existing AR path without copying retry policy. - bool force_ar_decode = false; -}; - -// Stable, backend-independent generation failure categories. Backends should -// use these for recurrent failures so callers do not need to understand -// architecture-specific strings. `generate_error_code()` is the daemon/API -// wire representation and must remain backward-compatible once published. -enum class GenerateErrorCode { - Incomplete, - AdapterUnavailable, - ContextOverflow, - SamplingUnsupported, - PrefillFailed, - DecodeSeedMissing, - DecodeFailed, - InvalidSnapshotSlot, - ModelParked, - BackendSpecific, -}; - -constexpr std::string_view generate_error_code(GenerateErrorCode error) { - switch (error) { - case GenerateErrorCode::Incomplete: return "incomplete"; - case GenerateErrorCode::AdapterUnavailable: return "adapter_unavailable"; - case GenerateErrorCode::ContextOverflow: return "context_overflow"; - case GenerateErrorCode::SamplingUnsupported: return "sampling_unsupported"; - case GenerateErrorCode::PrefillFailed: return "prefill_failed"; - case GenerateErrorCode::DecodeSeedMissing: return "decode_seed_missing"; - case GenerateErrorCode::DecodeFailed: return "decode_failed"; - case GenerateErrorCode::InvalidSnapshotSlot: return "invalid_snapshot_slot"; - case GenerateErrorCode::ModelParked: return "model_parked"; - case GenerateErrorCode::BackendSpecific: return "backend_specific"; - } - return "unknown_error"; -} - -struct GenerateError { - GenerateErrorCode code = GenerateErrorCode::Incomplete; - std::string detail; -}; - -struct GenerateResult { - // Default to an incomplete failure so a backend must explicitly call - // succeed() before returning a successful result. - std::optional error = GenerateError{}; - std::vector tokens; - double prefill_s = 0.0; - double decode_s = 0.0; - // Backend-confirmed prompt tokens supplied by a restored KV snapshot. - int restored_prefix_tokens = 0; - // True when the backend's Level 2 hook injected the close - // sequence during this generation (vs. the model self-closing). The - // server uses this to attribute close_kind correctly: if the model - // produced naturally we report "natural"; if the hook fired - // we report "hard". Without this flag, decoding the phase-1 token - // stream and grepping for "" cannot distinguish the two - // (the injected close decodes identically). - bool budget_forced_close = false; - // True iff the AR decode loop's post-close watchdog detected an n-gram - // repetition loop and broke out early. Caller surfaces this so clients - // can mark the answer as unreliable rather than treating the - // (truncated) content as a clean response. - bool degenerate_decode_close = false; - // DFlash chain accept rate: accepted_draft_tokens / total_draft_positions. - // 0.0 when spec decode did not run (AR fallback or no draft model). - float accept_rate = 0.0f; - // True when spec decode actually ran (accept_rate==0 still needs a bandit update). - bool spec_decode_ran = false; - // True when decode emitted only tokens that the API layer suppresses - // (for example an immediate EOS/EOT). This is semantically equivalent - // to zero output for clients and should take the same AR retry path as - // an empty token vector. - bool empty_visible_output = false; - - bool ok() const { - return !error.has_value(); - } - - std::string_view error_code() const { - return error ? generate_error_code(error->code) : std::string_view{}; - } - - std::string_view error_detail() const { - return error ? std::string_view(error->detail) : std::string_view{}; - } - - void succeed() { - error.reset(); - } - - void fail(GenerateErrorCode code, std::string detail = {}) { - error = GenerateError{code, std::move(detail)}; - } -}; - // ─── Backend interface ────────────────────────────────────────────────── struct ModelBackend { virtual ~ModelBackend() = default; diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index c8750bb28..f6af8c755 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -27,6 +27,7 @@ #include #include #include +#include namespace dflash::common { @@ -740,8 +741,8 @@ static MoeLayerDesc make_ds4_expert_layer_desc(const DeepSeek4Layer & layer) { } // namespace -DeepSeek4Backend::DeepSeek4Backend(const DeepSeek4BackendConfig & cfg) - : cfg_(cfg) {} +DeepSeek4Backend::DeepSeek4Backend(DeepSeek4BackendConfig cfg) + : cfg_(std::move(cfg)) {} DeepSeek4Backend::~DeepSeek4Backend() { shutdown(); @@ -808,20 +809,24 @@ bool DeepSeek4Backend::load_model() { } std::fprintf(stderr, "[deepseek4] explicit HIP full-model load failed: %s\n", - cfg_.model_path); + cfg_.model_path.c_str()); return false; } } else if (target_backend == PlacementBackend::Hip || heterogeneous_tp) { std::fprintf(stderr, "[deepseek4] heterogeneous target detected; using hybrid expert load path\n"); if (!init_hybrid_model()) { - std::fprintf(stderr, "[deepseek4] hybrid mode failed: %s\n", cfg_.model_path); + std::fprintf( + stderr, "[deepseek4] hybrid mode failed: %s\n", + cfg_.model_path.c_str()); return false; } } else if (!load_deepseek4_gguf(cfg_.model_path, backend_, w_)) { std::fprintf(stderr, "[deepseek4] full model load failed, trying hybrid mode...\n"); if (!init_hybrid_model()) { - std::fprintf(stderr, "[deepseek4] hybrid mode also failed: %s\n", cfg_.model_path); + std::fprintf( + stderr, "[deepseek4] hybrid mode also failed: %s\n", + cfg_.model_path.c_str()); return false; } } @@ -1410,7 +1415,7 @@ bool DeepSeek4Backend::init_hybrid_model() { plan.skip_expert_tensors = true; if (!load_deepseek4_gguf_partial(cfg_.model_path, backend_, plan, w_)) { std::fprintf(stderr, "[deepseek4] failed to partially load model for hybrid mode: %s\n", - cfg_.model_path); + cfg_.model_path.c_str()); return false; } @@ -1426,7 +1431,7 @@ bool DeepSeek4Backend::init_hybrid_model() { free_deepseek4_weights(w_); if (!load_deepseek4_gguf(cfg_.model_path, backend_, w_)) { std::fprintf(stderr, "[deepseek4] failed to reload full model after placement: %s\n", - cfg_.model_path); + cfg_.model_path.c_str()); return false; } return true; @@ -1474,7 +1479,7 @@ bool DeepSeek4Backend::init_hybrid_model() { if (!load_deepseek4_gguf(cfg_.model_path, backend_, w_)) { std::fprintf(stderr, "[deepseek4] monolithic fallback failed (model does not " - "fit resident): %s\n", cfg_.model_path); + "fit resident): %s\n", cfg_.model_path.c_str()); return false; } return true; diff --git a/server/src/deepseek4/deepseek4_backend.h b/server/src/deepseek4/deepseek4_backend.h index 583132042..bb55cea6d 100644 --- a/server/src/deepseek4/deepseek4_backend.h +++ b/server/src/deepseek4/deepseek4_backend.h @@ -43,7 +43,7 @@ int deepseek4_hybrid_prefill_step_tokens( int remaining_tokens); class DeepSeek4Backend : public ModelBackend { public: - explicit DeepSeek4Backend(const DeepSeek4BackendConfig & cfg); + explicit DeepSeek4Backend(DeepSeek4BackendConfig cfg); ~DeepSeek4Backend() override; DeepSeek4Backend(const DeepSeek4Backend &) = delete; diff --git a/server/src/deepseek4/deepseek4_daemon.cpp b/server/src/deepseek4/deepseek4_daemon.cpp index fabc1c184..c50da9883 100644 --- a/server/src/deepseek4/deepseek4_daemon.cpp +++ b/server/src/deepseek4/deepseek4_daemon.cpp @@ -14,7 +14,7 @@ int run_deepseek4_daemon(const char * model_path, int max_ctx, int chunk) { DeepSeek4BackendConfig cfg; - cfg.model_path = model_path; + cfg.model_path = model_path ? model_path : ""; cfg.device.gpu = gpu; cfg.stream_fd = stream_fd; cfg.max_ctx = max_ctx; diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 66d80e417..2a15d9b40 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -306,7 +306,7 @@ struct DeepSeek4RawRingSpan { // ─── Configuration ────────────────────────────────────────────────────── struct DeepSeek4BackendConfig { - const char * model_path = nullptr; + std::string model_path; DevicePlacement device; int stream_fd = -1; int chunk = 512; // prefill chunk size diff --git a/server/src/deepseek4/deepseek4_layer_split_adapter.cpp b/server/src/deepseek4/deepseek4_layer_split_adapter.cpp index 6b1cd737c..8e3fddc8a 100644 --- a/server/src/deepseek4/deepseek4_layer_split_adapter.cpp +++ b/server/src/deepseek4/deepseek4_layer_split_adapter.cpp @@ -20,6 +20,7 @@ #include #include #include +#include namespace dflash::common { @@ -95,8 +96,8 @@ static void log_split_tel(const char * phase, } // namespace DeepSeek4LayerSplitAdapter::DeepSeek4LayerSplitAdapter( - const DeepSeek4LayerSplitAdapterConfig & cfg) - : cfg_(cfg) { + DeepSeek4LayerSplitAdapterConfig cfg) + : cfg_(std::move(cfg)) { snapshots_.resize(PREFIX_SLOTS); } @@ -187,8 +188,8 @@ int DeepSeek4LayerSplitAdapter::compute_auto_split_layers() const { } bool DeepSeek4LayerSplitAdapter::init() { - if (!cfg_.target_path) { - std::fprintf(stderr, "[deepseek4-split] target_path is null\n"); + if (cfg_.target_path.empty()) { + std::fprintf(stderr, "[deepseek4-split] target_path is empty\n"); return false; } @@ -253,7 +254,7 @@ bool DeepSeek4LayerSplitAdapter::init() { // Multi-GPU local path (multiple CUDA GPUs available) LayerSplitRuntimeInit runtime_cfg; - runtime_cfg.target_path = cfg_.target_path; + runtime_cfg.target_path = cfg_.target_path.c_str(); runtime_cfg.device = &device; runtime_cfg.log_prefix = "deepseek4-split"; @@ -304,7 +305,7 @@ bool DeepSeek4LayerSplitAdapter::init_mixed_target_split_full(const DevicePlacem // Mixed target split: local CUDA shard + remote Halo shard via IPC daemon. // Only the first shard runs locally; remaining layers handled by remote daemon. - const auto info = inspect_gguf_model_info(cfg_.target_path); + const auto info = inspect_gguf_model_info(cfg_.target_path.c_str()); const int n_layer = info.n_layer; if (n_layer <= 0) { std::fprintf(stderr, "[deepseek4-split] failed to inspect target layer count\n"); diff --git a/server/src/deepseek4/deepseek4_layer_split_adapter.h b/server/src/deepseek4/deepseek4_layer_split_adapter.h index 2a52d1866..6a59b5dda 100644 --- a/server/src/deepseek4/deepseek4_layer_split_adapter.h +++ b/server/src/deepseek4/deepseek4_layer_split_adapter.h @@ -24,7 +24,7 @@ namespace dflash::common { struct DeepSeek4LayerSplitAdapterConfig { - const char * target_path = nullptr; + std::string target_path; DevicePlacement device; RemoteTargetShardConfig remote_target_shard; int chunk = 512; @@ -37,7 +37,7 @@ struct DeepSeek4LayerSplitShard : LayerSplitShardMeta { class DeepSeek4LayerSplitAdapter : public LayerSplitAdapter { public: - explicit DeepSeek4LayerSplitAdapter(const DeepSeek4LayerSplitAdapterConfig & cfg); + explicit DeepSeek4LayerSplitAdapter(DeepSeek4LayerSplitAdapterConfig cfg); ~DeepSeek4LayerSplitAdapter() override; DeepSeek4LayerSplitAdapter(const DeepSeek4LayerSplitAdapter &) = delete; diff --git a/server/src/engine/generation.cpp b/server/src/engine/generation.cpp new file mode 100644 index 000000000..3e94e085a --- /dev/null +++ b/server/src/engine/generation.cpp @@ -0,0 +1,316 @@ +#include "generation.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace dflash::engine { + +namespace detail { + +struct GenerationState { + explicit GenerationState(std::size_t capacity) + : token_capacity(capacity) {} + + std::mutex mutex; + std::condition_variable ready; + std::deque tokens; + std::size_t buffered_tokens = 0; + std::optional progress; + std::optional terminal; + std::optional cancel_reason; + std::size_t token_capacity; +}; + +} // namespace detail + +Generation::Generation(std::shared_ptr state) + : state_(std::move(state)) {} + +Generation::Generation(Generation && other) noexcept + : state_(std::move(other.state_)) {} + +Generation & Generation::operator=(Generation && other) noexcept { + if (this != &other) { + reset(); + state_ = std::move(other.state_); + } + return *this; +} + +Generation::~Generation() { + reset(); +} + +Generation::operator bool() const noexcept { + return static_cast(state_); +} + +GenerateEvent Generation::next() { + if (!state_) { + throw std::logic_error("next() called on an empty Generation"); + } + + std::unique_lock lock(state_->mutex); + state_->ready.wait(lock, [this] { + return !state_->tokens.empty() || state_->progress || state_->terminal; + }); + + if (!state_->tokens.empty()) { + TokenBatch batch = std::move(state_->tokens.front()); + state_->tokens.pop_front(); + state_->buffered_tokens -= batch.tokens.size(); + return batch; + } + if (state_->progress) { + GenerationProgress progress = *state_->progress; + state_->progress.reset(); + return progress; + } + return *state_->terminal; +} + +bool Generation::cancel(GenerationCancelReason reason) { + if (!state_) return false; + std::lock_guard lock(state_->mutex); + if (state_->terminal) return false; + state_->cancel_reason = reason; + common::GenerateResult result; + switch (reason) { + case GenerationCancelReason::OutputBackpressure: + result.fail(common::GenerateErrorCode::OutputBackpressure); + break; + case GenerationCancelReason::Shutdown: + result.fail(common::GenerateErrorCode::ShuttingDown); + break; + case GenerationCancelReason::ConsumerDropped: + case GenerationCancelReason::ClientDisconnected: + result.fail(common::GenerateErrorCode::Cancelled); + break; + } + state_->terminal = GenerateCompleted{std::move(result)}; + state_->ready.notify_all(); + return true; +} + +void Generation::reset() { + if (state_) cancel(GenerationCancelReason::ConsumerDropped); + state_.reset(); +} + +GenerationSource::GenerationSource( + std::shared_ptr state) + : state_(std::move(state)) {} + +GenerationSource & GenerationSource::operator=( + GenerationSource && other) noexcept { + if (this != &other) { + reset(); + state_ = std::move(other.state_); + } + return *this; +} + +GenerationSource::~GenerationSource() { + reset(); +} + +void GenerationSource::reset() { + // Losing the producer must wake a surviving consumer with a terminal + // result; otherwise Generation::next() could wait forever. + if (!state_) return; + common::GenerateResult result; + result.fail(common::GenerateErrorCode::Incomplete, + "generation ended without a terminal result"); + complete(std::move(result)); + state_.reset(); +} + +GenerationSource::operator bool() const noexcept { + return static_cast(state_); +} + +bool GenerationSource::publish(TokenBatch batch) { + if (!state_) return false; + std::lock_guard lock(state_->mutex); + if (state_->terminal) return false; + if (batch.tokens.empty()) return true; + if (batch.tokens.size() > + state_->token_capacity - state_->buffered_tokens) { + state_->cancel_reason = GenerationCancelReason::OutputBackpressure; + common::GenerateResult result; + result.fail(common::GenerateErrorCode::OutputBackpressure); + state_->terminal = GenerateCompleted{std::move(result)}; + state_->ready.notify_all(); + return false; + } + state_->buffered_tokens += batch.tokens.size(); + state_->tokens.push_back(std::move(batch)); + state_->ready.notify_one(); + return true; +} + +bool GenerationSource::publish(GenerationProgress progress) { + if (!state_) return false; + std::lock_guard lock(state_->mutex); + if (state_->terminal) return false; + state_->progress = progress; + state_->ready.notify_one(); + return true; +} + +bool GenerationSource::complete(common::GenerateResult result) { + if (!state_) return false; + std::lock_guard lock(state_->mutex); + if (state_->terminal) return false; + state_->terminal = GenerateCompleted{std::move(result)}; + state_->ready.notify_all(); + return true; +} + +bool GenerationSource::is_cancelled() const { + if (!state_) return true; + std::lock_guard lock(state_->mutex); + return state_->cancel_reason.has_value(); +} + +std::optional +GenerationSource::cancellation_reason() const { + if (!state_) return GenerationCancelReason::ConsumerDropped; + std::lock_guard lock(state_->mutex); + return state_->cancel_reason; +} + +GenerationPair make_generation(std::size_t token_capacity) { + if (token_capacity == 0) { + throw std::invalid_argument("Generation token capacity must be positive"); + } + auto state = std::make_shared(token_capacity); + return {Generation(state), GenerationSource(std::move(state))}; +} + +Generation make_completed_generation(common::GenerateResult result) { + GenerationPair pair = make_generation(1); + pair.source.complete(std::move(result)); + return std::move(pair.generation); +} + +struct GenerationQueue::Impl { + struct Queued { + common::GenerateRequest request; + std::shared_ptr state; + }; + + Impl(std::size_t requests, std::size_t tokens) + : request_capacity(requests), token_capacity(tokens) {} + + std::mutex mutex; + std::condition_variable ready; + std::deque queued; + // Shutdown can reach every live channel without extending its lifetime. + std::vector> live; + std::size_t request_capacity; + std::size_t token_capacity; + bool stopping = false; +}; + +GenerationQueue::GenerationQueue(std::size_t request_capacity, + std::size_t token_capacity) { + if (request_capacity == 0) { + throw std::invalid_argument( + "Generation request capacity must be positive"); + } + if (token_capacity == 0) { + throw std::invalid_argument( + "Generation token capacity must be positive"); + } + impl_ = std::make_unique(request_capacity, token_capacity); +} + +GenerationQueue::~GenerationQueue() { + shutdown(); +} + +Generation GenerationQueue::submit(common::GenerateRequest request) { + std::lock_guard lock(impl_->mutex); + impl_->queued.erase( + std::remove_if( + impl_->queued.begin(), impl_->queued.end(), + [](const Impl::Queued & queued) { + std::lock_guard state_lock(queued.state->mutex); + return queued.state->cancel_reason.has_value(); + }), + impl_->queued.end()); + impl_->live.erase( + std::remove_if(impl_->live.begin(), impl_->live.end(), + [](const auto & state) { return state.expired(); }), + impl_->live.end()); + if (impl_->stopping) { + common::GenerateResult result; + result.fail(common::GenerateErrorCode::ShuttingDown); + return make_completed_generation(std::move(result)); + } + if (impl_->queued.size() >= impl_->request_capacity) { + common::GenerateResult result; + result.fail(common::GenerateErrorCode::Overloaded); + return make_completed_generation(std::move(result)); + } + + auto state = std::make_shared( + impl_->token_capacity); + Generation generation(state); + impl_->live.push_back(state); + impl_->queued.push_back({std::move(request), std::move(state)}); + impl_->ready.notify_one(); + return generation; +} + +std::optional GenerationQueue::next() { + std::unique_lock lock(impl_->mutex); + while (true) { + impl_->ready.wait(lock, [this] { + return impl_->stopping || !impl_->queued.empty(); + }); + + while (!impl_->queued.empty()) { + Impl::Queued queued = std::move(impl_->queued.front()); + impl_->queued.pop_front(); + + { + std::lock_guard state_lock(queued.state->mutex); + if (queued.state->cancel_reason) continue; + } + + return Work{std::move(queued.request), + GenerationSource(std::move(queued.state))}; + } + + if (impl_->stopping) return std::nullopt; + } +} + +void GenerationQueue::shutdown() { + std::lock_guard queue_lock(impl_->mutex); + if (impl_->stopping) return; + impl_->stopping = true; + impl_->queued.clear(); + + for (const auto & weak : impl_->live) { + auto state = weak.lock(); + if (!state) continue; + std::lock_guard state_lock(state->mutex); + if (state->terminal) continue; + state->cancel_reason = GenerationCancelReason::Shutdown; + common::GenerateResult result; + result.fail(common::GenerateErrorCode::ShuttingDown); + state->terminal = GenerateCompleted{std::move(result)}; + state->ready.notify_all(); + } + impl_->ready.notify_all(); +} + +} // namespace dflash::engine diff --git a/server/src/engine/generation.h b/server/src/engine/generation.h new file mode 100644 index 000000000..bf0fe572a --- /dev/null +++ b/server/src/engine/generation.h @@ -0,0 +1,141 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/generation_types.h" + +namespace dflash::engine { + +struct TokenBatch { + std::vector tokens; +}; + +enum class GenerationPhase { + Queued, + Prefill, + Decode, +}; + +struct GenerationProgress { + GenerationPhase phase = GenerationPhase::Queued; + int processed_tokens = 0; +}; + +struct GenerateCompleted { + common::GenerateResult result; +}; + +using GenerateEvent = + std::variant; + +enum class GenerationCancelReason { + ConsumerDropped, + ClientDisconnected, + OutputBackpressure, + Shutdown, +}; + +namespace detail { +struct GenerationState; +} + +struct GenerationPair; +GenerationPair make_generation(std::size_t token_capacity); + +class Generation { +public: + Generation() = default; + Generation(Generation && other) noexcept; + Generation & operator=(Generation && other) noexcept; + Generation(const Generation &) = delete; + Generation & operator=(const Generation &) = delete; + ~Generation(); + + explicit operator bool() const noexcept; + // Blocks until the next buffered token batch, coalesced progress update, + // or terminal result. A terminal result remains observable after completion. + GenerateEvent next(); + // Records the first cancellation before completion. Returns false when a + // cancellation or terminal result was already present. + bool cancel(GenerationCancelReason reason); + +private: + explicit Generation(std::shared_ptr state); + void reset(); + + std::shared_ptr state_; + + friend struct GenerationPair; + friend class GenerationQueue; + friend GenerationPair make_generation(std::size_t token_capacity); +}; + +class GenerationSource { +public: + GenerationSource() = default; + GenerationSource(GenerationSource && other) noexcept = default; + GenerationSource & operator=(GenerationSource && other) noexcept; + GenerationSource(const GenerationSource &) = delete; + GenerationSource & operator=(const GenerationSource &) = delete; + ~GenerationSource(); + + explicit operator bool() const noexcept; + // Token publication is bounded. Filling the channel terminates this + // generation with OutputBackpressure instead of blocking the producer. + bool publish(TokenBatch batch); + // Only the latest unread progress value is retained. + bool publish(GenerationProgress progress); + // Publishes the terminal result exactly once. + bool complete(common::GenerateResult result); + bool is_cancelled() const; + std::optional cancellation_reason() const; + +private: + explicit GenerationSource(std::shared_ptr state); + void reset(); + + std::shared_ptr state_; + + friend struct GenerationPair; + friend class GenerationQueue; + friend GenerationPair make_generation(std::size_t token_capacity); +}; + +struct GenerationPair { + Generation generation; + GenerationSource source; +}; + +Generation make_completed_generation(common::GenerateResult result); + +class GenerationQueue { +public: + struct Work { + common::GenerateRequest request; + GenerationSource source; + }; + + GenerationQueue(std::size_t request_capacity, + std::size_t token_capacity); + GenerationQueue(const GenerationQueue &) = delete; + GenerationQueue & operator=(const GenerationQueue &) = delete; + ~GenerationQueue(); + + // Returns an already-completed rejection when the queue is full or stopped. + Generation submit(common::GenerateRequest request); + // Blocks until FIFO work is available or shutdown begins. + std::optional next(); + // Idempotently rejects new work and completes every live generation. + void shutdown(); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace dflash::engine diff --git a/server/src/engine/luce_engine.cpp b/server/src/engine/luce_engine.cpp new file mode 100644 index 000000000..078269a83 --- /dev/null +++ b/server/src/engine/luce_engine.cpp @@ -0,0 +1,69 @@ +#include "luce_engine.h" + +#include "common/model_backend.h" + +#include +#include + +namespace dflash::engine { + +LuceEngine::LuceEngine(std::unique_ptr backend) + : backend_(std::move(backend)) { + if (!backend_) { + throw std::invalid_argument("LuceEngine requires a backend"); + } +} + +LuceEngine::~LuceEngine() { + stop_serving(); +} + +common::ModelBackend &LuceEngine::backend() noexcept { + return *backend_; +} + +const common::ModelBackend &LuceEngine::backend() const noexcept { + return *backend_; +} + +bool LuceEngine::start_serving(ServingLoops loops, bool allow_concurrent) { + std::lock_guard lock(lifecycle_mu_); + if (worker_.joinable() || !loops.serial || !loops.concurrent || + !loops.request_stop) { + return false; + } + + common::SeqEngine *seq_engine = + allow_concurrent ? backend_->seq_engine() : nullptr; + request_stop_ = std::move(loops.request_stop); + + try { + if (seq_engine) { + auto concurrent = std::move(loops.concurrent); + worker_ = std::thread( + [concurrent = std::move(concurrent), seq_engine]() mutable { + concurrent(*seq_engine); + }); + } else { + worker_ = std::thread(std::move(loops.serial)); + } + } catch (...) { + request_stop_ = {}; + throw; + } + + return true; +} + +void LuceEngine::stop_serving() { + std::lock_guard lock(lifecycle_mu_); + if (!worker_.joinable()) { + return; + } + + request_stop_(); + worker_.join(); + request_stop_ = {}; +} + +} // namespace dflash::engine diff --git a/server/src/engine/luce_engine.h b/server/src/engine/luce_engine.h new file mode 100644 index 000000000..525605cc5 --- /dev/null +++ b/server/src/engine/luce_engine.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include +#include + +namespace dflash::common { +struct ModelBackend; +class SeqEngine; +} // namespace dflash::common + +namespace dflash::engine { + +// Owns the model backend and the single execution thread used to serve it. +// Transport-specific request handling remains outside this class; callers +// provide the established serial and concurrent serving loops. +class LuceEngine final { +public: + struct ServingLoops { + std::function serial; + std::function concurrent; + std::function request_stop; + }; + + explicit LuceEngine(std::unique_ptr backend); + ~LuceEngine(); + + LuceEngine(const LuceEngine &) = delete; + LuceEngine &operator=(const LuceEngine &) = delete; + LuceEngine(LuceEngine &&) = delete; + LuceEngine &operator=(LuceEngine &&) = delete; + + common::ModelBackend &backend() noexcept; + const common::ModelBackend &backend() const noexcept; + + // Starts exactly one serving loop. Concurrent serving is selected only + // when the caller permits it and the backend exposes a SeqEngine. + bool start_serving(ServingLoops loops, bool allow_concurrent); + void stop_serving(); + +private: + std::unique_ptr backend_; + std::function request_stop_; + std::thread worker_; + std::mutex lifecycle_mu_; +}; + +} // namespace dflash::engine diff --git a/server/src/gemma4/gemma4_backend.cpp b/server/src/gemma4/gemma4_backend.cpp index d9cdd9ad6..27c2f8256 100644 --- a/server/src/gemma4/gemma4_backend.cpp +++ b/server/src/gemma4/gemma4_backend.cpp @@ -20,13 +20,14 @@ #include #include #include +#include namespace dflash::common { // ── Ctor / dtor ──────────────────────────────────────────────────────── -Gemma4Backend::Gemma4Backend(const Gemma4BackendConfig & cfg) - : cfg_(cfg) {} +Gemma4Backend::Gemma4Backend(Gemma4BackendConfig cfg) + : cfg_(std::move(cfg)) {} Gemma4Backend::~Gemma4Backend() { shutdown(); } @@ -155,7 +156,8 @@ bool Gemma4Backend::unpark(ParkTarget target) { void Gemma4Backend::kvflash_read_config() { if (std::getenv("DFLASH_KVFLASH")) { - kvflash_drafter_path_ = kvflash_find_drafter(cfg_.model_path); + kvflash_drafter_path_ = kvflash_find_drafter( + cfg_.model_path.c_str()); } // "auto" sizes from the GPU (weights resident, cache not yet allocated): // gemma4 pools the FULL-attention layers only (F16 cache); SWA rings are @@ -1264,7 +1266,7 @@ bool Gemma4Backend::load_decode_draft() { std::fprintf(stderr, "[gemma4] draft CUDA init failed (gpu=%d)\n", draft_gpu); return false; } - if (!load_draft_gguf(cfg_.draft_path, draft_backend_, dw_, nullptr)) { + if (!load_draft_gguf(*cfg_.draft_path, draft_backend_, dw_, nullptr)) { std::fprintf(stderr, "[gemma4] draft load failed: %s\n", dflash27b_last_error()); ggml_backend_free(draft_backend_); draft_backend_ = nullptr; diff --git a/server/src/gemma4/gemma4_backend.h b/server/src/gemma4/gemma4_backend.h index ce05d84f0..689237c83 100644 --- a/server/src/gemma4/gemma4_backend.h +++ b/server/src/gemma4/gemma4_backend.h @@ -19,6 +19,7 @@ #include "ggml.h" #include "ggml-backend.h" +#include #include #include #include @@ -26,8 +27,8 @@ namespace dflash::common { struct Gemma4BackendConfig { - const char * model_path = nullptr; - const char * draft_path = nullptr; + std::string model_path; + std::optional draft_path; int draft_gpu = -1; // GPU for draft model (-1 = same as target) int draft_ctx_max = 2048; // max context for draft feature mirror DevicePlacement device; @@ -38,7 +39,7 @@ struct Gemma4BackendConfig { class Gemma4Backend : public ModelBackend { public: - explicit Gemma4Backend(const Gemma4BackendConfig & cfg); + explicit Gemma4Backend(Gemma4BackendConfig cfg); ~Gemma4Backend() override; Gemma4Backend(const Gemma4Backend &) = delete; diff --git a/server/src/gemma4/gemma4_daemon.cpp b/server/src/gemma4/gemma4_daemon.cpp index 8c653e001..fd97c597a 100644 --- a/server/src/gemma4/gemma4_daemon.cpp +++ b/server/src/gemma4/gemma4_daemon.cpp @@ -10,7 +10,7 @@ namespace dflash::common { int run_gemma4_daemon(const Gemma4DaemonArgs & args) { Gemma4BackendConfig cfg; - cfg.model_path = args.model_path; + cfg.model_path = args.model_path ? args.model_path : ""; cfg.device = args.device; cfg.stream_fd = args.stream_fd; cfg.chunk = args.chunk; diff --git a/server/src/gemma4/gemma4_layer_split_adapter.cpp b/server/src/gemma4/gemma4_layer_split_adapter.cpp index d89fca3a8..839c123d6 100644 --- a/server/src/gemma4/gemma4_layer_split_adapter.cpp +++ b/server/src/gemma4/gemma4_layer_split_adapter.cpp @@ -19,6 +19,7 @@ #include #include #include +#include namespace dflash::common { @@ -29,15 +30,15 @@ static bool tensor_ready(const ggml_tensor * t) { } static bool gemma4_align_split_for_kv_sharing( - const char * target_path, + const std::string & target_path, std::vector & shards) { - if (shards.size() <= 1 || !target_path) return true; + if (shards.size() <= 1 || target_path.empty()) return true; ggml_context * meta_ctx = nullptr; gguf_init_params gip{}; gip.no_alloc = true; gip.ctx = &meta_ctx; - gguf_context * gctx = gguf_init_from_file(target_path, gip); + gguf_context * gctx = gguf_init_from_file(target_path.c_str(), gip); if (!gctx) return true; int64_t arch_id = gguf_find_key(gctx, "general.architecture"); @@ -89,8 +90,8 @@ static bool gemma4_align_split_for_kv_sharing( } // namespace Gemma4LayerSplitAdapter::Gemma4LayerSplitAdapter( - const Gemma4LayerSplitAdapterConfig & cfg) - : cfg_(cfg) {} + Gemma4LayerSplitAdapterConfig cfg) + : cfg_(std::move(cfg)) {} Gemma4LayerSplitAdapter::~Gemma4LayerSplitAdapter() noexcept { try { @@ -106,7 +107,7 @@ bool Gemma4LayerSplitAdapter::init() { } const LayerSplitRuntimeInit runtime_cfg{ - cfg_.target_path, + cfg_.target_path.c_str(), &cfg_.device, "gemma4-target-split", }; @@ -196,7 +197,7 @@ bool Gemma4LayerSplitAdapter::init_mixed_target_split() { return false; } - const auto info = inspect_gguf_model_info(cfg_.target_path); + const auto info = inspect_gguf_model_info(cfg_.target_path.c_str()); const int n_layer = info.n_layer; if (n_layer <= 0) { std::fprintf(stderr, @@ -298,7 +299,7 @@ bool Gemma4LayerSplitAdapter::init_mixed_target_split() { TargetShardIpcLaunchConfig launch; launch.mode = BackendIpcMode::Gemma4TargetShard; launch.bin = cfg_.remote_target_shard.ipc_bin; - launch.target_path = cfg_.target_path ? cfg_.target_path : ""; + launch.target_path = cfg_.target_path; launch.gpus = remote_gpus; launch.layer_begins = remote_layer_begins; launch.layer_ends = remote_layer_ends; @@ -332,7 +333,7 @@ bool Gemma4LayerSplitAdapter::init_mixed_target_split() { void Gemma4LayerSplitAdapter::kvflash_read_config() { if (!std::getenv("DFLASH_KVFLASH") || shards_.empty()) return; - kvflash_drafter_path_ = kvflash_find_drafter(cfg_.target_path); + kvflash_drafter_path_ = kvflash_find_drafter(cfg_.target_path.c_str()); int64_t min_free = std::numeric_limits::max(); int64_t max_bytes_per_token = 0; diff --git a/server/src/gemma4/gemma4_layer_split_adapter.h b/server/src/gemma4/gemma4_layer_split_adapter.h index b4fb53ea3..5b1a2b1a3 100644 --- a/server/src/gemma4/gemma4_layer_split_adapter.h +++ b/server/src/gemma4/gemma4_layer_split_adapter.h @@ -22,7 +22,7 @@ namespace dflash::common { struct Gemma4LayerSplitAdapterConfig { - const char * target_path = nullptr; + std::string target_path; DevicePlacement device; RemoteTargetShardConfig remote_target_shard; int chunk = 512; @@ -44,7 +44,7 @@ struct Gemma4LayerSplitSnapshot { class Gemma4LayerSplitAdapter : public LayerSplitAdapter { public: - explicit Gemma4LayerSplitAdapter(const Gemma4LayerSplitAdapterConfig & cfg); + explicit Gemma4LayerSplitAdapter(Gemma4LayerSplitAdapterConfig cfg); ~Gemma4LayerSplitAdapter() noexcept override; Gemma4LayerSplitAdapter(const Gemma4LayerSplitAdapter &) = delete; diff --git a/server/src/laguna/laguna_backend.cpp b/server/src/laguna/laguna_backend.cpp index cf42b013c..bc97060d3 100644 --- a/server/src/laguna/laguna_backend.cpp +++ b/server/src/laguna/laguna_backend.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include "common/gguf_mmap.h" @@ -137,8 +138,8 @@ static float laguna_dspark_confidence_threshold() { // ── Construction / initialisation ─────────────────────────────────────── -LagunaBackend::LagunaBackend(const LagunaBackendArgs & args) - : args_(args) {} +LagunaBackend::LagunaBackend(LagunaBackendArgs args) + : args_(std::move(args)) {} LagunaBackend::~LagunaBackend() { shutdown(); } diff --git a/server/src/laguna/laguna_backend.h b/server/src/laguna/laguna_backend.h index e006c6446..342b6ccd6 100644 --- a/server/src/laguna/laguna_backend.h +++ b/server/src/laguna/laguna_backend.h @@ -56,7 +56,7 @@ struct LagunaDraftVariant { class LagunaBackend : public ModelBackend { public: - explicit LagunaBackend(const LagunaBackendArgs & args); + explicit LagunaBackend(LagunaBackendArgs args); ~LagunaBackend() override; // Initialise CUDA backend, load weights, create cache. diff --git a/server/src/laguna/laguna_daemon.cpp b/server/src/laguna/laguna_daemon.cpp index 952526581..52d187dde 100644 --- a/server/src/laguna/laguna_daemon.cpp +++ b/server/src/laguna/laguna_daemon.cpp @@ -15,6 +15,7 @@ #include "daemon_loop.h" #include +#include namespace dflash::common { @@ -26,7 +27,7 @@ int run_laguna_daemon(const LagunaDaemonArgs & args) { bargs.chunk = args.chunk; bargs.kv_type = args.kv_type; - LagunaBackend backend(bargs); + LagunaBackend backend(std::move(bargs)); if (!backend.init()) return 1; DaemonLoopArgs dargs; diff --git a/server/src/laguna/laguna_layer_split_adapter.cpp b/server/src/laguna/laguna_layer_split_adapter.cpp index 5c00f6666..3c5515cba 100644 --- a/server/src/laguna/laguna_layer_split_adapter.cpp +++ b/server/src/laguna/laguna_layer_split_adapter.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include namespace dflash::common { @@ -37,8 +38,8 @@ static bool tensor_ready(const ggml_tensor * t) { } // namespace LagunaLayerSplitAdapter::LagunaLayerSplitAdapter( - const LagunaLayerSplitAdapterConfig & cfg) - : cfg_(cfg) {} + LagunaLayerSplitAdapterConfig cfg) + : cfg_(std::move(cfg)) {} LagunaLayerSplitAdapter::~LagunaLayerSplitAdapter() { shutdown(); } @@ -48,7 +49,7 @@ bool LagunaLayerSplitAdapter::init() { } const LayerSplitRuntimeInit runtime_cfg{ - cfg_.target_path, + cfg_.target_path.c_str(), &cfg_.device, "laguna-target-split", }; @@ -131,7 +132,7 @@ bool LagunaLayerSplitAdapter::init_mixed_target_split() { return false; } - const auto info = inspect_gguf_model_info(cfg_.target_path); + const auto info = inspect_gguf_model_info(cfg_.target_path.c_str()); const int n_layer = info.n_layer; if (n_layer <= 0) { std::fprintf(stderr, @@ -214,7 +215,7 @@ bool LagunaLayerSplitAdapter::init_mixed_target_split() { TargetShardIpcLaunchConfig launch; launch.mode = BackendIpcMode::LagunaTargetShard; launch.bin = cfg_.remote_target_shard.ipc_bin; - launch.target_path = cfg_.target_path ? cfg_.target_path : ""; + launch.target_path = cfg_.target_path; launch.gpus = remote_gpus; launch.layer_begins = remote_layer_begins; launch.layer_ends = remote_layer_ends; @@ -258,7 +259,7 @@ KvFlashConfig LagunaLayerSplitAdapter::kvflash_config() const { void LagunaLayerSplitAdapter::kvflash_read_config() { if (!std::getenv("DFLASH_KVFLASH") || shards_.empty()) return; - kvflash_drafter_path_ = kvflash_find_drafter(cfg_.target_path); + kvflash_drafter_path_ = kvflash_find_drafter(cfg_.target_path.c_str()); int64_t min_free = std::numeric_limits::max(); int64_t max_bytes_per_token = 0; diff --git a/server/src/laguna/laguna_layer_split_adapter.h b/server/src/laguna/laguna_layer_split_adapter.h index 0131f0994..12085231c 100644 --- a/server/src/laguna/laguna_layer_split_adapter.h +++ b/server/src/laguna/laguna_layer_split_adapter.h @@ -23,7 +23,7 @@ namespace dflash::common { struct LagunaLayerSplitAdapterConfig { - const char * target_path = nullptr; + std::string target_path; DevicePlacement device; RemoteTargetShardConfig remote_target_shard; int chunk = 2048; @@ -44,7 +44,7 @@ struct LagunaLayerSplitSnapshot { class LagunaLayerSplitAdapter : public LayerSplitAdapter { public: - explicit LagunaLayerSplitAdapter(const LagunaLayerSplitAdapterConfig & cfg); + explicit LagunaLayerSplitAdapter(LagunaLayerSplitAdapterConfig cfg); ~LagunaLayerSplitAdapter() override; LagunaLayerSplitAdapter(const LagunaLayerSplitAdapter &) = delete; diff --git a/server/src/qwen3/qwen3_backend.cpp b/server/src/qwen3/qwen3_backend.cpp index 8748a67bc..fc9ed2daa 100644 --- a/server/src/qwen3/qwen3_backend.cpp +++ b/server/src/qwen3/qwen3_backend.cpp @@ -19,6 +19,7 @@ #include #include #include +#include namespace dflash::common { @@ -76,7 +77,8 @@ void free_qwen3_snapshot(Qwen3Snapshot & s) { // ── Construction / destruction ───────────────────────────────────────── -Qwen3Backend::Qwen3Backend(const Qwen3BackendConfig & cfg) : cfg_(cfg) {} +Qwen3Backend::Qwen3Backend(Qwen3BackendConfig cfg) + : cfg_(std::move(cfg)) {} Qwen3Backend::~Qwen3Backend() { shutdown(); } @@ -94,7 +96,7 @@ bool Qwen3Backend::init() { return false; } std::printf("[qwen3] loaded %s (%d layers, hidden=%d, vocab=%d)\n", - cfg_.model_path, w_.n_layer, w_.n_embd, w_.n_vocab); + cfg_.model_path.c_str(), w_.n_layer, w_.n_embd, w_.n_vocab); if (!create_qwen3_cache(backend_, w_, cfg_.device.max_ctx, cache_)) { std::fprintf(stderr, "[qwen3] cache creation failed\n"); diff --git a/server/src/qwen3/qwen3_backend.h b/server/src/qwen3/qwen3_backend.h index 0829a6b5f..d8d4c9668 100644 --- a/server/src/qwen3/qwen3_backend.h +++ b/server/src/qwen3/qwen3_backend.h @@ -27,7 +27,7 @@ namespace dflash::common { struct Qwen3BackendConfig { - const char * model_path = nullptr; + std::string model_path; DevicePlacement device; int stream_fd = -1; int chunk = 512; @@ -65,7 +65,7 @@ void free_qwen3_snapshot(Qwen3Snapshot & s); class Qwen3Backend : public ModelBackend { public: - explicit Qwen3Backend(const Qwen3BackendConfig & cfg); + explicit Qwen3Backend(Qwen3BackendConfig cfg); ~Qwen3Backend() override; Qwen3Backend(const Qwen3Backend &) = delete; diff --git a/server/src/qwen3/qwen3_daemon.cpp b/server/src/qwen3/qwen3_daemon.cpp index f5fd59132..6405767fc 100644 --- a/server/src/qwen3/qwen3_daemon.cpp +++ b/server/src/qwen3/qwen3_daemon.cpp @@ -10,7 +10,7 @@ namespace dflash::common { int run_qwen3_daemon(const Qwen3DaemonArgs & args) { Qwen3BackendConfig cfg; - cfg.model_path = args.model_path; + cfg.model_path = args.model_path ? args.model_path : ""; cfg.device = args.device; cfg.stream_fd = args.stream_fd; cfg.chunk = args.chunk; diff --git a/server/src/qwen35/qwen35_backend.cpp b/server/src/qwen35/qwen35_backend.cpp index 57d0f23d5..fecac5a32 100644 --- a/server/src/qwen35/qwen35_backend.cpp +++ b/server/src/qwen35/qwen35_backend.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #if !defined(_WIN32) #include #include @@ -256,7 +257,7 @@ static void apply_drafter_capture_layer_ids(const DraftWeights & dw, TargetWeigh // ── Construction / destruction ────────────────────────────────────────── -Qwen35Backend::Qwen35Backend(const Qwen35Config & cfg) : cfg_(cfg) {} +Qwen35Backend::Qwen35Backend(Qwen35Config cfg) : cfg_(std::move(cfg)) {} Qwen35Backend::~Qwen35Backend() { shutdown(); } @@ -339,7 +340,7 @@ bool Qwen35Backend::init() { const int cap = cfg_.remote_draft.ring_cap > 0 ? std::min(cfg_.remote_draft.ring_cap, cfg_.device.max_ctx) : std::min(cfg_.device.max_ctx, cfg_.draft_ctx_max); - if (!remote_draft_.start(cfg_.remote_draft.ipc_bin, cfg_.draft_path, + if (!remote_draft_.start(cfg_.remote_draft.ipc_bin, *cfg_.draft_path, cfg_.draft_gpu, cap, cfg_.remote_draft.work_dir)) { std::fprintf(stderr, "remote draft start failed\n"); @@ -351,10 +352,10 @@ bool Qwen35Backend::init() { std::printf("[draft] remote ipc ready gpu=%d cap=%d\n", cfg_.draft_gpu, cap); } else if (cfg_.draft_path) { - std::string dp(cfg_.draft_path); + const std::string & dp = *cfg_.draft_path; bool draft_ok = (dp.size() >= 5 && dp.substr(dp.size() - 5) == ".gguf") - ? load_draft_gguf(cfg_.draft_path, draft_backend_, dw_, &w_) - : load_draft_safetensors(cfg_.draft_path, draft_backend_, dw_, &w_); + ? load_draft_gguf(*cfg_.draft_path, draft_backend_, dw_, &w_) + : load_draft_safetensors(*cfg_.draft_path, draft_backend_, dw_, &w_); if (!draft_ok) { std::fprintf(stderr, "draft load: %s\n", dflash27b_last_error()); return false; @@ -419,7 +420,8 @@ bool Qwen35Backend::init() { // (or the explicit choice via --kvflash-policy lru). kvflash_qk_policy_ = kvflash_policy_is_qk(); if (std::getenv("DFLASH_KVFLASH") && !kvflash_qk_policy_) { - kvflash_drafter_path_ = kvflash_find_drafter(cfg_.target_path); + kvflash_drafter_path_ = kvflash_find_drafter( + cfg_.target_path.c_str()); } // "auto" sizes the pool from the GPU: weights are resident at this // point and the cache is not yet allocated, so device-free minus a @@ -914,17 +916,19 @@ bool Qwen35Backend::unpark(ParkTarget target) { const int cap = cfg_.remote_draft.ring_cap > 0 ? std::min(cfg_.remote_draft.ring_cap, cfg_.device.max_ctx) : std::min(cfg_.device.max_ctx, cfg_.draft_ctx_max); - if (!remote_draft_.start(cfg_.remote_draft.ipc_bin, cfg_.draft_path, + if (!remote_draft_.start( + cfg_.remote_draft.ipc_bin, *cfg_.draft_path, cfg_.draft_gpu, cap, cfg_.remote_draft.work_dir)) { std::fprintf(stderr, "[unpark] remote draft failed\n"); return false; } } else { - std::string dp(cfg_.draft_path); + const std::string & dp = *cfg_.draft_path; bool draft_ok = (dp.size() >= 5 && dp.substr(dp.size() - 5) == ".gguf") - ? load_draft_gguf(cfg_.draft_path, draft_backend_, dw_, &w_) - : load_draft_safetensors(cfg_.draft_path, draft_backend_, dw_, &w_); + ? load_draft_gguf(*cfg_.draft_path, draft_backend_, dw_, &w_) + : load_draft_safetensors( + *cfg_.draft_path, draft_backend_, dw_, &w_); if (!draft_ok) { std::fprintf(stderr, "[unpark] draft: %s\n", dflash27b_last_error()); return false; @@ -1484,12 +1488,18 @@ GenerateResult Qwen35Backend::generate_impl(const GenerateRequest & req, &result.degenerate_decode_close); out_io.emit(-1); } else { + const auto * hint_tokens = + req.hint_tokens.empty() ? nullptr : &req.hint_tokens; + const auto * stall_prefix = req.stall_tool_prefix_tokens.empty() + ? nullptr : &req.stall_tool_prefix_tokens; + const auto * stall_suffix = req.stall_action_suffix_tokens.empty() + ? nullptr : &req.stall_action_suffix_tokens; + const auto * stall_skip = req.stall_skip_tokens.empty() + ? nullptr : &req.stall_skip_tokens; decode_ok = do_spec_decode(committed, req.n_gen, result.tokens, out_io, result.accept_rate, result.spec_decode_ran, - req.hint_tokens, - req.stall_tool_prefix_tokens, - req.stall_action_suffix_tokens, - req.stall_skip_tokens, + hint_tokens, stall_prefix, stall_suffix, + stall_skip, &req.budget_hook, &result.budget_forced_close, &result.degenerate_decode_close); @@ -1652,12 +1662,18 @@ GenerateResult Qwen35Backend::restore_and_generate_impl(int slot, &result.degenerate_decode_close); out_io.emit(-1); } else { + const auto * hint_tokens = + req.hint_tokens.empty() ? nullptr : &req.hint_tokens; + const auto * stall_prefix = req.stall_tool_prefix_tokens.empty() + ? nullptr : &req.stall_tool_prefix_tokens; + const auto * stall_suffix = req.stall_action_suffix_tokens.empty() + ? nullptr : &req.stall_action_suffix_tokens; + const auto * stall_skip = req.stall_skip_tokens.empty() + ? nullptr : &req.stall_skip_tokens; decode_ok = do_spec_decode(committed, req.n_gen, result.tokens, out_io, result.accept_rate, result.spec_decode_ran, - req.hint_tokens, - req.stall_tool_prefix_tokens, - req.stall_action_suffix_tokens, - req.stall_skip_tokens, + hint_tokens, stall_prefix, stall_suffix, + stall_skip, &req.budget_hook, &result.budget_forced_close, &result.degenerate_decode_close); diff --git a/server/src/qwen35/qwen35_backend.h b/server/src/qwen35/qwen35_backend.h index 01bb1940d..58591db8e 100644 --- a/server/src/qwen35/qwen35_backend.h +++ b/server/src/qwen35/qwen35_backend.h @@ -46,8 +46,8 @@ class Qwen35TensorParallelContext; // ── Configuration passed at construction ──────────────────────────────── struct Qwen35Config { - const char * target_path = nullptr; - const char * draft_path = nullptr; + std::string target_path; + std::optional draft_path; DevicePlacement device; // target GPU placement int draft_gpu = 0; RemoteDraftConfig remote_draft; @@ -95,7 +95,7 @@ struct Qwen35Config { class Qwen35Backend : public ModelBackend { public: - explicit Qwen35Backend(const Qwen35Config & cfg); + explicit Qwen35Backend(Qwen35Config cfg); ~Qwen35Backend() override; // Non-copyable, non-movable (owns GPU resources). diff --git a/server/src/qwen35/qwen35_daemon.cpp b/server/src/qwen35/qwen35_daemon.cpp index d1a14a915..f79df8257 100644 --- a/server/src/qwen35/qwen35_daemon.cpp +++ b/server/src/qwen35/qwen35_daemon.cpp @@ -14,8 +14,8 @@ namespace dflash::common { int run_qwen35_daemon(const Qwen35DaemonArgs & args) { Qwen35Config cfg; - cfg.target_path = args.target_path; - cfg.draft_path = args.draft_path; + cfg.target_path = args.target_path ? args.target_path : ""; + if (args.draft_path) cfg.draft_path = args.draft_path; cfg.device = args.device; cfg.draft_gpu = args.draft_gpu; cfg.stream_fd = args.stream_fd; diff --git a/server/src/qwen35/qwen35_layer_split_adapter.cpp b/server/src/qwen35/qwen35_layer_split_adapter.cpp index 9b63e86f2..0f7ca0f77 100644 --- a/server/src/qwen35/qwen35_layer_split_adapter.cpp +++ b/server/src/qwen35/qwen35_layer_split_adapter.cpp @@ -27,12 +27,13 @@ #include #include #include +#include namespace dflash::common { Qwen35LayerSplitAdapter::Qwen35LayerSplitAdapter( - const Qwen35LayerSplitAdapterConfig & cfg) - : cfg_(cfg) {} + Qwen35LayerSplitAdapterConfig cfg) + : cfg_(std::move(cfg)) {} Qwen35LayerSplitAdapter::~Qwen35LayerSplitAdapter() { shutdown(); } @@ -42,7 +43,7 @@ bool Qwen35LayerSplitAdapter::init() { } const LayerSplitRuntimeInit runtime_cfg{ - cfg_.target_path, + cfg_.target_path.c_str(), &cfg_.device, "target-split", }; @@ -132,7 +133,7 @@ void Qwen35LayerSplitAdapter::kvflash_read_config() { cfg_.device.is_layer_split() && cfg_.remote_target_shard.enabled(); kvflash_drafter_path_ = target_shard_split ? std::string{} - : kvflash_find_drafter(cfg_.target_path); + : kvflash_find_drafter(cfg_.target_path.c_str()); ggml_type kv_k = GGML_TYPE_Q8_0; ggml_type kv_v = GGML_TYPE_Q8_0; @@ -297,7 +298,7 @@ bool Qwen35LayerSplitAdapter::init_mixed_target_split() { const size_t remote_begin = mixed_plan.remote_begin; const PlacementBackend remote_backend = mixed_plan.remote_backend; - const auto info = inspect_gguf_model_info(cfg_.target_path); + const auto info = inspect_gguf_model_info(cfg_.target_path.c_str()); const int n_layer = info.n_layer; if (n_layer <= 0) { std::fprintf(stderr, "[target-split] failed to inspect target layer count\n"); @@ -431,7 +432,8 @@ bool Qwen35LayerSplitAdapter::load_draft() { const int cap = cfg_.remote_draft.ring_cap > 0 ? std::min(cfg_.remote_draft.ring_cap, cfg_.device.max_ctx) : std::min(cfg_.device.max_ctx, cfg_.draft_ctx_max); - if (!remote_draft_.start(cfg_.remote_draft.ipc_bin, cfg_.draft_path, + if (!remote_draft_.start( + cfg_.remote_draft.ipc_bin, *cfg_.draft_path, cfg_.draft_gpu, cap, cfg_.remote_draft.work_dir)) { std::fprintf(stderr, @@ -468,12 +470,12 @@ bool Qwen35LayerSplitAdapter::load_draft() { draft_backend_owned_ = true; } - std::string draft_path(cfg_.draft_path ? cfg_.draft_path : ""); + const std::string & draft_path = *cfg_.draft_path; const bool draft_ok = draft_path.size() >= 5 && draft_path.substr(draft_path.size() - 5) == ".gguf" - ? load_draft_gguf(cfg_.draft_path, draft_backend_, draft_weights_, + ? load_draft_gguf(*cfg_.draft_path, draft_backend_, draft_weights_, &shards_.front().weights) - : load_draft_safetensors(cfg_.draft_path, draft_backend_, + : load_draft_safetensors(*cfg_.draft_path, draft_backend_, draft_weights_, &shards_.front().weights); if (!draft_ok) { std::fprintf(stderr, "[target-split] draft load gpu=%d: %s\n", diff --git a/server/src/qwen35/qwen35_layer_split_adapter.h b/server/src/qwen35/qwen35_layer_split_adapter.h index 7f5e7b129..06fb11c40 100644 --- a/server/src/qwen35/qwen35_layer_split_adapter.h +++ b/server/src/qwen35/qwen35_layer_split_adapter.h @@ -20,6 +20,7 @@ #include "ggml-backend.h" #include +#include #include #include #include @@ -27,8 +28,8 @@ namespace dflash::common { struct Qwen35LayerSplitAdapterConfig { - const char * target_path = nullptr; - const char * draft_path = nullptr; + std::string target_path; + std::optional draft_path; DevicePlacement device; int draft_gpu = 0; RemoteDraftConfig remote_draft; @@ -45,7 +46,7 @@ struct Qwen35LayerSplitAdapterConfig { class Qwen35LayerSplitAdapter : public LayerSplitAdapter { public: - explicit Qwen35LayerSplitAdapter(const Qwen35LayerSplitAdapterConfig & cfg); + explicit Qwen35LayerSplitAdapter(Qwen35LayerSplitAdapterConfig cfg); ~Qwen35LayerSplitAdapter() override; Qwen35LayerSplitAdapter(const Qwen35LayerSplitAdapter &) = delete; diff --git a/server/src/qwen35moe/qwen35moe_backend.cpp b/server/src/qwen35moe/qwen35moe_backend.cpp index 1f6ea7074..8d8534441 100644 --- a/server/src/qwen35moe/qwen35moe_backend.cpp +++ b/server/src/qwen35moe/qwen35moe_backend.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include "common/gguf_mmap.h" namespace dflash::common { @@ -74,8 +75,8 @@ static int qwen35moe_prefill_chunk_limit(int prompt_len) { } // namespace -Qwen35MoeBackend::Qwen35MoeBackend(const Qwen35Config & cfg) - : Qwen35Backend(cfg) {} +Qwen35MoeBackend::Qwen35MoeBackend(Qwen35Config cfg) + : Qwen35Backend(std::move(cfg)) {} bool Qwen35MoeBackend::init() { if (!Qwen35Backend::init()) { @@ -153,7 +154,8 @@ bool Qwen35MoeBackend::load_target_model(ggml_backend_t backend, TargetWeights & gguf_init_params gip{}; gip.no_alloc = true; gip.ctx = &expert_meta; - gguf_context * gctx = gguf_init_from_file(cfg_.target_path, gip); + gguf_context * gctx = + gguf_init_from_file(cfg_.target_path.c_str(), gip); if (!gctx) { set_last_error("failed to re-open GGUF for expert loading"); return false; @@ -320,7 +322,8 @@ bool Qwen35MoeBackend::rebuild_hybrid_from_placement(const MoeHybridPlacement & ggml_backend_t backend = target_backend(); gguf_init_params gip{}; - gguf_context * gctx = gguf_init_from_file(cfg_.target_path, gip); + gguf_context * gctx = + gguf_init_from_file(cfg_.target_path.c_str(), gip); if (!gctx) { err = "gguf reinit failed"; return false; } GgufMmap _mf; std::string _mferr; diff --git a/server/src/qwen35moe/qwen35moe_backend.h b/server/src/qwen35moe/qwen35moe_backend.h index 9c33e8553..7d63d4168 100644 --- a/server/src/qwen35moe/qwen35moe_backend.h +++ b/server/src/qwen35moe/qwen35moe_backend.h @@ -19,7 +19,7 @@ namespace dflash::common { class Qwen35MoeBackend : public Qwen35Backend { public: - explicit Qwen35MoeBackend(const Qwen35Config & cfg); + explicit Qwen35MoeBackend(Qwen35Config cfg); ~Qwen35MoeBackend() override = default; bool init() override; diff --git a/server/src/qwen35moe/qwen35moe_daemon.cpp b/server/src/qwen35moe/qwen35moe_daemon.cpp index 6ebae0e73..66da0eb40 100644 --- a/server/src/qwen35moe/qwen35moe_daemon.cpp +++ b/server/src/qwen35moe/qwen35moe_daemon.cpp @@ -7,8 +7,8 @@ namespace dflash::common { int run_qwen35moe_daemon(const Qwen35MoeDaemonArgs & args) { Qwen35Config cfg; - cfg.target_path = args.target_path; - cfg.draft_path = args.draft_path; + cfg.target_path = args.target_path ? args.target_path : ""; + if (args.draft_path) cfg.draft_path = args.draft_path; cfg.device = args.device; cfg.draft_gpu = args.draft_gpu; cfg.stream_fd = args.stream_fd; diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index a5a8032ae..c57fcc47e 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -17,6 +17,7 @@ #endif #include "http_server.h" +#include "engine/luce_engine.h" #include "admission.h" #include "sse_emitter.h" #include "prompt_normalize.h" @@ -1114,10 +1115,11 @@ static std::array compute_disk_cache_salt(const ServerConfig & cfg) // ─── HttpServer ───────────────────────────────────────────────────────── -HttpServer::HttpServer(ModelBackend & backend, +HttpServer::HttpServer(dflash::engine::LuceEngine & engine, Tokenizer & tokenizer, const ServerConfig & config) - : backend_(backend) + : engine_(engine) + , backend_(engine.backend()) , tokenizer_(tokenizer) , config_(config) , chat_format_(ChatFormat::QWEN3) // default, overridden by arch @@ -1126,7 +1128,7 @@ HttpServer::HttpServer(ModelBackend & backend, config.disk_cache_budget_mb * (size_t)(1024 * 1024), config.disk_cache_min_tokens, config.disk_cache_continued_interval, - config.disk_cache_cold_max_tokens}, backend) + config.disk_cache_cold_max_tokens}, backend_) { #ifdef DFLASH_HAS_CURL curl_global_init(CURL_GLOBAL_DEFAULT); @@ -1330,9 +1332,7 @@ void HttpServer::shutdown() { socket_close(listen_fd_); listen_fd_ = kInvalidSocket; } - if (worker_thread_.joinable()) { - worker_thread_.join(); - } + engine_.stop_serving(); // Close SSE client connections. { @@ -1437,15 +1437,18 @@ int HttpServer::run() { std::fprintf(stderr, "[server] listening on http://%s:%d\n", config_.host.c_str(), config_.port); - // A backend-provided sequence engine replaces the one-request worker - // with the concurrent scheduler. Upstream forwarding stays on the - // classic path even when the local backend exposes an engine. - if (SeqEngine * engine = backend_.seq_engine(); - engine && config_.pflash_upstream_base.empty()) { - worker_thread_ = - std::thread([this, engine]() { scheduler_loop(*engine); }); - } else { - worker_thread_ = std::thread([this]() { worker_loop(); }); + dflash::engine::LuceEngine::ServingLoops loops; + loops.serial = [this]() { worker_loop(); }; + loops.concurrent = + [this](SeqEngine & engine) { scheduler_loop(engine); }; + loops.request_stop = [this]() { + stopping_.store(true, std::memory_order_relaxed); + queue_cv_.notify_all(); + }; + if (!engine_.start_serving( + std::move(loops), config_.pflash_upstream_base.empty())) { + std::fprintf(stderr, "[server] failed to start LuceEngine\n"); + return 1; } // Accept loop. @@ -1505,10 +1508,8 @@ int HttpServer::run() { } } - // Wait for worker to finish. - if (worker_thread_.joinable()) { - worker_thread_.join(); - } + // Wait for LuceEngine's serving loop to finish. + engine_.stop_serving(); // Persist disk cache (worker joined — no race on slot_tokens_). if (!disk_cache_.disabled() && !slot_tokens_.empty()) { @@ -3081,6 +3082,9 @@ HttpServer::GenerationCacheState HttpServer::prepare_generation_cache( const ParsedRequest & req, PreparedPrompt & prepared, GenerateRequest & generate_request) { auto & effective_prompt = prepared.tokens; + const auto server_stopping = [this]() { + return stopping_.load(std::memory_order_acquire); + }; // Tool-heavy requests prefer the reusable system/tool boundary under eviction. const bool prefer_inline_snap = !req.tools.empty(); const bool prefer_tools_boundary = @@ -3298,6 +3302,7 @@ HttpServer::GenerationCacheState HttpServer::prepare_generation_cache( scoped_request.snap_pos = selected_boundary; DaemonIO scoped_io; scoped_io.stream_fd = -1; + scoped_io.should_cancel = server_stopping; const auto scoped_result = backend_.generate(scoped_request, scoped_io); if (scoped_result.ok() && @@ -3381,6 +3386,7 @@ HttpServer::GenerationCacheState HttpServer::prepare_generation_cache( cold_request.snap_pos = cold_boundary; DaemonIO cold_io; cold_io.stream_fd = -1; + cold_io.should_cancel = server_stopping; const auto cold_result = backend_.generate(cold_request, cold_io); if (cold_result.ok() && backend_.snapshot_used(kDiskStagingSlot)) { @@ -3693,7 +3699,10 @@ void HttpServer::remember_agent_turn( replay.snap_slot = slot; replay.snap_pos = canonical_end; DaemonIO replay_io; - replay_io.should_cancel = [this]() { return has_pending_jobs(); }; + replay_io.should_cancel = [this]() { + return stopping_.load(std::memory_order_acquire) || + has_pending_jobs(); + }; const GenerateResult replay_result = backend_.restore_and_generate( source_slot, replay, replay_io); backend_.release_scratch(); @@ -3715,8 +3724,8 @@ void HttpServer::remember_agent_turn( } } -// Generation setup owns backing storage for every pointer placed in -// GenerateRequest, keeping those pointers valid through the decode call. +// Populate model-ready input. GenerateRequest owns every retained token +// sequence, eliminating pointer lifetime coupling to GenerationInputs. void HttpServer::prepare_generation_inputs( const ParsedRequest & req, const PreparedPrompt & prepared, GenerationInputs & inputs) { @@ -3752,8 +3761,7 @@ void HttpServer::prepare_generation_inputs( ToolHintGenerator hint_generator(tokenizer_); auto hint = hint_generator.build_hint(req.tools, req.tool_choice); if (!hint.empty()) { - inputs.hint_tokens = std::move(hint.prefix_tokens); - inputs.request.hint_tokens = &inputs.hint_tokens; + inputs.request.hint_tokens = std::move(hint.prefix_tokens); } } @@ -3761,9 +3769,9 @@ void HttpServer::prepare_generation_inputs( return; } - inputs.stall_tool_prefix_tokens = tokenizer_.encode( + inputs.request.stall_tool_prefix_tokens = tokenizer_.encode( build_stall_tool_prefix(req.tools, req.tool_choice)); - inputs.stall_action_suffix_tokens = tokenizer_.encode(":"); + inputs.request.stall_action_suffix_tokens = tokenizer_.encode(":"); // The detector matches recent terminal tokens, not the full action // prefix. Collect the final token for common colon spellings. @@ -3771,30 +3779,26 @@ void HttpServer::prepare_generation_inputs( const auto ids = tokenizer_.encode(text); if (ids.empty()) return; const int32_t token = ids.back(); - if (std::find(inputs.stall_action_suffix_tokens.begin(), - inputs.stall_action_suffix_tokens.end(), token) == - inputs.stall_action_suffix_tokens.end()) { - inputs.stall_action_suffix_tokens.push_back(token); + if (std::find(inputs.request.stall_action_suffix_tokens.begin(), + inputs.request.stall_action_suffix_tokens.end(), token) == + inputs.request.stall_action_suffix_tokens.end()) { + inputs.request.stall_action_suffix_tokens.push_back(token); } }; add_suffix_terminal("`:"); add_suffix_terminal("):"); add_suffix_terminal("\":"); - inputs.stall_skip_tokens = tokenizer_.encode(" done"); - inputs.request.stall_tool_prefix_tokens = - &inputs.stall_tool_prefix_tokens; - inputs.request.stall_action_suffix_tokens = - &inputs.stall_action_suffix_tokens; - inputs.request.stall_skip_tokens = &inputs.stall_skip_tokens; + inputs.request.stall_skip_tokens = tokenizer_.encode(" done"); } void HttpServer::configure_generation_io( ServerJob * job, const ParsedRequest & req, SseEmitter & emitter, GenerationOutputState & output, DaemonIO & io) { io.stream_fd = -1; - io.should_cancel = [job]() { - return job->client_disconnected.load(std::memory_order_acquire); + io.should_cancel = [this, job]() { + return stopping_.load(std::memory_order_acquire) || + job->client_disconnected.load(std::memory_order_acquire); }; io.observer = [this](const char *, const std::vector & tokens) { std::vector token_strings; diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 52c36473b..2b621e118 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -4,7 +4,7 @@ // Architecture: // - Main thread: listen + accept // - Per-client thread: parse HTTP request, enqueue job, wait for completion -// - Single worker thread: dequeue jobs, call ModelBackend::generate() +// - LuceEngine execution thread: run the selected backend serving loop // // Client disconnect detection: the client thread watches the socket while the // worker generates, and streaming writes provide a second failure signal. @@ -49,6 +49,10 @@ #include #include +namespace dflash::engine { +class LuceEngine; +} + namespace dflash::common { using json = nlohmann::json; @@ -346,7 +350,7 @@ json build_props_body(const ServerConfig & config, // ─── HTTP server ──────────────────────────────────────────────────────── class HttpServer { public: - HttpServer(ModelBackend & backend, + HttpServer(dflash::engine::LuceEngine & engine, Tokenizer & tokenizer, const ServerConfig & config); ~HttpServer(); @@ -441,10 +445,6 @@ class HttpServer { struct GenerationInputs { GenerateRequest request; int generation_cap = 0; - std::vector hint_tokens; - std::vector stall_tool_prefix_tokens; - std::vector stall_action_suffix_tokens; - std::vector stall_skip_tokens; }; struct GenerationOutputState { @@ -540,6 +540,7 @@ class HttpServer { bool has_pending_jobs(); // Members. + dflash::engine::LuceEngine & engine_; ModelBackend & backend_; Tokenizer & tokenizer_; Tokenizer * drafter_tokenizer_ = nullptr; // pflash drafter (optional) @@ -599,8 +600,7 @@ class HttpServer { std::unordered_map frozen_content_cache_; - // Worker thread. - std::thread worker_thread_; + // Request queue consumed by the serving loop owned by LuceEngine. std::mutex queue_mu_; std::condition_variable queue_cv_; ServerJob * queue_head_ = nullptr; diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index e2db605fa..6da13157a 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -17,12 +17,14 @@ #include "common/backend_factory.h" #include "common/chain_rollback_policy.h" #include "common/layer_split_utils.h" +#include "common/model_capabilities.h" #include "common/spark_corpus.h" #include "common/moe_routing_collector.h" #include "common/moe_hybrid_routing_stats.h" #include "common/platform_env.h" #include "common/peer_access.h" #include "common/specla_mode.h" +#include "engine/luce_engine.h" #include "placement/pflash_placement.h" #include "placement/draft_residency.h" #include "kvflash_pager.h" @@ -249,10 +251,6 @@ int main(int argc, char ** argv) { bool target_devices_seen = false; bool fast_rollback_forced_off = false; bool target_split_fast_rollback_cli = false; - bool adaptive_experts_set = false; // --adaptive-experts (MoE architectures only) - bool ddtree_tau_set = false; - bool specla_top_k_set = false; - int specla_top_k = 4; // Track which thinking-budget tunables the operator set via CLI. // Those values win over the model card (spec §3.1: "Explicit CLI @@ -447,13 +445,15 @@ int main(int argc, char ** argv) { } else if (std::strcmp(argv[i], "--specla-top-k") == 0 && i + 1 < argc) { const char * value = argv[++i]; const char * end = value + std::strlen(value); - const auto parsed = std::from_chars(value, end, specla_top_k); - if (parsed.ec != std::errc{} || parsed.ptr != end || specla_top_k <= 0) { + const auto parsed = std::from_chars( + value, end, bargs.specla_top_k); + if (parsed.ec != std::errc{} || parsed.ptr != end || + bargs.specla_top_k <= 0) { std::fprintf(stderr, "--specla-top-k expects a positive integer, got '%s'\n", value); return 2; } - specla_top_k_set = true; + bargs.specla_top_k_explicit = true; } else if (std::strcmp(argv[i], "--ddtree") == 0) { bargs.ddtree_mode = true; bargs.fast_rollback = true; @@ -472,7 +472,7 @@ int main(int argc, char ** argv) { return 2; } bargs.ddtree_tau = tau; - ddtree_tau_set = true; + bargs.ddtree_tau_explicit = true; } else if (std::strcmp(argv[i], "--adaptive-experts") == 0) { const char * tau = "0.80"; if (i + 1 < argc && argv[i + 1][0] != '-') { @@ -486,7 +486,7 @@ int main(int argc, char ** argv) { return 1; } set_environment_variable("DFLASH_ADAPTIVE_K_TAU", tau, false); // explicit env still wins - adaptive_experts_set = true; + bargs.adaptive_experts_requested = true; } else if (std::strcmp(argv[i], "--verify-width") == 0 && i + 1 < argc) { bargs.verify_width = std::atoi(argv[++i]); } else if (std::strcmp(argv[i], "--no-fast-rollback") == 0) { @@ -677,7 +677,7 @@ int main(int argc, char ** argv) { return 2; } } - if (specla_top_k_set && !bargs.specla_mode) { + if (bargs.specla_top_k_explicit && !bargs.specla_mode) { std::fprintf(stderr, "[server] --specla-top-k requires --specla\n"); return 2; } @@ -686,8 +686,8 @@ int main(int argc, char ** argv) { "[server] --specla is incompatible with --no-fast-rollback\n"); return 2; } - if (bargs.specla_mode && !ddtree_tau_set) { - bargs.ddtree_tau = 6.0f; + if (bargs.specla_mode && !bargs.specla_top_k_explicit) { + bargs.specla_top_k = specla_tree_topk(); } if (fast_rollback_forced_off) { @@ -732,39 +732,58 @@ int main(int argc, char ** argv) { // Ask the factory to resolve model/placement facts and apply its feature // admission policy before any setup work. server_main only maps the // categorized result to the existing process exit convention. - BackendFeatureConfig backend_features; - backend_features.pflash_enabled = + bargs.routing_stats_requested = + sconfig.freq_tracking || !sconfig.collect_routing_path.empty(); + + BackendAdmissionContext backend_admission; + backend_admission.pflash_enabled = sconfig.pflash_mode != ServerConfig::PflashMode::OFF; - backend_features.pflash_drafter_configured = + backend_admission.pflash_drafter_configured = !sconfig.pflash_drafter_path.empty(); - backend_features.draft_residency = sconfig.draft_residency; - backend_features.routing_stats_requested = - sconfig.freq_tracking || !sconfig.collect_routing_path.empty(); - backend_features.adaptive_experts_requested = adaptive_experts_set; + backend_admission.draft_residency = sconfig.draft_residency; // Fixed pools are known incompatibilities before model setup. Automatic // sizing needs the backend's real VRAM budget; if it produces a live pool, // the backend rejects the pairing after sizing. - backend_features.kvflash_enabled = - kvflash_fixed_pool_requested(std::getenv("DFLASH_KVFLASH")); - const BackendPreparation backend_preparation = - prepare_backend(bargs, backend_features); - if (!backend_preparation.ok()) { + const char * kvflash_config = std::getenv("DFLASH_KVFLASH"); + backend_admission.kvflash = kvflash_fixed_pool_requested(kvflash_config) + ? KvFlashRequest::Fixed + : kvflash_pool_requested(kvflash_config) + ? KvFlashRequest::Auto + : KvFlashRequest::Off; + BackendPreparation backend_preparation = + prepare_backend(std::move(bargs), backend_admission); + if (const auto * failure = + std::get_if(&backend_preparation)) { + for (const std::string & warning : failure->warnings) { + std::fprintf(stderr, "[server] warning: %s\n", warning.c_str()); + } std::fprintf(stderr, "[server] %s\n", - backend_preparation.message.c_str()); - return backend_preparation.error == + failure->message.c_str()); + return failure->error == BackendPreparationError::FeatureCompatibility ? 2 : 1; } + BackendPlan backend_plan = + std::get(std::move(backend_preparation)); // Options that parsed cleanly but do nothing on this model. Reported up // front so they are visible before the backend's own startup chatter. - for (const std::string & warning : backend_preparation.warnings) { + for (const std::string & warning : backend_plan.warnings()) { std::fprintf(stderr, "[server] warning: %s\n", warning.c_str()); } - const ResolvedBackendPlan & backend_plan = backend_preparation.plan; + // All later reporting and serving setup reads the same grouped, + // normalized snapshot that backend construction consumes. + const BackendPlan::Model & backend_model = backend_plan.model(); + const BackendPlan::Placement & backend_placement = + backend_plan.placement(); + const BackendPlan::Cache & backend_cache = backend_plan.cache(); + const BackendPlan::Speculation & backend_speculation = + backend_plan.speculation(); + const BackendPlan::Execution & backend_execution = + backend_plan.execution(); + const BackendPlan::DeepSeek4 & backend_deepseek4 = + backend_plan.deepseek4(); const std::string & arch = backend_plan.arch(); - const bool kvflash_requested = - kvflash_pool_requested(std::getenv("DFLASH_KVFLASH")); if (target_split_fast_rollback_cli && arch != "qwen35") { std::fprintf(stderr, "[server] --target-split-fast-rollback is only supported for " @@ -772,54 +791,11 @@ int main(int argc, char ** argv) { return 2; } - // SpecLA is the verification mode, not a proposal algorithm. Select the - // proposal adapter supported by this model. Qwen3.6 currently has a - // DDTree adapter; future model families may select DSpark here instead. - if (bargs.specla_mode) { - const bool supported = arch == "qwen35" && !bargs.device.is_multi_device(); - if (supported) { - if (!bargs.draft_path) { - std::fprintf(stderr, - "[server] Qwen3.6 SpecLA requires --draft \n"); - return 2; - } - bargs.ddtree_mode = true; - if (kvflash_requested) { - // KVFlash installs a paged attention-KV layout and therefore - // disables the factor-cache migration required by SpecLA. - // Keep the compatible proposal adapter, but report the - // effective verification mode accurately. - std::fprintf(stderr, - "[server] warning: --specla is unavailable with KVFlash; " - "using ordinary DDTree verification\n"); - bargs.specla_mode = false; - unset_environment_variable("DFLASH_SPECLA"); - } else { - set_environment_variable("DFLASH_SPECLA", "1", true); - if (specla_top_k_set) { - set_environment_variable( - "DFLASH_SPECLA_TOPK", std::to_string(specla_top_k).c_str(), true); - } else { - specla_top_k = specla_tree_topk(); - } - } - } else { - std::fprintf(stderr, - "[server] warning: --specla is unavailable for architecture '%s' " - "with placement %s; using the architecture's normal decode path\n", - arch.c_str(), placement_device_name(bargs.device).c_str()); - bargs.specla_mode = false; - if (!ddtree_tau_set) { - bargs.ddtree_tau = std::numeric_limits::infinity(); - } - } - } - // Paged decode owns its K/V through a block table that the snapshot format // cannot describe yet, so the caches it would restore into are turned off. // This rewrites ServerConfig rather than rejecting the launch, which is why // it lives here and not in the gate. - if (bargs.paged_attention) { + if (backend_cache.paged_attention) { std::fprintf(stderr, "[server] --paged-attention disables prefix/prefill snapshots " "until their format stores page tables\n"); @@ -828,7 +804,7 @@ int main(int argc, char ** argv) { sconfig.disk_cache_dir.clear(); sconfig.disk_cache_policy.mode = DiskPrefixCacheMode::Off; } - if (sconfig.agent_turn_cache && bargs.paged_attention) { + if (sconfig.agent_turn_cache && backend_cache.paged_attention) { std::fprintf(stderr, "[server] --agent-turn-cache is not yet supported with " "--paged-attention or --max-concurrency\n"); @@ -844,11 +820,13 @@ int main(int argc, char ** argv) { // This prevents the HTTP server from accepting prompts larger than the // KV cache the backend actually allocates. if (sconfig.max_ctx <= 0) { - sconfig.max_ctx = bargs.device.max_ctx; + sconfig.max_ctx = backend_placement.target.max_ctx; } const PFlashDrafterPlacement pflash_placement = resolve_pflash_drafter_placement( - bargs.device, bargs.draft_device, bargs.remote_draft, + backend_placement.target, + backend_placement.draft, + backend_placement.remote_draft, sconfig.pflash_mode != ServerConfig::PflashMode::OFF); sconfig.pflash_drafter_gpu = pflash_placement.drafter_gpu; sconfig.pflash_remote_drafter = pflash_placement.remote_drafter; @@ -894,7 +872,7 @@ int main(int argc, char ** argv) { } if (sconfig.draft_residency == DraftResidencyPolicy::RequestScoped && - !(pflash_enabled || bargs.draft_path)) { + !(pflash_enabled || backend_speculation.draft_path)) { std::fprintf(stderr, "[server] --draft-residency=request-scoped ignored: requires " "--prefill-compression or --draft\n"); @@ -903,9 +881,11 @@ int main(int argc, char ** argv) { } // Load tokenizer. - std::fprintf(stderr, "[server] loading tokenizer from %s\n", bargs.model_path); + std::fprintf( + stderr, "[server] loading tokenizer from %s\n", + backend_model.path.c_str()); Tokenizer tokenizer; - if (!tokenizer.load_from_gguf(bargs.model_path)) { + if (!tokenizer.load_from_gguf(backend_model.path.c_str())) { std::fprintf(stderr, "[server] tokenizer load failed\n"); return 1; } @@ -938,7 +918,7 @@ int main(int argc, char ** argv) { } // Create backend. - g_peer_access_opt_in = bargs.device.peer_access; + g_peer_access_opt_in = backend_placement.target.peer_access; std::fprintf(stderr, "[server] creating backend...\n"); if (spark_autotune) { // Self-tuning hot/cold MoE residency: enable the bounded expert cache @@ -949,7 +929,8 @@ int main(int argc, char ** argv) { const bool is_laguna = (arch == "laguna"); if (arch_has_expert_offload(arch)) { const std::string pfx = is_laguna ? "DFLASH_LAGUNA_" : "DFLASH_QWEN35MOE_"; - const std::string profile = std::string(bargs.model_path) + ".spark.csv"; + const std::string profile = + backend_model.path + ".spark.csv"; std::FILE * pf = std::fopen(profile.c_str(), "rb"); const bool have_profile = (pf != nullptr); if (pf) std::fclose(pf); @@ -989,16 +970,18 @@ int main(int argc, char ** argv) { arch.c_str()); } } - auto backend = create_backend(bargs, backend_plan); - if (!backend) { + auto backend_owner = create_backend(backend_plan); + if (!backend_owner) { std::fprintf(stderr, "[server] backend creation failed\n"); return 1; } + ModelBackend * backend = backend_owner.get(); // Cross-check the capability table against the backend that was actually // built. arch_supports_remote_draft() admitted this launch from the arch // string alone; if the two ever disagree the table is stale, and failing // here beats routing draft work to a backend that cannot serve it. - if (bargs.remote_draft.enabled() && bargs.draft_path && + if (backend_placement.remote_draft.enabled() && + backend_speculation.draft_path && !backend->supports_remote_draft()) { std::fprintf(stderr, "[server] internal: architecture '%s' is listed as supporting " @@ -1010,14 +993,14 @@ int main(int argc, char ** argv) { // ── Thinking-budget v2: resolve model card and apply to ServerConfig ── // Reuse the metadata captured during factory preparation instead of // opening the GGUF header again. - const std::string & general_name = backend_plan.model().name; + const std::string & general_name = backend_model.metadata.name; const std::string & general_arch = backend_plan.arch(); std::fprintf(stderr, "[server] gguf meta: general.name='%s' general.architecture='%s'\n", general_name.c_str(), general_arch.c_str()); ModelCard card = resolve_model_card( - bargs.model_path ? bargs.model_path : "", + backend_model.path, general_name, general_arch, /*repo_root_hint=*/""); @@ -1112,7 +1095,8 @@ int main(int argc, char ** argv) { // Backends without hybrid/routing support skip this (live calibration still // applies). if (spark_autotune && backend->spark_wants_bootstrap()) { - const std::string spark_profile = std::string(bargs.model_path) + ".spark.csv"; + const std::string spark_profile = + backend_model.path + ".spark.csv"; std::FILE * spf = std::fopen(spark_profile.c_str(), "rb"); const bool spark_have_profile = (spf != nullptr); if (spf) std::fclose(spf); @@ -1150,8 +1134,14 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "[server] ╭─── Configuration ───────────────────────────────────╮\n"); std::fprintf(stderr, "[server] │ host = %s\n", sconfig.host.c_str()); std::fprintf(stderr, "[server] │ port = %d\n", sconfig.port); - std::fprintf(stderr, "[server] │ model = %s\n", bargs.model_path); - std::fprintf(stderr, "[server] │ draft = %s\n", bargs.draft_path ? bargs.draft_path : "(none)"); + std::fprintf( + stderr, "[server] │ model = %s\n", + backend_model.path.c_str()); + std::fprintf( + stderr, "[server] │ draft = %s\n", + backend_speculation.draft_path + ? backend_speculation.draft_path->c_str() + : "(none)"); std::fprintf(stderr, "[server] │ model_name = %s\n", sconfig.model_name.c_str()); std::fprintf(stderr, "[server] │ max_ctx = %d\n", sconfig.max_ctx); // max_tokens default for requests that omit the field. The request @@ -1181,77 +1171,88 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "[server] │ max=%d (%s)\n", sconfig.effort_tiers.max, src_of(cli_set.effort_max)); std::fprintf(stderr, "[server] │ target_device = %s\n", - placement_device_name(bargs.device).c_str()); + placement_device_name(backend_placement.target).c_str()); std::fprintf(stderr, "[server] │ target_split = %s\n", - target_split_mode_name(bargs.device.split_mode)); - if (bargs.device.is_multi_device()) { + target_split_mode_name(backend_placement.target.split_mode)); + if (backend_placement.target.is_multi_device()) { std::fprintf(stderr, "[server] │ target_devices ="); - for (size_t i = 0; i < bargs.device.layer_split_gpus.size(); ++i) { + for (size_t i = 0; i < backend_placement.target.layer_split_gpus.size(); ++i) { std::fprintf(stderr, " %s:%d", - placement_backend_name(bargs.device.layer_split_backend(i)), - bargs.device.layer_split_gpus[i]); + placement_backend_name( + backend_placement.target.layer_split_backend(i)), + backend_placement.target.layer_split_gpus[i]); } std::fprintf(stderr, "\n"); - if (bargs.remote_target_shard.enabled()) { + if (backend_placement.remote_target_shard.enabled()) { std::fprintf(stderr, "[server] │ target_shard_ipc= %s\n", - bargs.remote_target_shard.ipc_bin.c_str()); - if (!bargs.remote_target_shard.work_dir.empty()) { + backend_placement.remote_target_shard.ipc_bin.c_str()); + if (!backend_placement.remote_target_shard.work_dir.empty()) { std::fprintf(stderr, "[server] │ target_shard_dir= %s\n", - bargs.remote_target_shard.work_dir.c_str()); + backend_placement.remote_target_shard.work_dir.c_str()); } } } std::fprintf(stderr, "[server] │ draft_device = %s\n", - placement_device_name(bargs.draft_device).c_str()); + placement_device_name(backend_placement.draft).c_str()); std::fprintf(stderr, "[server] │ draft_exec = %s\n", - bargs.remote_draft.enabled() && bargs.draft_path ? "remote-ipc" : "local"); - if (bargs.remote_draft.enabled()) { + backend_placement.remote_draft.enabled() && + backend_speculation.draft_path + ? "remote-ipc" + : "local"); + if (backend_placement.remote_draft.enabled()) { std::fprintf(stderr, "[server] │ draft_ipc_bin = %s\n", - bargs.remote_draft.ipc_bin.c_str()); - if (!bargs.remote_draft.work_dir.empty()) { + backend_placement.remote_draft.ipc_bin.c_str()); + if (!backend_placement.remote_draft.work_dir.empty()) { std::fprintf(stderr, "[server] │ draft_ipc_dir = %s\n", - bargs.remote_draft.work_dir.c_str()); + backend_placement.remote_draft.work_dir.c_str()); } std::fprintf(stderr, "[server] │ draft_ipc_cap = %d\n", - bargs.remote_draft.ring_cap); + backend_placement.remote_draft.ring_cap); } std::fprintf(stderr, "[server] │ peer_access = %s\n", - bargs.device.peer_access ? "ON" : "off"); - std::fprintf(stderr, "[server] │ chunk = %d\n", bargs.chunk); + backend_placement.target.peer_access ? "ON" : "off"); + std::fprintf(stderr, "[server] │ chunk = %d\n", backend_execution.chunk); std::fprintf(stderr, "[server] │ admission_wait = %d ms\n", sconfig.admission_coalesce_ms); if (arch == "deepseek4") { std::fprintf(stderr, "[server] │ ds4_fused = %s\n", - bargs.ds4_fused_decode ? "ON" : "off"); + backend_deepseek4.fused_decode ? "ON" : "off"); std::fprintf(stderr, "[server] │ ds4_verify_f16kv= %s\n", - bargs.ds4_fused_verify_f16_kv ? "ON" : "off"); - if (bargs.ds4_expert_top_k > 0) { + backend_deepseek4.fused_verify_f16_kv ? "ON" : "off"); + if (backend_deepseek4.expert_top_k > 0) { std::fprintf(stderr, "[server] │ ds4_expert_topk= %d\n", - bargs.ds4_expert_top_k); + backend_deepseek4.expert_top_k); } else { std::fprintf(stderr, "[server] │ ds4_expert_topk= model default\n"); } std::fprintf(stderr, "[server] │ ds4_prefill = %s\n", - prefill_attention_mode_name(bargs.ds4_prefill_mode)); + prefill_attention_mode_name( + backend_deepseek4.prefill_mode)); } - std::fprintf(stderr, "[server] │ fa_window = %d\n", bargs.fa_window); - if (bargs.fa_window > 0) { + std::fprintf(stderr, "[server] │ fa_window = %d\n", backend_cache.fa_window); + if (backend_cache.fa_window > 0) { std::fprintf(stderr, "[server] │ ⚠ fa_window > 0 drops system prompt / " "tool definitions from attention at long contexts.\n" "[server] │ Use --fa-window 0 for tool-call workloads.\n"); } - std::fprintf(stderr, "[server] │ ddtree = %s\n", bargs.ddtree_mode ? "ON" : "off"); - std::fprintf(stderr, "[server] │ specla = %s\n", bargs.specla_mode ? "ON" : "off"); - if (bargs.specla_mode) { - std::fprintf(stderr, "[server] │ specla_top_k = %d\n", specla_top_k); - std::fprintf(stderr, "[server] │ ddtree_tau = %.3g\n", bargs.ddtree_tau); + std::fprintf(stderr, "[server] │ ddtree = %s\n", + backend_speculation.ddtree_mode ? "ON" : "off"); + std::fprintf(stderr, "[server] │ specla = %s\n", + backend_speculation.specla_mode ? "ON" : "off"); + if (backend_speculation.specla_mode) { + std::fprintf(stderr, "[server] │ specla_top_k = %d\n", + backend_speculation.specla_top_k); + std::fprintf(stderr, "[server] │ ddtree_tau = %.3g\n", + backend_speculation.ddtree_tau); } - std::fprintf(stderr, "[server] │ fast_rollback = %s\n", bargs.fast_rollback ? "ON" : "off"); - if (bargs.device.is_layer_split()) { + std::fprintf(stderr, "[server] │ fast_rollback = %s\n", + backend_speculation.fast_rollback ? "ON" : "off"); + if (backend_placement.target.is_layer_split()) { std::fprintf(stderr, "[server] │ split_rollback = %s\n", split_chain_fast_rollback_enabled() ? "ON" : "off"); } - std::fprintf(stderr, "[server] │ ddtree_budget = %d\n", bargs.ddtree_budget); + std::fprintf(stderr, "[server] │ ddtree_budget = %d\n", + backend_speculation.ddtree_budget); std::fprintf(stderr, "[server] │ prefix_cache = %d slots\n", sconfig.prefix_cache_cap); std::fprintf(stderr, "[server] │ agent_turn_cache= %s\n", sconfig.agent_turn_cache ? "ON" : "off"); @@ -1285,7 +1286,7 @@ int main(int argc, char ** argv) { } std::fprintf(stderr, "[server] │ draft_residency = %s\n", draft_residency_policy_name(sconfig.draft_residency)); - if (bargs.draft_path) { + if (backend_speculation.draft_path) { std::fprintf(stderr, "[server] │ lazy_draft = %s\n", sconfig.lazy_draft ? "ON" : "off"); } std::fprintf(stderr, "[server] ╰─────────────────────────────────────────────────────╯\n\n"); @@ -1294,12 +1295,12 @@ int main(int argc, char ** argv) { // — the /props handler reads them lockless from config_ so they need to // be set BEFORE the HttpServer constructor copies sconfig. sconfig.arch = arch; - sconfig.model_path = bargs.model_path ? bargs.model_path : ""; - sconfig.draft_path = bargs.draft_path ? bargs.draft_path : ""; - sconfig.fa_window = bargs.fa_window; - sconfig.ddtree_budget = bargs.ddtree_budget; - sconfig.speculative_enabled = bargs.ddtree_mode; - sconfig.target_sharding = bargs.device.is_layer_split(); + sconfig.model_path = backend_model.path; + sconfig.draft_path = backend_speculation.draft_path.value_or(""); + sconfig.fa_window = backend_cache.fa_window; + sconfig.ddtree_budget = backend_speculation.ddtree_budget; + sconfig.speculative_enabled = backend_speculation.ddtree_mode; + sconfig.target_sharding = backend_placement.target.is_layer_split(); // KV type: report the operator's choice if set, else the family default // the backend resolves (the tq3_0 auto policy was removed; laguna uses // q8_0, base default q4_0). Matches the printed table above. @@ -1311,10 +1312,10 @@ int main(int argc, char ** argv) { #else "cuda"; #endif - sconfig.chunk = bargs.chunk; - sconfig.target_device = placement_device_name(bargs.device); - sconfig.draft_device = bargs.draft_path - ? placement_device_name(bargs.draft_device) + sconfig.chunk = backend_execution.chunk; + sconfig.target_device = placement_device_name(backend_placement.target); + sconfig.draft_device = backend_speculation.draft_path + ? placement_device_name(backend_placement.draft) : std::string(); // Tokenizer ID: best-effort. The Tokenizer class doesn't currently // expose the GGUF metadata key it was loaded from, so leave empty @@ -1371,7 +1372,8 @@ int main(int argc, char ** argv) { } } - HttpServer server(*backend, tokenizer, sconfig); + dflash::engine::LuceEngine engine(std::move(backend_owner)); + HttpServer server(engine, tokenizer, sconfig); server.set_chat_format(chat_format_for_arch(arch)); g_server = &server; std::signal(SIGTERM, signal_handler); @@ -1381,7 +1383,7 @@ int main(int argc, char ** argv) { } // Lazy-draft: park decode draft at startup to free VRAM (~3.3 GB). - if (sconfig.lazy_draft && bargs.draft_path) { + if (sconfig.lazy_draft && backend_speculation.draft_path) { backend->park(ParkTarget::DraftModel); } @@ -1419,7 +1421,5 @@ int main(int argc, char ** argv) { routing_collector.close(); } - // Cleanup. - backend->shutdown(); return ret; } diff --git a/server/test/test_backend_plan.cpp b/server/test/test_backend_plan.cpp new file mode 100644 index 000000000..94797f17e --- /dev/null +++ b/server/test/test_backend_plan.cpp @@ -0,0 +1,209 @@ +// Unit tests for model-aware backend request normalization. +// +// These tests call the internal builder with already-inspected GGUF facts, so +// policy stays testable without a model file, a GPU, or backend construction. + +#include "CppUnitTestFramework.hpp" +#include "common/backend_plan_internal.h" + +#include +#include +#include +#include +#include + +using namespace CppUnitTestFramework; +using namespace dflash::common; + +namespace { + +template +struct CanCreateBackend : std::false_type {}; + +template +struct CanCreateBackend< + T, + std::void_t()))>> + : std::true_type {}; + +template +struct HasFlatArgsView : std::false_type {}; + +template +struct HasFlatArgsView< + T, + std::void_t().args())>> + : std::true_type {}; + +static_assert(CanCreateBackend::value); +static_assert(CanCreateBackend::value); +static_assert(!CanCreateBackend::value); +static_assert(std::is_same_v< + decltype(create_backend(std::declval())), + std::unique_ptr>); +static_assert(!HasFlatArgsView::value); +static_assert(std::is_move_constructible_v); +static_assert(!std::is_move_assignable_v); + +struct BackendPlanFixture : CommonFixture { + using CommonFixture::CommonFixture; + +BackendPreparation resolve( + BackendArgs args, + const std::string & arch, + BackendAdmissionContext admission = {}) { + GgufModelInfo model; + model.arch = arch; + model.name = "test-model"; + return detail::BackendPlanBuilder::resolve( + std::move(args), + std::move(admission), + std::move(model), + compiled_placement_backend()); +} + +BackendArgs plain_args() { + BackendArgs args; + args.model_path = "/models/target.gguf"; + return args; +} + +void test_plan_owns_the_effective_request() { + std::string target = "/models/target.gguf"; + std::string draft = "/models/draft.gguf"; + BackendArgs args = plain_args(); + args.model_path = target; + args.draft_path = draft; + args.device.gpu = 3; + args.fa_window = 128; + args.chunk = 256; + + BackendPreparation result = resolve(std::move(args), "qwen35"); + CHECK(std::holds_alternative(result)); + BackendPlan plan = std::get(std::move(result)); + + target.assign("changed"); + draft.assign("changed"); + CHECK(plan.model().path == "/models/target.gguf"); + CHECK(plan.model().metadata.name == "test-model"); + CHECK(plan.speculation().draft_path == "/models/draft.gguf"); + CHECK(plan.placement().target.gpu == 3); + CHECK(plan.cache().fa_window == 128); + CHECK(plan.execution().chunk == 256); + CHECK(plan.arch() == "qwen35"); +} + +void test_supported_specla_selects_ddtree() { + BackendArgs args = plain_args(); + args.draft_path = "/models/draft.gguf"; + args.specla_mode = true; + + BackendPreparation result = resolve(std::move(args), "qwen35"); + CHECK(std::holds_alternative(result)); + const BackendPlan & plan = std::get(result); + CHECK(plan.speculation().specla_mode); + CHECK(plan.speculation().ddtree_mode); + CHECK(plan.speculation().ddtree_tau == 6.0f); + CHECK(plan.warnings().empty()); +} + +void test_deepseek_options_have_an_explicit_group() { + BackendArgs args = plain_args(); + args.ds4_expert_top_k = 4; + + BackendPreparation result = resolve(std::move(args), "deepseek4"); + CHECK(std::holds_alternative(result)); + const BackendPlan & plan = std::get(result); + CHECK(plan.deepseek4().expert_top_k == 4); + CHECK(plan.execution().chunk == 512); +} + +void test_explicit_specla_tau_is_preserved() { + BackendArgs args = plain_args(); + args.draft_path = "/models/draft.gguf"; + args.specla_mode = true; + args.ddtree_tau = 2.5f; + args.ddtree_tau_explicit = true; + + BackendPreparation result = resolve(std::move(args), "qwen35"); + CHECK(std::holds_alternative(result)); + const BackendPlan & plan = std::get(result); + CHECK(plan.speculation().specla_mode); + CHECK(plan.speculation().ddtree_mode); + CHECK(plan.speculation().ddtree_tau == 2.5f); +} + +void test_kvflash_falls_back_to_ordinary_ddtree() { + BackendArgs args = plain_args(); + args.draft_path = "/models/draft.gguf"; + args.specla_mode = true; + BackendAdmissionContext admission; + admission.kvflash = KvFlashRequest::Auto; + + BackendPreparation result = + resolve(std::move(args), "qwen35", admission); + CHECK(std::holds_alternative(result)); + const BackendPlan & plan = std::get(result); + CHECK(!plan.speculation().specla_mode); + CHECK(plan.speculation().ddtree_mode); + CHECK(std::isinf(plan.speculation().ddtree_tau)); + CHECK(plan.warnings().size() == 1); +} + +void test_kvflash_fallback_preserves_explicit_tau() { + BackendArgs args = plain_args(); + args.draft_path = "/models/draft.gguf"; + args.specla_mode = true; + args.ddtree_tau = 2.5f; + args.ddtree_tau_explicit = true; + BackendAdmissionContext admission; + admission.kvflash = KvFlashRequest::Auto; + + BackendPreparation result = + resolve(std::move(args), "qwen35", admission); + CHECK(std::holds_alternative(result)); + const BackendPlan & plan = std::get(result); + CHECK(!plan.speculation().specla_mode); + CHECK(plan.speculation().ddtree_mode); + CHECK(plan.speculation().ddtree_tau == 2.5f); +} + +void test_unsupported_specla_falls_back_without_hidden_tau() { + BackendArgs args = plain_args(); + args.specla_mode = true; + + BackendPreparation result = resolve(std::move(args), "qwen3"); + CHECK(std::holds_alternative(result)); + const BackendPlan & plan = std::get(result); + CHECK(!plan.speculation().specla_mode); + CHECK(!plan.speculation().ddtree_mode); + CHECK(std::isinf(plan.speculation().ddtree_tau)); + CHECK(plan.warnings().size() == 1); +} + +void test_supported_specla_requires_a_draft() { + BackendArgs args = plain_args(); + args.specla_mode = true; + + BackendPreparation result = resolve(std::move(args), "qwen35"); + CHECK(std::holds_alternative(result)); + const BackendPreparationFailure & failure = + std::get(result); + CHECK(failure.error == BackendPreparationError::FeatureCompatibility); + CHECK(failure.message == "Qwen3.6 SpecLA requires --draft "); +} + +}; + +} // namespace + +TEST_CASE(BackendPlanFixture, backend_plan_suite) { + test_plan_owns_the_effective_request(); + test_deepseek_options_have_an_explicit_group(); + test_supported_specla_selects_ddtree(); + test_explicit_specla_tau_is_preserved(); + test_kvflash_falls_back_to_ordinary_ddtree(); + test_kvflash_fallback_preserves_explicit_tau(); + test_unsupported_specla_falls_back_without_hidden_tau(); + test_supported_specla_requires_a_draft(); +} diff --git a/server/test/test_feature_gate.cpp b/server/test/test_feature_gate.cpp index 787e0c6af..dbed8b5db 100644 --- a/server/test/test_feature_gate.cpp +++ b/server/test/test_feature_gate.cpp @@ -1,11 +1,9 @@ -// Unit tests for the backend feature/architecture gate. +// Unit tests for lightweight backend planning policy. // -// check_feature_compatibility(), collect_feature_warnings() and the -// model_capabilities.h table are pure functions over resolved facts, so this -// binary needs no model file, no GPU, and none of the backend stack — it -// compiles against feature_gate.cpp and placement_config.cpp alone. Keeping -// it separate from test_server_unit keeps that true: a gate rule stays -// testable in seconds rather than behind a full CUDA build. +// The plan builder, feature gate, and model_capabilities.h table operate on +// resolved facts. This binary needs no model file, GPU, or backend stack. +// Keeping it separate from test_server_unit keeps policy rules testable in +// seconds rather than behind a full CUDA build. // // Build: cmake --build . --target test_feature_gate // Run: ./test_feature_gate @@ -45,7 +43,7 @@ static std::string gate_result( const BackendArgs & args, const std::string & arch, PlacementBackend backend, - const BackendFeatureConfig & features = {}) { + const BackendAdmissionContext & features = {}) { return check_feature_compatibility( args, features, arch, backend, backend); } @@ -55,7 +53,7 @@ static std::string gate_result_for_binary( const std::string & arch, PlacementBackend target_backend, PlacementBackend compiled_backend, - const BackendFeatureConfig & features = {}) { + const BackendAdmissionContext & features = {}) { return check_feature_compatibility( args, features, arch, target_backend, compiled_backend); } @@ -150,7 +148,7 @@ void test_feature_gate_pflash_requires_drafter_and_supported_arch() { BackendArgs args; args.model_path = "/nonexistent/model.gguf"; - BackendFeatureConfig features; + BackendAdmissionContext features; features.pflash_enabled = true; CHECK(!gate_result( args, "qwen35", PlacementBackend::Cuda, features).empty()); @@ -238,7 +236,7 @@ void test_feature_gate_tensor_parallel_requirements() { CHECK(!gate_result( remote, "qwen35", PlacementBackend::Cuda).empty()); - BackendFeatureConfig pflash; + BackendAdmissionContext pflash; pflash.pflash_enabled = true; pflash.pflash_drafter_configured = true; CHECK(!gate_result( @@ -350,7 +348,7 @@ void test_feature_gate_remote_draft_requires_supported_arch() { // Without a draft model or PFlash, remote draft IPC is unnecessary. BackendArgs no_draft = args; - no_draft.draft_path = nullptr; + no_draft.draft_path.reset(); CHECK(!gate_result( no_draft, "gemma4", PlacementBackend::Cuda).empty()); } @@ -427,13 +425,13 @@ void test_feature_gate_paged_attention_allows_fixed_local_chains() { CHECK(gate_result( concurrent_chain, "qwen35", PlacementBackend::Hip).empty()); - BackendFeatureConfig request_scoped; + BackendAdmissionContext request_scoped; request_scoped.draft_residency = DraftResidencyPolicy::RequestScoped; CHECK(!gate_result( concurrent_chain, "qwen35", PlacementBackend::Cuda, request_scoped).empty()); - BackendFeatureConfig persistent; + BackendAdmissionContext persistent; persistent.draft_residency = DraftResidencyPolicy::Persistent; CHECK(gate_result( concurrent_chain, "qwen35", PlacementBackend::Cuda, @@ -453,14 +451,14 @@ void test_feature_gate_paged_attention_allows_fixed_local_chains() { CHECK(!gate_result( windowed, "qwen35", PlacementBackend::Cuda).empty()); - BackendFeatureConfig pflash; + BackendAdmissionContext pflash; pflash.pflash_enabled = true; pflash.pflash_drafter_configured = true; CHECK(!gate_result( base, "qwen35", PlacementBackend::Cuda, pflash).empty()); - BackendFeatureConfig kvflash; - kvflash.kvflash_enabled = true; + BackendAdmissionContext kvflash; + kvflash.kvflash = KvFlashRequest::Fixed; CHECK(!gate_result( base, "qwen35", PlacementBackend::Cuda, kvflash).empty()); @@ -581,11 +579,11 @@ void test_feature_gate_parallel_and_kv_pool_rules() { std::vector warn_result( const BackendArgs & args, const std::string & arch, - const BackendFeatureConfig & features = {}) { + const BackendAdmissionContext & features = {}) { CHECK(check_feature_compatibility( args, features, arch, compiled_placement_backend(), compiled_placement_backend()).empty()); - return collect_feature_warnings(args, features, arch); + return collect_feature_warnings(args, arch); } static bool warns_about(const std::vector & warnings, @@ -622,7 +620,7 @@ void test_feature_warnings_report_inert_draft() { BackendArgs split = args; CHECK(parse_placement_device_list("cuda:0,cuda:1", split.device)); - const std::vector w = collect_feature_warnings(split, {}, "laguna"); + const std::vector w = collect_feature_warnings(split, "laguna"); CHECK(warns_about(w, "--draft")); CHECK(w[0].find("single-device placement") != std::string::npos); } @@ -666,14 +664,13 @@ void test_feature_warnings_report_inert_moe_options() { BackendArgs args; args.model_path = "/nonexistent/model.gguf"; - BackendFeatureConfig moe_opts; - moe_opts.routing_stats_requested = true; - moe_opts.adaptive_experts_requested = true; + args.routing_stats_requested = true; + args.adaptive_experts_requested = true; - CHECK(warn_result(args, "laguna", moe_opts).empty()); - CHECK(warn_result(args, "qwen35moe", moe_opts).empty()); - CHECK(warn_result(args, "qwen35", moe_opts).size() == 2); - CHECK(warn_result(args, "deepseek4", moe_opts).size() == 2); + CHECK(warn_result(args, "laguna").empty()); + CHECK(warn_result(args, "qwen35moe").empty()); + CHECK(warn_result(args, "qwen35").size() == 2); + CHECK(warn_result(args, "deepseek4").size() == 2); } void test_model_capability_tables() { diff --git a/server/test/test_generation.cpp b/server/test/test_generation.cpp new file mode 100644 index 000000000..266676d8e --- /dev/null +++ b/server/test/test_generation.cpp @@ -0,0 +1,312 @@ +#include "CppUnitTestFramework.hpp" +#include "engine/generation.h" + +#include +#include + +using namespace CppUnitTestFramework; +using namespace dflash::common; +using namespace dflash::engine; + +namespace { + +static_assert(!std::is_copy_constructible_v); +static_assert(std::is_move_constructible_v); +static_assert(!std::is_copy_constructible_v); +static_assert(std::is_move_constructible_v); + +GenerateRequest owned_request() { + GenerateRequest request; + request.prompt = {1, 2}; + request.hint_tokens = {3, 4}; + request.stall_tool_prefix_tokens = {5}; + request.stall_action_suffix_tokens = {6}; + request.stall_skip_tokens = {7}; + return request; +} + +struct GenerationFixture : CommonFixture { + using CommonFixture::CommonFixture; + + void request_payloads_are_owned() { + GenerateRequest request = owned_request(); + CHECK(request.prompt == std::vector({1, 2})); + CHECK(request.hint_tokens == std::vector({3, 4})); + CHECK(request.stall_tool_prefix_tokens == std::vector({5})); + CHECK(request.stall_action_suffix_tokens == std::vector({6})); + CHECK(request.stall_skip_tokens == std::vector({7})); + } + + void tokens_precede_terminal_result() { + GenerationPair pair = make_generation(2); + CHECK(pair.source.publish(TokenBatch{{11, 12}})); + GenerateResult result; + result.tokens = {11, 12}; + result.succeed(); + CHECK(pair.source.complete(result)); + CHECK(!pair.source.complete(result)); + + auto token_event = pair.generation.next(); + CHECK(std::get(token_event).tokens == + std::vector({11, 12})); + auto terminal_event = pair.generation.next(); + CHECK(std::get(terminal_event).result.ok()); + } + + void progress_is_coalesced() { + GenerationPair pair = make_generation(1); + CHECK(pair.source.publish(GenerationProgress{GenerationPhase::Prefill, 4})); + CHECK(pair.source.publish(GenerationProgress{GenerationPhase::Prefill, 9})); + auto event = pair.generation.next(); + const auto & progress = std::get(event); + CHECK(progress.phase == GenerationPhase::Prefill); + CHECK(progress.processed_tokens == 9); + } + + void full_channel_cancels_only_its_generation() { + GenerationPair full = make_generation(1); + GenerationPair healthy = make_generation(1); + CHECK(full.source.publish(TokenBatch{{1}})); + CHECK(!full.source.publish(TokenBatch{{2}})); + CHECK(full.source.cancellation_reason() == + GenerationCancelReason::OutputBackpressure); + + GenerateResult ok; + ok.succeed(); + CHECK(healthy.source.complete(ok)); + CHECK(std::get(healthy.generation.next()).result.ok()); + + CHECK(std::get(full.generation.next()).tokens == + std::vector({1})); + const auto terminal = + std::get(full.generation.next()); + CHECK(terminal.result.error->code == + GenerateErrorCode::OutputBackpressure); + } + + void token_capacity_counts_tokens_not_batches() { + GenerationPair pair = make_generation(2); + CHECK(pair.source.publish(TokenBatch{})); + CHECK(pair.source.publish(TokenBatch{})); + CHECK(pair.source.publish(TokenBatch{{1, 2}})); + CHECK(std::get(pair.generation.next()).tokens == + std::vector({1, 2})); + + CHECK(pair.source.publish(TokenBatch{{3, 4}})); + CHECK(!pair.source.publish(TokenBatch{{5}})); + CHECK(std::get(pair.generation.next()).tokens == + std::vector({3, 4})); + const auto terminal = + std::get(pair.generation.next()); + CHECK(terminal.result.error->code == + GenerateErrorCode::OutputBackpressure); + } + + void cancellation_is_idempotent() { + GenerationPair pair = make_generation(1); + CHECK(pair.generation.cancel(GenerationCancelReason::ClientDisconnected)); + CHECK(!pair.generation.cancel(GenerationCancelReason::Shutdown)); + CHECK(pair.source.cancellation_reason() == + GenerationCancelReason::ClientDisconnected); + const auto terminal = + std::get(pair.generation.next()); + CHECK(terminal.result.error->code == GenerateErrorCode::Cancelled); + } + + void source_destruction_preserves_terminal_result() { + for (auto reason : {GenerationCancelReason::ConsumerDropped, + GenerationCancelReason::ClientDisconnected, + GenerationCancelReason::OutputBackpressure, + GenerationCancelReason::Shutdown}) { + GenerationPair pair = make_generation(1); + CHECK(pair.generation.cancel(reason)); + const auto before = std::get(pair.generation.next()); + CHECK(!pair.source.publish(TokenBatch{{1}})); + CHECK(!pair.source.publish(GenerationProgress{})); + pair.source = GenerationSource{}; + const auto after = std::get(pair.generation.next()); + CHECK(after.result.error->code == before.result.error->code); + } + + GenerationPair pair = make_generation(1); + GenerateResult result; + result.tokens = {7}; + result.succeed(); + CHECK(pair.source.complete(result)); + CHECK(!pair.generation.cancel(GenerationCancelReason::Shutdown)); + pair.source = GenerationSource{}; + const auto terminal = std::get(pair.generation.next()); + CHECK(terminal.result.ok()); + CHECK(terminal.result.tokens == result.tokens); + } + + void dropping_handle_requests_cancellation() { + GenerationPair pair = make_generation(1); + pair.generation = Generation{}; + CHECK(pair.source.cancellation_reason() == + GenerationCancelReason::ConsumerDropped); + } + + void source_destruction_wakes_handle() { + Generation generation; + { + GenerationPair pair = make_generation(1); + generation = std::move(pair.generation); + } + const auto terminal = std::get(generation.next()); + CHECK(terminal.result.error->code == GenerateErrorCode::Incomplete); + } + + void source_move_assignment_completes_replaced_handle() { + GenerationPair first = make_generation(1); + GenerationPair second = make_generation(1); + first.source = std::move(second.source); + + const auto replaced = + std::get(first.generation.next()); + CHECK(replaced.result.error->code == GenerateErrorCode::Incomplete); + + GenerateResult result; + result.succeed(); + CHECK(first.source.complete(std::move(result))); + CHECK(std::get(second.generation.next()).result.ok()); + } + + void immediate_rejection_is_terminal() { + GenerateResult result; + result.fail(GenerateErrorCode::Overloaded); + Generation generation = make_completed_generation(std::move(result)); + const auto terminal = std::get(generation.next()); + CHECK(terminal.result.error->code == GenerateErrorCode::Overloaded); + } + + void request_queue_is_bounded_and_fifo() { + GenerationQueue queue(2, 2); + GenerateRequest first; + first.prompt = {1}; + Generation first_generation = queue.submit(std::move(first)); + + GenerateRequest second; + second.prompt = {2}; + Generation second_generation = queue.submit(std::move(second)); + + GenerateRequest third; + third.prompt = {3}; + Generation rejected = queue.submit(std::move(third)); + const auto rejection = + std::get(rejected.next()); + CHECK(rejection.result.error->code == GenerateErrorCode::Overloaded); + + auto first_work = queue.next(); + CHECK(first_work.has_value()); + CHECK(first_work->request.prompt == std::vector({1})); + auto second_work = queue.next(); + CHECK(second_work.has_value()); + CHECK(second_work->request.prompt == std::vector({2})); + + GenerateResult result; + result.succeed(); + CHECK(first_work->source.complete(result)); + CHECK(second_work->source.complete(std::move(result))); + CHECK(std::get(first_generation.next()).result.ok()); + CHECK(std::get(second_generation.next()).result.ok()); + } + + void request_queue_skips_cancelled_work() { + GenerationQueue queue(2, 1); + Generation cancelled = queue.submit(GenerateRequest{}); + GenerateRequest request; + request.prompt = {8}; + Generation accepted = queue.submit(std::move(request)); + CHECK(cancelled.cancel(GenerationCancelReason::ClientDisconnected)); + + auto work = queue.next(); + CHECK(work.has_value()); + CHECK(work->request.prompt == std::vector({8})); + const auto terminal = + std::get(cancelled.next()); + CHECK(terminal.result.error->code == GenerateErrorCode::Cancelled); + + GenerateResult result; + result.succeed(); + CHECK(work->source.complete(std::move(result))); + CHECK(std::get(accepted.next()).result.ok()); + } + + void cancelled_queued_work_releases_capacity() { + GenerationQueue queue(1, 1); + Generation cancelled = queue.submit(GenerateRequest{}); + CHECK(cancelled.cancel(GenerationCancelReason::ClientDisconnected)); + + Generation replacement = queue.submit(GenerateRequest{}); + queue.shutdown(); + const auto terminal = + std::get(replacement.next()); + CHECK(terminal.result.error->code == + GenerateErrorCode::ShuttingDown); + } + + void shutdown_completes_queued_and_active_work() { + GenerationQueue queue(2, 1); + Generation active = queue.submit(GenerateRequest{}); + auto work = queue.next(); + CHECK(work.has_value()); + Generation queued = queue.submit(GenerateRequest{}); + + queue.shutdown(); + queue.shutdown(); + + const auto active_terminal = + std::get(active.next()); + const auto queued_terminal = + std::get(queued.next()); + CHECK(active_terminal.result.error->code == + GenerateErrorCode::ShuttingDown); + CHECK(queued_terminal.result.error->code == + GenerateErrorCode::ShuttingDown); + CHECK(work->source.cancellation_reason() == + GenerationCancelReason::Shutdown); + CHECK(!work->source.publish(TokenBatch{{1}})); + CHECK(!queue.next().has_value()); + + Generation rejected = queue.submit(GenerateRequest{}); + const auto rejection = + std::get(rejected.next()); + CHECK(rejection.result.error->code == + GenerateErrorCode::ShuttingDown); + } + + void handle_outlives_request_queue() { + Generation generation; + { + GenerationQueue queue(1, 1); + generation = queue.submit(GenerateRequest{}); + } + + const auto terminal = + std::get(generation.next()); + CHECK(terminal.result.error->code == + GenerateErrorCode::ShuttingDown); + } +}; + +} // namespace + +TEST_CASE(GenerationFixture, generation_channel_suite) { + request_payloads_are_owned(); + tokens_precede_terminal_result(); + progress_is_coalesced(); + full_channel_cancels_only_its_generation(); + token_capacity_counts_tokens_not_batches(); + cancellation_is_idempotent(); + source_destruction_preserves_terminal_result(); + dropping_handle_requests_cancellation(); + source_destruction_wakes_handle(); + source_move_assignment_completes_replaced_handle(); + immediate_rejection_is_terminal(); + request_queue_is_bounded_and_fifo(); + request_queue_skips_cancelled_work(); + cancelled_queued_work_releases_capacity(); + shutdown_completes_queued_and_active_work(); + handle_outlives_request_queue(); +} diff --git a/server/test/test_luce_engine.cpp b/server/test/test_luce_engine.cpp new file mode 100644 index 000000000..68b9d9954 --- /dev/null +++ b/server/test/test_luce_engine.cpp @@ -0,0 +1,242 @@ +#include "CppUnitTestFramework.hpp" +#include "common/model_backend.h" +#include "engine/luce_engine.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace CppUnitTestFramework; +using namespace dflash::common; +using namespace dflash::engine; + +namespace { + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_move_constructible_v); + +class FakeSeqEngine final : public SeqEngine { +public: + int slot_count() const override { return 1; } + int max_context() const override { return 1024; } + AdmitResult admit(uint64_t, const std::vector &, + const SamplerCfg &) override { + return {}; + } + StepPlanLimits step_plan_limits(int) const override { return {}; } + StepResult step(const StepPlan &) override { return {}; } + void retire(int) override {} + bool token_is_eos(int32_t) const override { return false; } +}; + +class FakeBackend final : public ModelBackend { +public: + FakeBackend(bool concurrent, std::atomic * destroyed = nullptr, + std::atomic * shutdown_calls = nullptr) + : concurrent_(concurrent), destroyed_(destroyed), + shutdown_calls_(shutdown_calls) {} + + ~FakeBackend() override { + shutdown(); + if (destroyed_) destroyed_->store(true); + } + + void print_ready_banner() const override {} + bool park(ParkTarget) override { return true; } + bool unpark(ParkTarget) override { return true; } + bool is_target_parked() const override { return false; } + GenerateResult generate_impl(const GenerateRequest &, + const DaemonIO &) override { + return {}; + } + SeqEngine * seq_engine() override { + return concurrent_ ? &seq_engine_ : nullptr; + } + bool snapshot_save(int) override { return true; } + void snapshot_free(int) override {} + bool snapshot_used(int) const override { return false; } + int snapshot_cur_pos(int) const override { return 0; } + GenerateResult restore_and_generate_impl( + int, const GenerateRequest &, const DaemonIO &) override { + return {}; + } + bool handle_compress(const std::string &, const DaemonIO &) override { + return false; + } + void free_drafter() override {} + void shutdown() override { + if (shutdown_calls_) shutdown_calls_->fetch_add(1); + } + +private: + bool concurrent_; + std::atomic * destroyed_; + std::atomic * shutdown_calls_; + FakeSeqEngine seq_engine_; +}; + +struct ServingProbe { + std::mutex mutex; + std::condition_variable ready; + bool started = false; + bool stopped = false; + bool ran_concurrent = false; + + LuceEngine::ServingLoops loops() { + LuceEngine::ServingLoops result; + result.serial = [this] { run(false); }; + result.concurrent = [this](SeqEngine &) { run(true); }; + result.request_stop = [this] { + std::lock_guard lock(mutex); + stopped = true; + ready.notify_all(); + }; + return result; + } + + bool wait_until_started() { + std::unique_lock lock(mutex); + return ready.wait_for(lock, std::chrono::seconds(1), + [this] { return started; }); + } + +private: + void run(bool concurrent) { + std::unique_lock lock(mutex); + ran_concurrent = concurrent; + started = true; + ready.notify_all(); + ready.wait(lock, [this] { return stopped; }); + } +}; + +struct LuceEngineFixture : CommonFixture { + using CommonFixture::CommonFixture; + + void selects_serial_loop_without_sequence_engine() { + LuceEngine engine(std::make_unique(false)); + ServingProbe probe; + REQUIRE(engine.start_serving(probe.loops(), true)); + REQUIRE(probe.wait_until_started()); + CHECK(!probe.ran_concurrent); + engine.stop_serving(); + } + + void selects_concurrent_loop_when_allowed() { + LuceEngine engine(std::make_unique(true)); + ServingProbe probe; + REQUIRE(engine.start_serving(probe.loops(), true)); + REQUIRE(probe.wait_until_started()); + CHECK(probe.ran_concurrent); + engine.stop_serving(); + } + + void policy_can_force_serial_loop() { + LuceEngine engine(std::make_unique(true)); + ServingProbe probe; + REQUIRE(engine.start_serving(probe.loops(), false)); + REQUIRE(probe.wait_until_started()); + CHECK(!probe.ran_concurrent); + engine.stop_serving(); + } + + void owns_backend_lifetime_without_duplicate_shutdown() { + std::atomic destroyed{false}; + std::atomic shutdown_calls{0}; + { + LuceEngine engine( + std::make_unique(false, &destroyed, + &shutdown_calls)); + CHECK(!destroyed.load()); + CHECK(shutdown_calls.load() == 0); + } + CHECK(shutdown_calls.load() == 1); + CHECK(destroyed.load()); + } + + void stop_is_idempotent_and_allows_restart() { + ServingProbe first; + ServingProbe second; + LuceEngine engine(std::make_unique(true)); + engine.stop_serving(); + REQUIRE(engine.start_serving(first.loops(), false)); + REQUIRE(first.wait_until_started()); + CHECK(!engine.start_serving(second.loops(), true)); + engine.stop_serving(); + engine.stop_serving(); + REQUIRE(engine.start_serving(second.loops(), true)); + REQUIRE(second.wait_until_started()); + CHECK(second.ran_concurrent); + engine.stop_serving(); + } + + void finished_loop_must_be_joined_before_restart() { + std::promise finished; + auto completion = finished.get_future(); + int stops = 0; + LuceEngine engine(std::make_unique(false)); + LuceEngine::ServingLoops loops; + loops.serial = [&] { finished.set_value_at_thread_exit(); }; + loops.concurrent = [](SeqEngine &) {}; + loops.request_stop = [&] { ++stops; }; + REQUIRE(engine.start_serving(loops, false)); + REQUIRE(completion.wait_for(std::chrono::seconds(1)) == + std::future_status::ready); + CHECK(!engine.start_serving(loops, false)); + engine.stop_serving(); + engine.stop_serving(); + CHECK(stops == 1); + loops.serial = [] {}; + REQUIRE(engine.start_serving(loops, false)); + engine.stop_serving(); + CHECK(stops == 2); + } + + void destruction_joins_before_destroying_backend() { + ServingProbe probe; + std::atomic destroyed{false}; + bool backend_alive_at_loop_exit = false; + { + LuceEngine engine(std::make_unique(false, &destroyed)); + auto loops = probe.loops(); + loops.serial = [&, run = std::move(loops.serial)] { + run(); + backend_alive_at_loop_exit = !destroyed.load(); + }; + REQUIRE(engine.start_serving(std::move(loops), false)); + REQUIRE(probe.wait_until_started()); + CHECK(!destroyed.load()); + } + CHECK(probe.stopped); + CHECK(backend_alive_at_loop_exit); + CHECK(destroyed.load()); + } + + void rejects_missing_backend() { + bool rejected = false; + try { + LuceEngine engine(nullptr); + } catch (const std::invalid_argument &) { + rejected = true; + } + CHECK(rejected); + } +}; + +} // namespace + +TEST_CASE(LuceEngineFixture, luce_engine_lifecycle_suite) { + selects_serial_loop_without_sequence_engine(); + selects_concurrent_loop_when_allowed(); + policy_can_force_serial_loop(); + owns_backend_lifetime_without_duplicate_shutdown(); + stop_is_idempotent_and_allows_restart(); + finished_loop_must_be_joined_before_restart(); + destruction_joins_before_destroying_backend(); + rejects_missing_backend(); +}