From 65f2babcec784cd99e598155565a85f6980b63ba Mon Sep 17 00:00:00 2001 From: exzile Date: Fri, 26 Jun 2026 17:24:41 -0400 Subject: [PATCH 01/10] Add idle model unload for mediapipe LLM graphs Adds an opt-in idle timeout that unloads an LLM graph's heavy resources (freeing GPU/CPU memory) after a period with no inference, and lazily reloads on the next request, so the GPU can be shared with other workloads. Follows llama.cpp's --sleep-idle-seconds model. - Config: idle_unload_timeout_seconds on the mediapipe config entry (0 = disabled, default). Added to the JSON schema. Phase-1 scope: restricted to LLM continuous-batching graphs (HttpLLMCalculator); rejected for graphs with Python nodes or non-LLM calculators at config validation. - State machine: new UNLOADED state + UnloadEvent. AVAILABLE->UNLOADED on idle; UNLOADED wakes via the existing reload path. isAvailable() is false for UNLOADED, convertToModelStatus() maps it to AVAILABLE so health/ readiness still see the servable (it auto-reloads). - Concurrency: a per-definition recursive lifecycleMtx serializes reload/retire/unload/wakeUp so the watcher and config threads never race on graph state or side packets. unload() is non-blocking, tears down only after confirming the AVAILABLE->UNLOADED transition, and skips while requests or inferences are in flight. The idle timeout is cached in an atomic for lock-free watcher reads. - In-flight guard: an RAII ActiveInferenceGuard (held for the executor's lifetime) keeps a shared_ptr inference counter so a generation that outlives the idle timeout is never unloaded mid-stream; completing an inference refreshes the activity timestamp. lastActivityTimeNs and the counter are shared_ptr so they outlive the definition if an executor is still running during retire. - Wake-up failure is retryable: if the lazy reload fails, the graph reverts to UNLOADED (not a wedged failed state) so the next request re-attempts the wake and self-heals once the underlying issue is resolved; the current request gets a clean error. - Metrics: ovms_graph_loaded gauge (1 loaded / 0 unloaded) per graph. - Composes with --cache_dir so wake-up is a cache import, not a recompile. Tested: state-machine + schema unit tests; in-flight-guard and wake-failure unit tests; functional unload/reload/idle-reset/disabled-default/concurrency with a real model; and end-to-end on an Intel Arc GPU (idle unload frees resources, long generations are not unloaded mid-stream, wake-up reloads and serves, soak shows no leak, failed wake self-heals). No regressions vs main. Implements #4141 Co-Authored-By: Claude Opus 4.8 --- docs/llm/reference.md | 10 + src/dags/pipelinedefinitionstatus.cpp | 56 +- src/dags/pipelinedefinitionstatus.hpp | 37 +- .../mediapipegraphconfig.cpp | 9 + .../mediapipegraphconfig.hpp | 16 + .../mediapipegraphdefinition.cpp | 205 +++++- .../mediapipegraphdefinition.hpp | 59 ++ .../mediapipegraphexecutor.cpp | 18 +- .../mediapipegraphexecutor.hpp | 58 +- src/metrics/metric_config.cpp | 1 + src/metrics/metric_config.hpp | 2 + src/model_metric_reporter.cpp | 12 + src/model_metric_reporter.hpp | 3 + src/modelmanager.cpp | 82 +++ src/modelmanager.hpp | 6 + src/schema.cpp | 4 + src/test/llm/llmnode_test.cpp | 638 ++++++++++++++++++ src/test/mediapipeflow_test.cpp | 97 +++ src/test/pipelinedefinitionstatus_test.cpp | 130 ++++ src/test/schema_test.cpp | 60 ++ src/test/test_utils.hpp | 17 + 21 files changed, 1509 insertions(+), 11 deletions(-) diff --git a/docs/llm/reference.md b/docs/llm/reference.md index 698d05031b..00a9602e3e 100644 --- a/docs/llm/reference.md +++ b/docs/llm/reference.md @@ -109,6 +109,16 @@ The calculator supports the following `node_options` for tuning the pipeline con - `optional string tool_parser` - name of the parser to use for tool calls extraction from model output before creating a response; - `optional bool enable_tool_guided_generation` - enable enforcing tool schema during generation. Requires setting response parser. [default = false]; - `optional SparseAttentionConfig sparse_attention_config` - Sparse attention configuration. Disabled if not specified. +- `optional int64 idle_unload_timeout_seconds` - unload the graph's model resources after this many seconds with no inference requests, freeing GPU/CPU memory; the model is reloaded automatically on the next request. `0` disables the feature [default = 0]. See [Idle model unload](#idle-model-unload). + +### Idle model unload +When `idle_unload_timeout_seconds` is set to a positive value, the model server unloads the LLM graph's heavy resources (the continuous batching pipeline, freeing GPU VRAM / host memory) after the configured period without any inference requests. The first request after an unload transparently reloads the model and is served once it is ready, so the GPU can be used by other workloads while a model is idle. + +Notes: +- Only inference requests reset the idle timer; status/metrics/health endpoints do not keep a model loaded. +- The first request after an idle unload pays the reload latency. Combine with [model caching](../model_cache.md) (`--cache_dir`) so the reload is a fast cache import rather than a full recompile. +- The graph reports as `AVAILABLE` while idle-unloaded (it auto-reloads on demand). The `ovms_graph_loaded` metric reports `1` when loaded and `0` when idle-unloaded. +- Supported for LLM continuous-batching graphs. Graphs containing Python nodes are not supported with this setting. ### Caching settings The value of `cache_size` might have performance and stability implications. It is used for storing LLM model KV cache data. Adjust it based on your environment capabilities, model size and expected level of concurrency. diff --git a/src/dags/pipelinedefinitionstatus.cpp b/src/dags/pipelinedefinitionstatus.cpp index 5fb27479b0..17afaa3a0e 100644 --- a/src/dags/pipelinedefinitionstatus.cpp +++ b/src/dags/pipelinedefinitionstatus.cpp @@ -35,7 +35,8 @@ const std::string& pipelineDefinitionStateCodeToString(PipelineDefinitionStateCo {PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION, "LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION"}, {PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION, "AVAILABLE_REQUIRED_REVALIDATION"}, {PipelineDefinitionStateCode::AVAILABLE, "AVAILABLE"}, - {PipelineDefinitionStateCode::RETIRED, "RETIRED"}}; + {PipelineDefinitionStateCode::RETIRED, "RETIRED"}, + {PipelineDefinitionStateCode::UNLOADED, "UNLOADED"}}; return names.at(code); } @@ -62,6 +63,9 @@ StateKeeper BeginState::handle(const RetireEvent& e) const { throw std::logic_error(INVALID_TRANSITION_MESSAGE); return {}; } +StateKeeper BeginState::handle(const UnloadEvent& e) const { + return {}; // unload is a no-op when not yet loaded +} PipelineDefinitionStateCode ReloadState::getStateCode() const { return code; @@ -84,6 +88,9 @@ StateKeeper ReloadState::handle(const RetireEvent& e) const { throw std::logic_error(INVALID_TRANSITION_MESSAGE); return {}; } +StateKeeper ReloadState::handle(const UnloadEvent& e) const { + return {}; // unload is a no-op while reloading +} PipelineDefinitionStateCode AvailableState::getStateCode() const { return code; @@ -105,6 +112,9 @@ StateChanger AvailableState::handle(const UsedMod StateChanger AvailableState::handle(const RetireEvent& e) const { return {}; } +StateChanger AvailableState::handle(const UnloadEvent& e) const { + return {}; +} PipelineDefinitionStateCode AvailableRequiredRevalidation::getStateCode() const { return code; @@ -124,6 +134,9 @@ StateKeeper AvailableRequiredRevalidation::handle(const UsedModelChangedEvent& e StateChanger AvailableRequiredRevalidation::handle(const RetireEvent& e) const { return {}; } +StateKeeper AvailableRequiredRevalidation::handle(const UnloadEvent& e) const { + return {}; // unload is a no-op in AVAILABLE_REQUIRED_REVALIDATION +} PipelineDefinitionStateCode LoadingPreconditionFailedState::getStateCode() const { return code; @@ -145,6 +158,10 @@ StateChanger LoadingPreconditio StateChanger LoadingPreconditionFailedState::handle(const RetireEvent& e) const { return {}; } +StateChanger LoadingPreconditionFailedState::handle(const UnloadEvent& e) const { + // Revert a failed wake-up reload back to UNLOADED so the next request retries. + return {}; +} PipelineDefinitionStateCode LoadingFailedLastValidationRequiredRevalidation::getStateCode() const { return code; @@ -164,6 +181,9 @@ StateKeeper LoadingFailedLastValidationRequiredRevalidation::handle(const UsedMo StateChanger LoadingFailedLastValidationRequiredRevalidation::handle(const RetireEvent& e) const { return {}; } +StateKeeper LoadingFailedLastValidationRequiredRevalidation::handle(const UnloadEvent& e) const { + return {}; // unload is a no-op when loading already failed +} PipelineDefinitionStateCode RetiredState::getStateCode() const { return code; @@ -187,6 +207,31 @@ StateKeeper RetiredState::handle(const RetireEvent& e) const { throw std::logic_error(INVALID_TRANSITION_MESSAGE); return {}; } +StateKeeper RetiredState::handle(const UnloadEvent& e) const { + return {}; // unload is a no-op when already retired +} + +PipelineDefinitionStateCode UnloadedState::getStateCode() const { + return code; +} +StateChanger UnloadedState::handle(const ReloadEvent& e) const { + return {}; // wake-up: transition through reload path +} +StateChanger UnloadedState::handle(const RetireEvent& e) const { + return {}; // config removal while unloaded +} +StateChanger UnloadedState::handle(const ValidationPassedEvent& e) const { + return {}; // defensive: if validation passes directly, go available +} +StateKeeper UnloadedState::handle(const ValidationFailedEvent& e) const { + return {}; +} +StateKeeper UnloadedState::handle(const UsedModelChangedEvent& e) const { + return {}; +} +StateKeeper UnloadedState::handle(const UnloadEvent& e) const { + return {}; // already unloaded, idempotent +} PipelineDefinitionStatus::PipelineDefinitionStatus(const std::string& type, const std::string& name) : MachineState(type, name) {} @@ -233,6 +278,15 @@ std::tuple PipelineDefinitionSta ModelVersionState::END, ModelVersionStatusErrorCode::OK}; + case PipelineDefinitionStateCode::UNLOADED: + // Report AVAILABLE: the graph auto-reloads on the next inference request, + // so health checks and routing should treat it as available. Reporting END + // or UNLOADING would cause clients and load-balancers to permanently + // exclude this servable from their pools. + return { + ModelVersionState::AVAILABLE, + ModelVersionStatusErrorCode::OK}; + default: return {}; } diff --git a/src/dags/pipelinedefinitionstatus.hpp b/src/dags/pipelinedefinitionstatus.hpp index 59039f08a0..05366e8c99 100644 --- a/src/dags/pipelinedefinitionstatus.hpp +++ b/src/dags/pipelinedefinitionstatus.hpp @@ -34,7 +34,8 @@ enum class PipelineDefinitionStateCode { LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION, AVAILABLE_REQUIRED_REVALIDATION, AVAILABLE, - RETIRED + RETIRED, + UNLOADED }; const std::string& pipelineDefinitionStateCodeToString(PipelineDefinitionStateCode code); @@ -112,6 +113,11 @@ struct LoadingFailedLastValidationRequiredRevalidation; * State in which pipeline is retired - removed from config */ struct RetiredState; +/** + * State in which pipeline is idle-unloaded (resources freed) but not retired. + * Auto-reloads on the next inference request. + */ +struct UnloadedState; #define EVENT_STRUCT_WITH_NAME(x) \ struct x { \ @@ -131,6 +137,7 @@ EVENT_STRUCT_WITH_NAME(ValidationFailedEvent); EVENT_STRUCT_WITH_NAME(ValidationPassedEvent); EVENT_STRUCT_WITH_NAME(UsedModelChangedEvent); EVENT_STRUCT_WITH_NAME(RetireEvent); +EVENT_STRUCT_WITH_NAME(UnloadEvent); template struct StateChanger { @@ -155,6 +162,7 @@ struct BeginState { StateChanger handle(const ValidationFailedEvent& e) const; StateKeeper handle(const UsedModelChangedEvent& e) const; StateKeeper handle(const RetireEvent& e) const; + StateKeeper handle(const UnloadEvent& e) const; }; struct ReloadState { @@ -165,6 +173,7 @@ struct ReloadState { StateChanger handle(const ValidationFailedEvent& e) const; StateKeeper handle(const UsedModelChangedEvent& e) const; StateKeeper handle(const RetireEvent& e) const; + StateKeeper handle(const UnloadEvent& e) const; }; struct AvailableState { @@ -175,6 +184,7 @@ struct AvailableState { StateKeeper handle(const ValidationFailedEvent& e) const; StateChanger handle(const UsedModelChangedEvent& e) const; StateChanger handle(const RetireEvent& e) const; + StateChanger handle(const UnloadEvent& e) const; }; struct AvailableRequiredRevalidation { @@ -185,6 +195,7 @@ struct AvailableRequiredRevalidation { StateChanger handle(const ValidationFailedEvent& e) const; StateKeeper handle(const UsedModelChangedEvent& e) const; StateChanger handle(const RetireEvent& e) const; + StateKeeper handle(const UnloadEvent& e) const; }; struct LoadingPreconditionFailedState { @@ -195,6 +206,11 @@ struct LoadingPreconditionFailedState { StateKeeper handle(const ValidationFailedEvent& e) const; StateChanger handle(const UsedModelChangedEvent& e) const; StateChanger handle(const RetireEvent& e) const; + // A failed wake-up reload of an idle graph reverts to UNLOADED so the next + // inference request can retry the wake (self-healing once the underlying issue + // is resolved). Only wakeUpIfUnloaded() sends UnloadEvent from this state; + // the watcher's unload() only does so from AVAILABLE. + StateChanger handle(const UnloadEvent& e) const; }; struct LoadingFailedLastValidationRequiredRevalidation { @@ -205,6 +221,7 @@ struct LoadingFailedLastValidationRequiredRevalidation { StateChanger handle(const ValidationFailedEvent& e) const; StateKeeper handle(const UsedModelChangedEvent& e) const; StateChanger handle(const RetireEvent& e) const; + StateKeeper handle(const UnloadEvent& e) const; }; struct RetiredState { @@ -215,9 +232,25 @@ struct RetiredState { StateChanger handle(const ValidationFailedEvent& e) const; StateKeeper handle(const UsedModelChangedEvent& e) const; StateKeeper handle(const RetireEvent& e) const; + StateKeeper handle(const UnloadEvent& e) const; +}; + +struct UnloadedState { + static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::UNLOADED; + PipelineDefinitionStateCode getStateCode() const; + // Wake-up: reuse the reload path + StateChanger handle(const ReloadEvent& e) const; + // Config removal while unloaded + StateChanger handle(const RetireEvent& e) const; + // Defensive: if validation somehow passes after an unload, go back to AVAILABLE + StateChanger handle(const ValidationPassedEvent& e) const; + // All other events are no-ops in UNLOADED + StateKeeper handle(const ValidationFailedEvent& e) const; + StateKeeper handle(const UsedModelChangedEvent& e) const; + StateKeeper handle(const UnloadEvent& e) const; }; -class PipelineDefinitionStatus : public MachineState { +class PipelineDefinitionStatus : public MachineState { public: PipelineDefinitionStatus(const std::string& type, const std::string& name); bool isAvailable() const; diff --git a/src/mediapipe_internal/mediapipegraphconfig.cpp b/src/mediapipe_internal/mediapipegraphconfig.cpp index 200de9c289..14930f115c 100644 --- a/src/mediapipe_internal/mediapipegraphconfig.cpp +++ b/src/mediapipe_internal/mediapipegraphconfig.cpp @@ -119,6 +119,15 @@ Status MediapipeGraphConfig::parseNode(const rapidjson::Value& v) { this->setSubconfigPath(DEFAULT_SUBCONFIG_FILENAME); this->setModelMeshSubconfigPath(DEFAULT_MODELMESH_SUBCONFIG_FILENAME); } + if (v.HasMember("idle_unload_timeout_seconds")) { + int timeoutSeconds = v["idle_unload_timeout_seconds"].GetInt(); + if (timeoutSeconds < 0) { + SPDLOG_ERROR("idle_unload_timeout_seconds must be >= 0 for mediapipe graph: {}", this->getGraphName()); + return StatusCode::JSON_INVALID; + } + this->setIdleUnloadTimeoutSeconds(timeoutSeconds); + SPDLOG_DEBUG("Mediapipe graph {} idle_unload_timeout_seconds set to {}", this->getGraphName(), timeoutSeconds); + } } catch (std::logic_error& e) { SPDLOG_DEBUG("Relative path error: {}", e.what()); return StatusCode::INTERNAL_ERROR; diff --git a/src/mediapipe_internal/mediapipegraphconfig.hpp b/src/mediapipe_internal/mediapipegraphconfig.hpp index a8237b1e0f..46c9240fe8 100644 --- a/src/mediapipe_internal/mediapipegraphconfig.hpp +++ b/src/mediapipe_internal/mediapipegraphconfig.hpp @@ -75,6 +75,14 @@ class MediapipeGraphConfig { */ GraphQueueSizeValue graphQueueSize; + /** + * @brief Idle unload timeout in seconds. + * 0 (default) = feature disabled. + * When > 0, the graph's heavy resources are freed after this many seconds + * of zero in-flight requests, and lazily reloaded on the next inference. + */ + int idleUnloadTimeoutSeconds = 0; + public: MediapipeGraphConfig(const std::string& graphName = "", const std::string& basePath = "", @@ -195,6 +203,14 @@ class MediapipeGraphConfig { return std::get(*this->graphQueueSize); } + int getIdleUnloadTimeoutSeconds() const { + return this->idleUnloadTimeoutSeconds; + } + + void setIdleUnloadTimeoutSeconds(int seconds) { + this->idleUnloadTimeoutSeconds = seconds; + } + bool isReloadRequired(const MediapipeGraphConfig& rhs) const; /** diff --git a/src/mediapipe_internal/mediapipegraphdefinition.cpp b/src/mediapipe_internal/mediapipegraphdefinition.cpp index 945dbf1d70..0302dc9308 100644 --- a/src/mediapipe_internal/mediapipegraphdefinition.cpp +++ b/src/mediapipe_internal/mediapipegraphdefinition.cpp @@ -210,6 +210,42 @@ Status MediapipeGraphDefinition::validate(const ServableNameChecker& checker) { if (!validationResult.ok()) { return validationResult; } + // Phase-1 restriction: idle unload is not supported for graphs with Python nodes. + // Python nodes may hold per-request iterator state (e.g. PythonExecutorCalculator) + // that cannot be safely reconstructed after a resource-free/reload cycle. + if (mgconfig.getIdleUnloadTimeoutSeconds() > 0) { + for (int i = 0; i < this->config.node_size(); ++i) { + const std::string& calculator = this->config.node(i).calculator(); + if (calculator == "PythonExecutorCalculator" || calculator == "PyTorchCalculator") { + SPDLOG_LOGGER_ERROR(modelmanager_logger, + "Mediapipe graph {}: idle_unload_timeout_seconds is not supported for graphs " + "containing Python calculator nodes ({}). " + "Remove idle_unload_timeout_seconds or remove the Python node.", + getName(), calculator); + return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; + } + } + // Phase-1 scope restriction: idle_unload_timeout_seconds is validated only for + // LLM/VLM continuous-batching graphs (graphs containing HttpLLMCalculator). + // Other node types (embeddings, rerank, STT, TTS, image-gen, plain passthrough) + // have not been validated for the idle-unload/lazy-reload cycle. + bool hasLlmCalculator = false; + for (int i = 0; i < this->config.node_size(); ++i) { + if (this->config.node(i).calculator() == "HttpLLMCalculator") { + hasLlmCalculator = true; + break; + } + } + if (!hasLlmCalculator) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, + "Mediapipe graph {}: idle_unload_timeout_seconds is only supported for " + "LLM/VLM continuous-batching graphs (HttpLLMCalculator) in this release. " + "Remove idle_unload_timeout_seconds from non-LLM graph configurations.", + getName()); + return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; + } + } + validationResult = resolveGraphQueueSize(); if (!validationResult.ok()) { return validationResult; @@ -257,6 +293,8 @@ Status MediapipeGraphDefinition::validate(const ServableNameChecker& checker) { lock.unlock(); notifier.passed = true; + // Graph resources are now loaded (covers both initial load and wake-up reload). + SET_IF_ENABLED(this->reporter->graphLoaded, 1); SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Finished validation of mediapipe: {}", getName()); SPDLOG_LOGGER_INFO(modelmanager_logger, "Mediapipe: {} inputs: {}", getName(), getTensorMapString(inputsInfo)); SPDLOG_LOGGER_INFO(modelmanager_logger, "Mediapipe: {} outputs: {}", getName(), getTensorMapString(outputsInfo)); @@ -294,7 +332,18 @@ MediapipeGraphDefinition::MediapipeGraphDefinition(const std::string name, pythonBackend(pythonBackend), reporter(std::make_unique(metricConfig, registry, name)) { mgconfig = config; + idleUnloadTimeoutSecondsCache.store(mgconfig.getIdleUnloadTimeoutSeconds(), std::memory_order_relaxed); passKfsRequestFlag = false; + // Allocate lastActivityTimeNs initialized to now so the idle timer starts from + // when the graph was loaded, not from the steady_clock epoch (which would + // cause immediate unload of a freshly loaded graph before any request arrives). + // Held as a shared_ptr so executors can safely refresh it even after the + // definition is retired/destroyed. + lastActivityTimeNs = std::make_shared>( + std::chrono::steady_clock::now().time_since_epoch().count()); + // Allocate the active-inference counter once; shared with every executor + // created by this definition so executors can decrement it on completion. + activeInferenceCount = std::make_shared>(0); } Status MediapipeGraphDefinition::createInputsInfo() { @@ -351,6 +400,13 @@ Status MediapipeGraphDefinition::createOutputsInfo() { } Status MediapipeGraphDefinition::create(std::unique_ptr& pipeline) { + // Update idle-tracking timestamp on every inference acquisition path. + // Status endpoints / health checks do not reach this method, so idle + // tracking is automatically inference-only. + lastActivityTimeNs->store( + std::chrono::steady_clock::now().time_since_epoch().count(), + std::memory_order_relaxed); + std::unique_ptr unloadGuard; Status status = waitForLoaded(unloadGuard); if (!status.ok()) { @@ -363,12 +419,14 @@ Status MediapipeGraphDefinition::create(std::unique_ptr& pipeline = std::make_unique(getName(), std::to_string(getVersion()), this->config, this->inputTypes, this->outputTypes, this->inputNames, this->outputNames, *this->sidePacketMaps, - this->pythonBackend, this->reporter.get(), std::move(graphIdGuard)); + this->pythonBackend, this->reporter.get(), std::move(graphIdGuard), + this->activeInferenceCount, this->lastActivityTimeNs); } else { pipeline = std::make_unique(getName(), std::to_string(getVersion()), this->config, this->inputTypes, this->outputTypes, this->inputNames, this->outputNames, *this->sidePacketMaps, - this->pythonBackend, this->reporter.get()); + this->pythonBackend, this->reporter.get(), + this->activeInferenceCount, this->lastActivityTimeNs); } SPDLOG_DEBUG("Created Mediapipe graph executor: {}", getName()); return status; @@ -438,18 +496,25 @@ Status MediapipeGraphDefinition::setStreamTypes() { } Status MediapipeGraphDefinition::reload(const ServableNameChecker& checker, const MediapipeGraphConfig& config) { + // Serialize against unload()/wakeUp() on the watcher/request threads. + // Recursive: wakeUpIfUnloaded() already holds this and calls reload(). + std::lock_guard lock(lifecycleMtx); // block creating new unloadGuards this->status.handle(ReloadEvent()); while (requestsHandlesCounter > 0) { std::this_thread::sleep_for(std::chrono::microseconds(1)); } this->mgconfig = config; + // Refresh the lock-free cache while we still hold lifecycleMtx. + idleUnloadTimeoutSecondsCache.store(this->mgconfig.getIdleUnloadTimeoutSeconds(), std::memory_order_relaxed); this->queue.reset(); this->sidePacketMaps = std::make_shared(); return validate(checker); } void MediapipeGraphDefinition::retire() { + // Serialize against unload()/wakeUp()/reload() on other threads. + std::lock_guard lock(lifecycleMtx); // Block creating new unloadGuards this->status.handle(RetireEvent()); while (requestsHandlesCounter > 0) { @@ -459,6 +524,142 @@ void MediapipeGraphDefinition::retire() { this->sidePacketMaps.reset(); } +bool MediapipeGraphDefinition::isIdleUnloadEnabled() const { + // Lock-free read of the cached timeout (mgconfig is only safe under lifecycleMtx). + return idleUnloadTimeoutSecondsCache.load(std::memory_order_relaxed) > 0; +} + +bool MediapipeGraphDefinition::shouldUnloadDueToIdle() const { + // Advisory pre-filter ONLY — reads no unsynchronized per-definition state. + // It must NOT read this->status (the state-machine variant) without the lock, + // since the config thread can mutate it concurrently. unload() performs the + // authoritative state==AVAILABLE check under lifecycleMtx. + // requestsHandlesCounter, lastActivityTimeNs and idleUnloadTimeoutSecondsCache + // are all atomics, so every read here is data-race-free. We never read mgconfig + // (only safe under lifecycleMtx) on this advisory path. + int64_t timeoutSeconds = idleUnloadTimeoutSecondsCache.load(std::memory_order_relaxed); + if (timeoutSeconds <= 0) { + return false; + } + if (requestsHandlesCounter.load(std::memory_order_relaxed) != 0) { + return false; + } + // Guard: if inferences are actively executing, never report idle. + // activeInferenceCount is bumped when a MediapipeGraphExecutor is created (in + // create()) by its RAII ActiveInferenceGuard, held for the executor's lifetime + // (which spans the inference), and decremented (with a lastActivityTimeNs refresh) + // when the executor is destroyed after the inference completes or throws. + if (activeInferenceCount && activeInferenceCount->load(std::memory_order_acquire) > 0) { + return false; + } + int64_t lastActivity = lastActivityTimeNs->load(std::memory_order_relaxed); + int64_t nowNs = std::chrono::steady_clock::now().time_since_epoch().count(); + int64_t timeoutNs = timeoutSeconds * 1'000'000'000LL; + return (nowNs - lastActivity) >= timeoutNs; +} + +Status MediapipeGraphDefinition::unload() { + // Serialize against wakeUpIfUnloaded()/reload()/retire() using the SAME lock so + // all lifecycle mutations are mutually exclusive. This prevents the watcher thread + // from tearing down resources while the config thread reloads/retires, or while a + // request thread is in the middle of a wake-up reload. + std::lock_guard lock(lifecycleMtx); + + // Re-check the preconditions under the lock. Only AVAILABLE graphs with no + // in-flight requests may be unloaded. If the state changed (e.g. a wake-up + // moved us to RELOADING/AVAILABLE) or a request arrived after the watcher's + // shouldUnloadDueToIdle() check, skip this cycle WITHOUT touching resources. + if (status.getStateCode() != PipelineDefinitionStateCode::AVAILABLE) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, + "Skipping idle-unload of mediapipe graph {}: state is no longer AVAILABLE", getName()); + return StatusCode::OK; + } + if (requestsHandlesCounter.load(std::memory_order_acquire) != 0) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, + "Skipping idle-unload of mediapipe graph {}: requests in flight", getName()); + return StatusCode::OK; + } + // Guard: if inferences are actively executing, skip this unload cycle. + // This catches the case where the executor's inference is running (past create()) + // but before ActiveInferenceGuard has decremented — i.e. a long generation that + // outlives idle_unload_timeout_seconds. Re-checked under the lock so the decision + // is consistent with the in-flight inference completing concurrently. + if (activeInferenceCount && activeInferenceCount->load(std::memory_order_acquire) > 0) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, + "Skipping idle-unload of mediapipe graph {}: active inferences in progress", getName()); + return StatusCode::OK; + } + + // Transition state: AVAILABLE -> UNLOADED (blocks new unloadGuards in waitForLoaded). + this->status.handle(UnloadEvent()); + + // Defensive: only tear down resources if the transition actually happened. + // (UnloadEvent is a no-op on any non-AVAILABLE state.) + if (status.getStateCode() != PipelineDefinitionStateCode::UNLOADED) { + SPDLOG_LOGGER_WARN(modelmanager_logger, + "Idle-unload of mediapipe graph {} aborted: state did not transition to UNLOADED (now {})", + getName(), pipelineDefinitionStateCodeToString(status.getStateCode())); + return StatusCode::OK; + } + + // Once UNLOADED, no new unloadGuards can be acquired and we verified + // requestsHandlesCounter == 0 above, so there is nothing to drain. + // Release queue (pooled graphs hold GPU/CPU resources). + this->queue.reset(); + // Release heavy side-packet resources (GenAI servables, embeddings, etc.) + // Keep the sidePacketMaps object itself — clear() drops the shared_ptrs inside, + // freeing GPU VRAM. validate()/initializeNodes() will repopulate it on wake-up. + this->sidePacketMaps->clear(); + + SET_IF_ENABLED(this->reporter->graphLoaded, 0); + SPDLOG_LOGGER_INFO(modelmanager_logger, + "Mediapipe graph {} idle-unloaded (freed GPU/CPU resources after {}s idle timeout)", + getName(), mgconfig.getIdleUnloadTimeoutSeconds()); + return StatusCode::OK; +} + +Status MediapipeGraphDefinition::wakeUpIfUnloaded(const ServableNameChecker& checker) { + // Recursive: this holds lifecycleMtx and then calls reload(), which re-acquires it. + std::lock_guard lock(lifecycleMtx); + // Double-check under lock: another thread may have already completed the reload. + if (status.getStateCode() != PipelineDefinitionStateCode::UNLOADED) { + return StatusCode::OK; + } + // Re-use the existing reload path: + // handle(ReloadEvent) -> fresh sidePacketMaps -> validate() -> initializeNodes() + // The stored mgconfig holds all required configuration. + SPDLOG_LOGGER_INFO(modelmanager_logger, + "Mediapipe graph {} is UNLOADED; triggering lazy wake-up reload", getName()); + auto start = std::chrono::steady_clock::now(); + Status reloadStatus = reload(checker, this->mgconfig); + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + if (reloadStatus.ok()) { + // Only reset the idle timer on a successful wake; on failure leave it so + // the existing failure-state handling applies and we don't mask the error. + lastActivityTimeNs->store( + std::chrono::steady_clock::now().time_since_epoch().count(), + std::memory_order_relaxed); + SPDLOG_LOGGER_INFO(modelmanager_logger, + "Mediapipe graph {} wake-up completed in {}ms", + getName(), elapsed.count()); + } else { + // Wake-up reload failed (e.g. model files temporarily unavailable). reload() + // ran validate() which left the state in LOADING_PRECONDITION_FAILED. Revert + // to UNLOADED so the NEXT inference request retries the wake — making a + // transient failure self-healing rather than permanently wedging a previously + // healthy idle graph. We are still holding lifecycleMtx here. + // (If validate() somehow ended elsewhere, UnloadEvent is a no-op on states + // other than AVAILABLE/LOADING_PRECONDITION_FAILED, so this is safe.) + this->status.handle(UnloadEvent()); + SPDLOG_LOGGER_ERROR(modelmanager_logger, + "Mediapipe graph {} wake-up failed after {}ms: {}. Reverted to UNLOADED; " + "next request will retry the wake.", + getName(), elapsed.count(), reloadStatus.string()); + } + return reloadStatus; +} + bool MediapipeGraphDefinition::isReloadRequired(const MediapipeGraphConfig& config) const { if (getStateCode() == PipelineDefinitionStateCode::RETIRED) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reloading previously retired mediapipe definition: {}", getName()); diff --git a/src/mediapipe_internal/mediapipegraphdefinition.hpp b/src/mediapipe_internal/mediapipegraphdefinition.hpp index b49ed7e456..5d4ea8d631 100644 --- a/src/mediapipe_internal/mediapipegraphdefinition.hpp +++ b/src/mediapipe_internal/mediapipegraphdefinition.hpp @@ -14,8 +14,11 @@ // limitations under the License. //***************************************************************************** #pragma once +#include +#include #include #include +#include #include #include #include @@ -78,6 +81,28 @@ class MediapipeGraphDefinition : public SingleVersionServableDefinition { Status initializeNodes(); bool isReloadRequired(const MediapipeGraphConfig& config) const; + // Idle unload feature + Status unload(); + // wakeUpIfUnloaded: thread-safe wrapper — holds lifecycleMtx, double-checks + // the UNLOADED state, and calls wakeUp() exactly once; other concurrent callers + // wait on the mutex then return immediately since the state is no longer UNLOADED. + Status wakeUpIfUnloaded(const ServableNameChecker& checker); + bool isIdleUnloadEnabled() const; + bool shouldUnloadDueToIdle() const; + + // Test-only: backdate the last-activity timestamp by the given number of seconds + // so idle-timeout behavior can be exercised deterministically without sleeping. + void backdateLastActivityForTest(int64_t seconds) { + int64_t nowNs = std::chrono::steady_clock::now().time_since_epoch().count(); + lastActivityTimeNs->store(nowNs - seconds * 1'000'000'000LL, std::memory_order_relaxed); + } + + // Returns the shared active-inference counter so create() can hand it to the executor. + // Not exposed in tests directly — use shouldUnloadDueToIdle() to observe the effect. + const std::shared_ptr>& getActiveInferenceCount() const { + return activeInferenceCount; + } + static const std::string SCHEDULER_CLASS_NAME; protected: @@ -148,5 +173,39 @@ class MediapipeGraphDefinition : public SingleVersionServableDefinition { std::unique_ptr reporter; std::shared_ptr queue; + + // Idle unload: timestamp (nanoseconds from steady_clock epoch) of the last + // inference activity. Updated in create() on every inference acquisition and + // when an in-flight inference finishes (via ActiveInferenceGuard destructor). + // Held as shared_ptr so executors can safely write to it even after the + // definition is retired/destroyed — the atomic outlives the definition. + std::shared_ptr> lastActivityTimeNs; + + // Count of inferences currently executing on this graph. Incremented when a + // MediapipeGraphExecutor is created (in create()) via the executor's RAII + // ActiveInferenceGuard, and decremented when that executor is destroyed (after + // the caller finishes infer()/inferStream()). The count is therefore held for + // the executor's lifetime, which spans the inference. A non-zero value prevents + // shouldUnloadDueToIdle()/unload() from tearing down the definition. + // Shared_ptr so MediapipeGraphExecutor can hold a copy safely beyond the + // create() call — the executor owns the counter reference for its lifetime. + std::shared_ptr> activeInferenceCount; + + // Cached copy of mgconfig.getIdleUnloadTimeoutSeconds() so the watcher thread can + // read it lock-free. mgconfig itself is only safe to read under lifecycleMtx + // (reload() reassigns it). Updated in the constructor and in reload() (under the + // lock) whenever mgconfig is assigned. + std::atomic idleUnloadTimeoutSecondsCache{0}; + + // Serializes ALL per-definition lifecycle mutations (reload/retire/unload/wakeUp) + // so they are mutually exclusive regardless of which thread runs them or which + // outer lock (ModelManager::configMtx) is held by the caller. This is required + // because unload() runs on the watcher thread (no configMtx) while reload()/retire() + // run on the config thread (under configMtx) and they mutate the same per-definition + // state (this->status variant, this->sidePacketMaps). + // Recursive because wakeUpIfUnloaded() holds it and calls reload(), which also takes it. + // Lock ordering is one-directional: configMtx -> lifecycleMtx. Nothing here ever + // acquires configMtx, so no deadlock is possible. + mutable std::recursive_mutex lifecycleMtx; }; } // namespace ovms diff --git a/src/mediapipe_internal/mediapipegraphexecutor.cpp b/src/mediapipe_internal/mediapipegraphexecutor.cpp index 26757de401..0ae0b5f4e9 100644 --- a/src/mediapipe_internal/mediapipegraphexecutor.cpp +++ b/src/mediapipe_internal/mediapipegraphexecutor.cpp @@ -47,7 +47,9 @@ MediapipeGraphExecutor::MediapipeGraphExecutor( const GraphSidePackets& sidePacketMaps, PythonBackend* pythonBackend, MediapipeServableMetricReporter* mediapipeServableMetricReporter, - GraphIdGuard&& guard) : + GraphIdGuard&& guard, + std::shared_ptr> activeInferenceCount, + std::shared_ptr> lastActivityTimeNs) : name(name), version(version), config(config), @@ -59,7 +61,10 @@ MediapipeGraphExecutor::MediapipeGraphExecutor( pythonBackend(pythonBackend), currentStreamTimestamp(::mediapipe::Timestamp(STARTING_TIMESTAMP_VALUE)), mediapipeServableMetricReporter(mediapipeServableMetricReporter), - guard(std::move(guard)) {} + guard(std::move(guard)), + activeInferenceGuard(activeInferenceCount + ? std::optional(ActiveInferenceGuard(std::move(activeInferenceCount), std::move(lastActivityTimeNs))) + : std::nullopt) {} MediapipeGraphExecutor::MediapipeGraphExecutor( const std::string& name, const std::string& version, @@ -70,7 +75,9 @@ MediapipeGraphExecutor::MediapipeGraphExecutor( std::vector outputNames, const GraphSidePackets& sidePacketMaps, PythonBackend* pythonBackend, - MediapipeServableMetricReporter* mediapipeServableMetricReporter) : + MediapipeServableMetricReporter* mediapipeServableMetricReporter, + std::shared_ptr> activeInferenceCount, + std::shared_ptr> lastActivityTimeNs) : name(name), version(version), config(config), @@ -81,6 +88,9 @@ MediapipeGraphExecutor::MediapipeGraphExecutor( sidePacketMaps(sidePacketMaps), pythonBackend(pythonBackend), currentStreamTimestamp(::mediapipe::Timestamp(STARTING_TIMESTAMP_VALUE)), - mediapipeServableMetricReporter(mediapipeServableMetricReporter) {} + mediapipeServableMetricReporter(mediapipeServableMetricReporter), + activeInferenceGuard(activeInferenceCount + ? std::optional(ActiveInferenceGuard(std::move(activeInferenceCount), std::move(lastActivityTimeNs))) + : std::nullopt) {} } // namespace ovms diff --git a/src/mediapipe_internal/mediapipegraphexecutor.hpp b/src/mediapipe_internal/mediapipegraphexecutor.hpp index 8dae471191..3948c6d722 100644 --- a/src/mediapipe_internal/mediapipegraphexecutor.hpp +++ b/src/mediapipe_internal/mediapipegraphexecutor.hpp @@ -14,6 +14,8 @@ // limitations under the License. //***************************************************************************** #pragma once +#include +#include #include #include #include @@ -48,6 +50,47 @@ namespace ovms { class PythonBackend; class ServableMetricReporter; + +// RAII guard that tracks an in-flight inference on a MediapipeGraphDefinition. +// Increments the counter on construction; decrements it and refreshes +// lastActivityTimeNs on destruction (even if the inference threw). Both are held as +// shared_ptrs so they remain valid even if the definition is reloaded or retired +// while the inference (and thus the owning executor) is still alive. +// The lastActivityTimeNs refresh on decrement ensures that completing a long +// generation resets the idle timer — preventing an immediate re-unload on the next +// watcher cycle. +struct ActiveInferenceGuard { + std::shared_ptr> counter; + std::shared_ptr> lastActivityTimeNs; + + ActiveInferenceGuard(std::shared_ptr> counter, + std::shared_ptr> lastActivityTimeNs) : + counter(std::move(counter)), + lastActivityTimeNs(std::move(lastActivityTimeNs)) { + if (this->counter) { + this->counter->fetch_add(1, std::memory_order_acq_rel); + } + } + + ~ActiveInferenceGuard() { + if (counter) { + // Refresh activity timestamp BEFORE decrementing so the watcher sees a + // recent activity time if it samples between the refresh and the decrement. + if (lastActivityTimeNs) { + lastActivityTimeNs->store( + std::chrono::steady_clock::now().time_since_epoch().count(), + std::memory_order_relaxed); + } + counter->fetch_sub(1, std::memory_order_acq_rel); + } + } + + // Non-copyable, movable. + ActiveInferenceGuard(const ActiveInferenceGuard&) = delete; + ActiveInferenceGuard& operator=(const ActiveInferenceGuard&) = delete; + ActiveInferenceGuard(ActiveInferenceGuard&&) = default; + ActiveInferenceGuard& operator=(ActiveInferenceGuard&&) = default; +}; class MediapipeGraphExecutor; inline StatusCode mediapipeAbslToOvmsStatus(absl::StatusCode code) { @@ -140,6 +183,13 @@ class MediapipeGraphExecutor { MediapipeServableMetricReporter* mediapipeServableMetricReporter; std::optional guard; + // RAII guard tracking this executor's active inference on the parent definition. + // Held for the entire lifetime of the executor so that the in-flight-inference + // check in shouldUnloadDueToIdle() / unload() sees a non-zero count while any + // inference method (infer / inferStream) is executing. On destruction (when the + // executor goes out of scope after inference completes) the counter decrements + // and lastActivityTimeNs is refreshed. + std::optional activeInferenceGuard; public: MediapipeGraphExecutor(const std::string& name, @@ -150,7 +200,9 @@ class MediapipeGraphExecutor { std::vector inputNames, std::vector outputNames, const GraphSidePackets& sidePacketMaps, PythonBackend* pythonBackend, - MediapipeServableMetricReporter* mediapipeServableMetricReporter, GraphIdGuard&& guard); + MediapipeServableMetricReporter* mediapipeServableMetricReporter, GraphIdGuard&& guard, + std::shared_ptr> activeInferenceCount = nullptr, + std::shared_ptr> lastActivityTimeNs = nullptr); // Constructor without graph queue (old path - graph created per-request) MediapipeGraphExecutor(const std::string& name, const std::string& version, @@ -160,7 +212,9 @@ class MediapipeGraphExecutor { std::vector inputNames, std::vector outputNames, const GraphSidePackets& sidePacketMaps, PythonBackend* pythonBackend, - MediapipeServableMetricReporter* mediapipeServableMetricReporter); + MediapipeServableMetricReporter* mediapipeServableMetricReporter, + std::shared_ptr> activeInferenceCount = nullptr, + std::shared_ptr> lastActivityTimeNs = nullptr); template Status infer(const RequestType* request, ResponseType* response, ExecutionContext executionContext) { diff --git a/src/metrics/metric_config.cpp b/src/metrics/metric_config.cpp index fedfb4a806..95997a4ab0 100644 --- a/src/metrics/metric_config.cpp +++ b/src/metrics/metric_config.cpp @@ -55,6 +55,7 @@ const std::string METRIC_NAME_WAIT_FOR_INFER_REQ_TIME = "ovms_wait_for_infer_req // MediaPipe const std::string METRIC_NAME_CURRENT_GRAPHS = "ovms_current_graphs"; +const std::string METRIC_NAME_GRAPH_LOADED = "ovms_graph_loaded"; const std::string METRIC_NAME_RESPONSES = "ovms_responses"; const std::string METRIC_NAME_REQUESTS_ACCEPTED = "ovms_requests_accepted"; diff --git a/src/metrics/metric_config.hpp b/src/metrics/metric_config.hpp index bf1430f9c2..365a5cdb67 100644 --- a/src/metrics/metric_config.hpp +++ b/src/metrics/metric_config.hpp @@ -41,6 +41,7 @@ extern const std::string METRIC_NAME_WAIT_FOR_INFER_REQ_TIME; // MediaPipe extern const std::string METRIC_NAME_CURRENT_GRAPHS; +extern const std::string METRIC_NAME_GRAPH_LOADED; extern const std::string METRIC_NAME_RESPONSES; extern const std::string METRIC_NAME_REQUESTS_ACCEPTED; @@ -100,6 +101,7 @@ class MetricConfig { {METRIC_NAME_INFERENCE_TIME}, {METRIC_NAME_WAIT_FOR_INFER_REQ_TIME}, {METRIC_NAME_CURRENT_GRAPHS}, + {METRIC_NAME_GRAPH_LOADED}, {METRIC_NAME_REQUESTS_ACCEPTED}, {METRIC_NAME_REQUESTS_REJECTED}, {METRIC_NAME_GRAPH_ERROR}, diff --git a/src/model_metric_reporter.cpp b/src/model_metric_reporter.cpp index 93d40140cc..7fd6378ae0 100644 --- a/src/model_metric_reporter.cpp +++ b/src/model_metric_reporter.cpp @@ -349,6 +349,18 @@ MediapipeServableMetricReporter::MediapipeServableMetricReporter(const MetricCon SPDLOG_INFO("DISABLED {}", METRIC_NAME_CURRENT_GRAPHS); } + familyName = METRIC_NAME_GRAPH_LOADED; + if (metricConfig->isFamilyEnabled(familyName)) { + auto family = registry->createFamily(familyName, + "Whether the MediaPipe graph resources are loaded (1) or idle-unloaded (0)."); + THROW_IF_NULL(family, "cannot create family"); + this->graphLoaded = family->addMetric( + {{"name", graphName}}); + THROW_IF_NULL(this->graphLoaded, "cannot create metric"); + } else { + SPDLOG_INFO("DISABLED {}", METRIC_NAME_GRAPH_LOADED); + } + familyName = METRIC_NAME_REQUESTS_ACCEPTED; if (metricConfig->isFamilyEnabled(familyName)) { auto family = registry->createFamily(familyName, diff --git a/src/model_metric_reporter.hpp b/src/model_metric_reporter.hpp index e300334742..c784b3f935 100644 --- a/src/model_metric_reporter.hpp +++ b/src/model_metric_reporter.hpp @@ -168,6 +168,9 @@ class MediapipeServableMetricReporter : public StatusMetricReporter { public: std::unique_ptr currentGraphs; + // 1 = graph resources loaded, 0 = idle-unloaded. Always 1 after a successful + // load for graphs that never enable idle unload. + std::unique_ptr graphLoaded; // KFS std::unique_ptr requestAcceptedGrpcModelInfer; diff --git a/src/modelmanager.cpp b/src/modelmanager.cpp index 25f6d49b2f..565fe36357 100644 --- a/src/modelmanager.cpp +++ b/src/modelmanager.cpp @@ -1052,6 +1052,37 @@ Status ModelManager::configFileReloadNeeded(bool& isNeeded) { return StatusCode::OK; } +void ModelManager::unloadIdleGraphs() { +#if (MEDIAPIPE_DISABLE == 0) + // Collect names of definitions that should be unloaded; iterate under + // a brief shared lock (inside the factory getters), then call unload() + // outside it. unload() re-checks all preconditions under lifecycleMtx + // and is non-blocking (skips graphs with in-flight requests). + std::vector toUnload; + { + const auto& names = mediapipeFactory->getMediapipePipelinesNames(); + for (const auto& name : names) { + MediapipeGraphDefinition* def = mediapipeFactory->findDefinitionByName(name); + if (def && def->shouldUnloadDueToIdle()) { + toUnload.push_back(name); + } + } + } + for (const auto& name : toUnload) { + MediapipeGraphDefinition* def = mediapipeFactory->findDefinitionByName(name); + if (def) { + // Re-check under the per-definition idle mutex to avoid racing with + // a concurrent wakeUp() that may have already transitioned the state. + auto status = def->unload(); + if (!status.ok()) { + SPDLOG_LOGGER_WARN(modelmanager_logger, + "Failed to idle-unload mediapipe graph {}: {}", name, status.string()); + } + } + } +#endif +} + void ModelManager::watcher(std::future exitSignal, bool watchConfigFile) { SPDLOG_LOGGER_INFO(modelmanager_logger, "Started model manager thread"); while (exitSignal.wait_for(std::chrono::milliseconds(this->watcherIntervalMillisec)) == std::future_status::timeout) { @@ -1066,6 +1097,12 @@ void ModelManager::watcher(std::future exitSignal, bool watchConfigFile) { } updateConfigurationWithoutConfigFile(); loadingLock.unlock(); + // Idle-unload sweep: free resources of graphs idle past their timeout. + // Done AFTER releasing configMtx — unload() only needs the factory's + // definitions lock and the per-definition lifecycleMtx, and is + // non-blocking (it skips graphs with in-flight requests rather than + // draining). This keeps configMtx hold time minimal. + unloadIdleGraphs(); SPDLOG_LOGGER_TRACE(modelmanager_logger, "Models configuration and filesystem check cycle end"); } SPDLOG_LOGGER_INFO(modelmanager_logger, "Stopped model manager thread"); @@ -1595,6 +1632,51 @@ const std::vector ModelManager::getNamesOfAvailableModels() const { Status ModelManager::createPipeline(std::unique_ptr& graph, const std::string& name) { #if (MEDIAPIPE_DISABLE == 0) + // Lazy wake-up with bounded retry. A request can observe state==AVAILABLE here, + // then have the watcher flip it to UNLOADED before create()->waitForLoaded() runs, + // which returns MEDIAPIPE_DEFINITION_NOT_LOADED_YET. We retry a bounded number of + // times: wake if UNLOADED, then create(); if create() fails specifically because + // the graph is not-loaded-yet AND it is currently UNLOADED, wake and retry. + // wakeUpIfUnloaded() serialises the transition internally so exactly one of N + // concurrent callers triggers the actual reload; the rest wait and then proceed. + constexpr int kMaxWakeAttempts = 3; + for (int attempt = 0; attempt < kMaxWakeAttempts; ++attempt) { + // Re-fetch the definition each iteration: a concurrent config reload may + // retire+erase it mid-loop, so a cached pointer could dangle. Bail cleanly + // if it is gone. + MediapipeGraphDefinition* def = this->mediapipeFactory->findDefinitionByName(name); + if (def == nullptr) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, + "Mediapipe graph {} no longer exists during wake-up loop", name); + return StatusCode::MEDIAPIPE_DEFINITION_NAME_MISSING; + } + if (def->getStateCode() == PipelineDefinitionStateCode::UNLOADED) { + auto wakeStatus = def->wakeUpIfUnloaded(*this); + if (!wakeStatus.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, + "Mediapipe graph {} wake-up failed: {}", name, wakeStatus.string()); + return wakeStatus; + } + } + auto createStatus = this->mediapipeFactory->create(graph, name); + if (createStatus.ok()) { + return createStatus; + } + // Only retry the specific race: graph got idle-unloaded between our check and + // waitForLoaded(). Any other failure (genuine load failure, missing graph, etc.) + // is returned immediately. Re-fetch to avoid using a possibly-stale pointer. + MediapipeGraphDefinition* defAfter = this->mediapipeFactory->findDefinitionByName(name); + bool racedWithUnload = defAfter && + (createStatus == StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_YET) && + (defAfter->getStateCode() == PipelineDefinitionStateCode::UNLOADED); + if (!racedWithUnload) { + return createStatus; + } + SPDLOG_LOGGER_DEBUG(modelmanager_logger, + "Mediapipe graph {} was idle-unloaded during request; retrying wake-up (attempt {}/{})", + name, attempt + 1, kMaxWakeAttempts); + } + // Exhausted retries — make one final attempt and return whatever it yields. return this->mediapipeFactory->create(graph, name); #else SPDLOG_ERROR("Mediapipe support was disabled during build process..."); diff --git a/src/modelmanager.hpp b/src/modelmanager.hpp index 22b80b5251..be4e9dcf06 100644 --- a/src/modelmanager.hpp +++ b/src/modelmanager.hpp @@ -128,6 +128,12 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M */ void watcher(std::future exitSignal, bool watchConfigFile); + /** + * @brief Sweep mediapipe graph definitions and unload any that have been + * idle past their configured idle_unload_timeout_seconds. + */ + void unloadIdleGraphs(); + /** * @brief Cleaner thread for resources cleanup */ diff --git a/src/schema.cpp b/src/schema.cpp index 7d76980dca..0c10a0da93 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -357,6 +357,10 @@ const std::string MODELS_CONFIG_SCHEMA = R"({ }, "subconfig": { "type": "string" + }, + "idle_unload_timeout_seconds": { + "type": "integer", + "minimum": 0 } }, "additionalProperties": false diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index e13cf29919..850215c88d 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -5178,3 +5178,641 @@ TEST(BaseGenerationConfigBuilderTest, SeedPreservedWhenExplicitlySet) { builder.parseConfigFromRequest(request); EXPECT_EQ(builder.getConfig().rng_seed, 42u); } + +// --------------------------------------------------------------------------- +// Idle unload feature: LLM graph lifecycle (issue #4141) +// These tests require the opt-125m model fixture. +// --------------------------------------------------------------------------- + +class LLMIdleUnloadTest : public ::testing::Test { +protected: + // Builds a minimal continuous-batching LLM graph pbtxt pointing at opt-125m. + static std::string buildOptGraphPbtxt() { + std::string modelsPath = getGenericFullPathForSrcTest("/ovms/src/test/llm_testing/facebook/opt-125m"); + std::string testPbtxt = R"( + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + + node: { + name: "llmNode" + calculator: "HttpLLMCalculator" + input_stream: "LOOPBACK:loopback" + input_stream: "HTTP_REQUEST_PAYLOAD:input" + input_side_packet: "LLM_NODE_RESOURCES:llm" + output_stream: "LOOPBACK:loopback" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + input_stream_info: { + tag_index: 'LOOPBACK:0', + back_edge: true + } + node_options: { + [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { + models_path: ")" + + modelsPath + R"(" + cache_size: 1 + } + } + input_stream_handler { + input_stream_handler: "SyncSetInputStreamHandler", + options { + [mediapipe.SyncSetInputStreamHandlerOptions.ext] { + sync_set { + tag_index: "LOOPBACK:0" + } + } + } + } + } + )"; + adjustConfigForTargetPlatform(testPbtxt); + return testPbtxt; + } +}; + +// Unload after idle: build LLM graph with small timeout, simulate idle, unload, assert freed. +TEST_F(LLMIdleUnloadTest, UnloadAfterIdleFreesResources) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + ASSERT_NE(def.getGenAiServable("llmNode"), nullptr); + ASSERT_TRUE(def.isIdleUnloadEnabled()); + + // Not yet idle -> should not unload. + ASSERT_FALSE(def.shouldUnloadDueToIdle()); + + // Backdate activity well past the timeout. + def.backdateLastActivityForTest(60); + ASSERT_TRUE(def.shouldUnloadDueToIdle()); + + ASSERT_EQ(def.unload(), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + // Resources freed: the GenAi servable map should be empty. + ASSERT_TRUE(def.getGenAiServableMap().empty()); + ASSERT_FALSE(def.isAvailable()); +} + +// Lazy reload: after unload, wakeUpIfUnloaded brings it back to AVAILABLE with resources. +TEST_F(LLMIdleUnloadTest, WakeUpReloadsResources) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + def.backdateLastActivityForTest(60); + ASSERT_EQ(def.unload(), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + ASSERT_TRUE(def.getGenAiServableMap().empty()); + + // Wake up. + ASSERT_EQ(def.wakeUpIfUnloaded(manager), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + ASSERT_TRUE(def.isAvailable()); + ASSERT_NE(def.getGenAiServable("llmNode"), nullptr); + + // Wake-up while already AVAILABLE is a no-op success. + ASSERT_EQ(def.wakeUpIfUnloaded(manager), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); +} + +// Idle timer reset: acquiring the graph (create) refreshes lastActivity. +TEST_F(LLMIdleUnloadTest, CreateResetsIdleTimer) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + // Make it look idle. + def.backdateLastActivityForTest(60); + ASSERT_TRUE(def.shouldUnloadDueToIdle()); + + // Acquiring the graph updates lastActivity, so it is no longer idle. + std::unique_ptr executor; + ASSERT_EQ(def.create(executor), StatusCode::OK); + ASSERT_NE(executor, nullptr); + ASSERT_FALSE(def.shouldUnloadDueToIdle()); +} + +// Disabled by default: timeout 0 -> never idle-unloads. +TEST_F(LLMIdleUnloadTest, DisabledByDefaultNeverUnloads) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + // idle_unload_timeout_seconds not set -> defaults to 0 (disabled) + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + ASSERT_FALSE(def.isIdleUnloadEnabled()); + def.backdateLastActivityForTest(100000); + ASSERT_FALSE(def.shouldUnloadDueToIdle()); +} + +// Python-node guard: a graph with a Python node + idle timeout > 0 fails validation. +#if (PYTHON_DISABLE == 0) +TEST_F(LLMIdleUnloadTest, PythonNodeWithIdleUnloadRejected) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = R"( + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + node: { + name: "pythonNode" + calculator: "PythonExecutorCalculator" + input_side_packet: "PYTHON_NODE_RESOURCES:py" + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + } + )"; + adjustConfigForTargetPlatform(testPbtxt); + + ovms::MediapipeGraphConfig mgc{"mediaPy", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaPy", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + auto status = def.validate(manager); + ASSERT_EQ(status, StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID) << status.string(); +} +#endif + +// Exactly-one-reload under concurrency: N threads call wakeUpIfUnloaded on an UNLOADED def. +// Best-effort: asserts all end AVAILABLE and the graph is loaded exactly once afterwards. +// Note: this verifies the end-state invariant (single AVAILABLE graph, resources present); +// the per-definition mutex guarantees a single reload, but counting reloads deterministically +// from the test would require instrumentation hooks not present, so we assert the observable +// post-condition instead. +TEST_F(LLMIdleUnloadTest, ConcurrentWakeUpEndsAvailable) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + def.backdateLastActivityForTest(60); + ASSERT_EQ(def.unload(), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + + constexpr int kThreads = 8; + std::vector threads; + std::vector results(kThreads, StatusCode::UNKNOWN_ERROR); + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&def, &manager, &results, i]() { + results[i] = def.wakeUpIfUnloaded(manager); + }); + } + for (auto& t : threads) { + t.join(); + } + for (int i = 0; i < kThreads; ++i) { + ASSERT_EQ(results[i], StatusCode::OK) << "thread " << i << " status: " << results[i].string(); + } + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + ASSERT_NE(def.getGenAiServable("llmNode"), nullptr); +} + +// Best-effort stress: interleave unload() (watcher role) and wakeUpIfUnloaded() +// (request role) repeatedly and assert the graph never ends in a torn state. +// lifecycleMtx makes unload and wake mutually exclusive, so every observed +// settled state must be internally consistent: AVAILABLE with resources, or +// cleanly UNLOADED (empty maps). Determinism is limited by thread scheduling; +// this exercises the FIX 1/FIX 2 serialization rather than asserting an exact +// sequence. +TEST_F(LLMIdleUnloadTest, ConcurrentUnloadWakeNeverTearsState) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + std::atomic stop{false}; + std::atomic errors{0}; + + // Unloader thread: keeps backdating + trying to unload. + std::thread unloader([&]() { + while (!stop.load()) { + def.backdateLastActivityForTest(60); + auto s = def.unload(); + if (!s.ok()) + errors.fetch_add(1); + std::this_thread::yield(); + } + }); + + // Several waker threads: keep waking it back up. + constexpr int kWakers = 4; + std::vector wakers; + for (int i = 0; i < kWakers; ++i) { + wakers.emplace_back([&]() { + while (!stop.load()) { + auto s = def.wakeUpIfUnloaded(manager); + if (!s.ok()) + errors.fetch_add(1); + std::this_thread::yield(); + } + }); + } + + // Run for a short bounded period. + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + stop.store(true); + unloader.join(); + for (auto& t : wakers) { + t.join(); + } + + ASSERT_EQ(errors.load(), 0); + + // Quiesce: ensure it ends AVAILABLE with resources (no torn RELOADING/null state). + ASSERT_EQ(def.wakeUpIfUnloaded(manager), StatusCode::OK); + auto finalState = def.getStateCode(); + // A settled state must be either AVAILABLE (with resources) or UNLOADED (empty). + if (finalState == ovms::PipelineDefinitionStateCode::AVAILABLE) { + ASSERT_NE(def.getGenAiServable("llmNode"), nullptr); + } else { + ASSERT_EQ(finalState, ovms::PipelineDefinitionStateCode::UNLOADED); + ASSERT_TRUE(def.getGenAiServableMap().empty()); + } +} + +// Best-effort: exercise unload() (watcher role) concurrently with reload() and +// retire() (config role) on the same definition. Verifies the lifecycleMtx +// serialization (NEW-1 fix): no crash, and a consistent final state. +// NOTE: data races are not deterministically catchable without TSAN (unavailable +// in this environment), so this is a smoke/stress test, not a proof of absence. +TEST_F(LLMIdleUnloadTest, ConcurrentUnloadReloadRetireNoCrash) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + std::atomic stop{false}; + std::atomic retired{false}; + + // Watcher-role thread: keep trying to idle-unload. + std::thread unloader([&]() { + while (!stop.load()) { + def.backdateLastActivityForTest(60); + (void)def.unload(); + std::this_thread::yield(); + } + }); + + // Config-role thread: keep reloading (re-bring it up after unload). + std::thread reloader([&]() { + while (!stop.load()) { + (void)def.reload(manager, def.getMediapipeGraphConfig()); + std::this_thread::yield(); + } + }); + + std::this_thread::sleep_for(std::chrono::milliseconds(400)); + stop.store(true); + unloader.join(); + reloader.join(); + + // Now retire concurrently is not needed for crash-safety beyond above, but + // exercise retire() once after the storm to confirm it serializes cleanly. + def.retire(); + retired.store(true); + ASSERT_TRUE(retired.load()); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::RETIRED); +} + +// ───────────────────────────────────────────────────────────────────────────── +// TASK 1 tests: ActiveInferenceGuard — in-flight inference prevents idle unload +// ───────────────────────────────────────────────────────────────────────────── + +// Model-free unit test: directly exercise the activeInferenceCount atomic that +// shouldUnloadDueToIdle() and unload() consult. No LLM model required. +TEST(MediapipeIdleUnloadGuard, ActiveInferenceCountBlocksShouldUnload) { + // Build a minimal graph definition with idle unload enabled. + // Use buildOptGraphPbtxt() indirectly via LLMIdleUnloadTest helpers is not + // available here — we just need a definition with a non-zero timeout and + // a synthetic counter. We can use the shared_ptr that getActiveInferenceCount() + // returns directly, bypassing the executor machinery. + + // A standalone atomic acts as the counter. + auto counter = std::make_shared>(0); + auto lastActivity = std::make_shared>( + std::chrono::steady_clock::now().time_since_epoch().count() - 60LL * 1'000'000'000LL); + + // Simulate increment (inference start). + { + ovms::ActiveInferenceGuard guard(counter, lastActivity); + EXPECT_EQ(counter->load(), 1); + } + // After destruction, counter back to 0 and lastActivity refreshed. + EXPECT_EQ(counter->load(), 0); + int64_t nowNs = std::chrono::steady_clock::now().time_since_epoch().count(); + // lastActivity should be within 2 seconds of now (generous for slow machines). + EXPECT_GT(lastActivity->load(), nowNs - 2LL * 1'000'000'000LL); +} + +TEST(MediapipeIdleUnloadGuard, ActiveInferenceCountExceptionSafe) { + auto counter = std::make_shared>(0); + auto lastActivity = std::make_shared>(0); + + try { + ovms::ActiveInferenceGuard guard(counter, lastActivity); + EXPECT_EQ(counter->load(), 1); + throw std::runtime_error("simulated inference error"); + } catch (...) { + } + // Must be 0 even after exception path. + EXPECT_EQ(counter->load(), 0); +} + +TEST(MediapipeIdleUnloadGuard, MultipleGuardsNested) { + auto counter = std::make_shared>(0); + auto lastActivity = std::make_shared>(0); + { + ovms::ActiveInferenceGuard g1(counter, lastActivity); + EXPECT_EQ(counter->load(), 1); + { + ovms::ActiveInferenceGuard g2(counter, lastActivity); + EXPECT_EQ(counter->load(), 2); + } + EXPECT_EQ(counter->load(), 1); + } + EXPECT_EQ(counter->load(), 0); +} + +// Integration test: create() on a real definition increments the counter; +// when the executor is destroyed the counter returns to 0. +// Requires the LLM model (opt-125m). Guard under GTEST_SKIP for CI environments. +TEST_F(LLMIdleUnloadTest, ActiveInferenceGuardIntegration) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + const std::string testModelsPath = getGenericFullPathForSrcTest("/ovms/src/test/llm_testing/facebook/opt-125m"); + if (!std::filesystem::exists(testModelsPath)) { + GTEST_SKIP() << "opt-125m model not present; skipping integration guard test"; + } + + ovms::MediapipeGraphConfig mgc{"mediaGuard", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaGuard", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + auto counterPtr = def.getActiveInferenceCount(); + ASSERT_NE(counterPtr, nullptr); + EXPECT_EQ(counterPtr->load(), 0); + + { + std::unique_ptr executor; + ASSERT_EQ(def.create(executor), StatusCode::OK); + ASSERT_NE(executor, nullptr); + // Counter incremented: executor is alive. + EXPECT_EQ(counterPtr->load(), 1); + + // Backdate activity to look idle — should NOT unload because count > 0. + def.backdateLastActivityForTest(60); + EXPECT_FALSE(def.shouldUnloadDueToIdle()); + EXPECT_EQ(def.unload(), StatusCode::OK); + // unload() should have been skipped (counter > 0) so we stay AVAILABLE. + EXPECT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + } // executor destroyed here -> counter decremented back to 0 + + EXPECT_EQ(counterPtr->load(), 0); + // Completing the inference refreshed lastActivityTimeNs (the ActiveInferenceGuard + // destructor resets the idle timer), so the graph is NOT idle immediately after — + // this is the key behavior preventing an immediate re-unload right after a long + // generation finishes. + EXPECT_FALSE(def.shouldUnloadDueToIdle()); + // After the idle period elapses again (post-inference), it should unload. + def.backdateLastActivityForTest(60); + EXPECT_TRUE(def.shouldUnloadDueToIdle()); + EXPECT_EQ(def.unload(), StatusCode::OK); + EXPECT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Wake-failure recovery: a failed wake-up reload must leave the graph UNLOADED +// (retryable), NOT LOADING_PRECONDITION_FAILED (wedged). Then once the underlying +// problem is resolved, the next wake self-heals to AVAILABLE. +// ───────────────────────────────────────────────────────────────────────────── + +// Returns an LLM graph pbtxt whose models_path points at a nonexistent directory, +// so validate() fails (LLM_NODE_DIRECTORY_DOES_NOT_EXIST) — but it still contains +// HttpLLMCalculator, so the idle-unload scope check passes and we exercise the +// wake/reload/validate failure path. +static std::string buildBrokenOptGraphPbtxt() { + std::string testPbtxt = R"( + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + + node: { + name: "llmNode" + calculator: "HttpLLMCalculator" + input_stream: "LOOPBACK:loopback" + input_stream: "HTTP_REQUEST_PAYLOAD:input" + input_side_packet: "LLM_NODE_RESOURCES:llm" + output_stream: "LOOPBACK:loopback" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + input_stream_info: { + tag_index: 'LOOPBACK:0', + back_edge: true + } + node_options: { + [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { + models_path: "/this/path/definitely/does/not/exist/opt-125m" + cache_size: 1 + } + } + input_stream_handler { + input_stream_handler: "SyncSetInputStreamHandler", + options { + [mediapipe.SyncSetInputStreamHandlerOptions.ext] { + sync_set { + tag_index: "LOOPBACK:0" + } + } + } + } + } + )"; + adjustConfigForTargetPlatform(testPbtxt); + return testPbtxt; +} + +TEST_F(LLMIdleUnloadTest, FailedWakeLeavesGraphUnloadedAndRetryable) { + ConstructorEnabledModelManager manager; + std::string goodPbtxt = buildOptGraphPbtxt(); + std::string brokenPbtxt = buildBrokenOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaWakeFail", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaWakeFail", mgc, goodPbtxt, nullptr); + def.inputConfig = goodPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + + // Idle-unload the healthy graph. + def.backdateLastActivityForTest(60); + ASSERT_EQ(def.unload(), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + + // Simulate the model becoming temporarily unavailable: swap in a broken config + // so the wake-up reload's validate() fails. + def.inputConfig = brokenPbtxt; + auto failStatus = def.wakeUpIfUnloaded(manager); + EXPECT_FALSE(failStatus.ok()) << "expected wake-up to fail with broken model"; + // CRITICAL: the graph must be retryable, i.e. back in UNLOADED — not wedged in + // LOADING_PRECONDITION_FAILED. + EXPECT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + + // A second attempt while still broken also fails but stays retryable. + auto failStatus2 = def.wakeUpIfUnloaded(manager); + EXPECT_FALSE(failStatus2.ok()); + EXPECT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + + // Restore the model: the next wake self-heals to AVAILABLE. + def.inputConfig = goodPbtxt; + auto okStatus = def.wakeUpIfUnloaded(manager); + EXPECT_EQ(okStatus, StatusCode::OK) << okStatus.string(); + EXPECT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + EXPECT_NE(def.getGenAiServable("llmNode"), nullptr); +} + +// ───────────────────────────────────────────────────────────────────────────── +// TASK 2 tests: non-LLM scope restriction — idle_unload_timeout on non-LLM graphs +// ───────────────────────────────────────────────────────────────────────────── + +// A plain passthrough graph (no LLM calculator) with idle_unload_timeout > 0 must fail +// validation with MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID. +TEST_F(LLMIdleUnloadTest, NonLlmGraphWithIdleUnloadRejected) { + ConstructorEnabledModelManager manager; + // Minimal graph with a generic passthrough-style calculator (not HttpLLMCalculator). + std::string testPbtxt = R"( + input_stream: "INPUT:input" + output_stream: "OUTPUT:output" + node: { + name: "passthroughNode" + calculator: "PassThroughCalculator" + input_stream: "INPUT:input" + output_stream: "OUTPUT:output" + } + )"; + + ovms::MediapipeGraphConfig mgc{"mediaPassthrough", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaPassthrough", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + auto status = def.validate(manager); + EXPECT_EQ(status, StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID) + << "Expected non-LLM graph with idle_unload_timeout_seconds to be rejected, got: " << status.string(); +} + +// An embeddings-style graph (no HttpLLMCalculator) with idle_unload_timeout > 0 +// must also be rejected. +TEST_F(LLMIdleUnloadTest, EmbeddingsGraphWithIdleUnloadRejected) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = R"( + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + node: { + name: "embeddingNode" + calculator: "HttpOpenVINOEmbeddingsCalculator" + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + input_side_packet: "LLM_NODE_RESOURCES:llm" + } + )"; + + ovms::MediapipeGraphConfig mgc{"mediaEmbed", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(5); + DummyMediapipeGraphDefinition def("mediaEmbed", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + auto status = def.validate(manager); + EXPECT_EQ(status, StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID) + << "Expected embeddings graph with idle_unload_timeout_seconds to be rejected, got: " << status.string(); +} + +// A valid LLM graph with idle_unload_timeout_seconds > 0 must still pass the calculator check. +// (The full validate() may fail if the model files are absent — that's fine; we test the +// calculator-guard path specifically by checking that the rejection is NOT due to the +// non-LLM guard. In CI without the model, validate() may fail with a different status, +// but it must NOT be MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID from our new guard.) +TEST_F(LLMIdleUnloadTest, LlmGraphPassesCalculatorGuard) { + ConstructorEnabledModelManager manager; + // Intentionally malformed LLM graph (no model path) — will fail downstream validation + // but must NOT fail with the non-LLM calculator guard. + std::string testPbtxt = R"( + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + node: { + name: "llmNode" + calculator: "HttpLLMCalculator" + input_stream: "LOOPBACK:loopback" + input_stream: "HTTP_REQUEST_PAYLOAD:input" + input_side_packet: "LLM_NODE_RESOURCES:llm" + output_stream: "LOOPBACK:loopback" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + input_stream_info: { + tag_index: 'LOOPBACK:0', + back_edge: true + } + node_options: { + [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { + models_path: "/nonexistent/path/to/model" + cache_size: 1 + } + } + input_stream_handler { + input_stream_handler: "SyncSetInputStreamHandler", + options { + [mediapipe.SyncSetInputStreamHandlerOptions.ext] { + sync_set { + tag_index: "LOOPBACK:0" + } + } + } + } + } + )"; + + ovms::MediapipeGraphConfig mgc{"mediaLlmGuard", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaLlmGuard", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + auto status = def.validate(manager); + // The non-LLM calculator guard must NOT have fired (the status must not be the + // "idle_unload only supported for LLM" rejection). Any other failure (e.g. model + // path not found) is acceptable. + // We cannot distinguish the exact downstream error code without running the model, + // so we simply assert it's not the guard-specific code or (if it happens to pass + // on machines with the model) OK. + bool isNonLlmGuardRejection = (status == StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); + if (isNonLlmGuardRejection) { + // If the code is MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID, it must be from a different + // guard (e.g. Python node or queue config), not our new non-LLM guard — verify by + // checking the error message doesn't contain our sentinel phrase. + // Since we can't inspect the message here, we check that HttpLLMCalculator is present + // (it is) and assert that the guard check passed (no rejection for non-LLM). + FAIL() << "LLM graph was unexpectedly rejected by idle_unload scope guard; status: " + << status.string(); + } + // Any other status (OK, model-not-found, etc.) is acceptable. +} diff --git a/src/test/mediapipeflow_test.cpp b/src/test/mediapipeflow_test.cpp index 459a0d5115..8c40b57b8c 100644 --- a/src/test/mediapipeflow_test.cpp +++ b/src/test/mediapipeflow_test.cpp @@ -55,6 +55,7 @@ #include "../model_service.hpp" #include "../ovms_exit_codes.hpp" #include "../precision.hpp" +#include "../servable_definition_unload_guard.hpp" #include "../servablemanagermodule.hpp" #include "../server.hpp" #include "../shape.hpp" @@ -4410,3 +4411,99 @@ TEST_F(UnaryQueueReinitTest, GraphIsReinitializedAfterCalculatorError) { ASSERT_TRUE(status.ok()); } } + +// --------------------------------------------------------------------------- +// Idle unload feature: unload() guard correctness (issue #4141, model-free) +// Verifies FIX 1: unload() must NOT tear down resources unless the state was +// actually AVAILABLE and the UnloadEvent transition really happened. +// --------------------------------------------------------------------------- + +// A trivial pbtxt is enough; these tests never reach validate(), they drive the +// state machine directly to exercise unload()'s preconditions. +static const std::string kIdleUnloadDummyPbtxt = R"( + input_stream: "in" + output_stream: "out" +)"; + +TEST(MediapipeIdleUnloadGuard, UnloadIsNoOpWhenStateBegin) { + ovms::MediapipeGraphConfig mgc{"idleGuard", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("idleGuard", mgc, kIdleUnloadDummyPbtxt, nullptr); + // Fresh definition is in BEGIN (validate never called). + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::BEGIN); + def.insertSidePacketMarkerForTest("marker"); + const void* mapsBefore = def.sidePacketMapsPtrForTest(); + + ASSERT_EQ(def.unload(), ovms::StatusCode::OK); + + // State unchanged and resources untouched. + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::BEGIN); + ASSERT_TRUE(def.hasSidePacketMarkerForTest("marker")); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), mapsBefore); +} + +TEST(MediapipeIdleUnloadGuard, UnloadIsNoOpWhenStateReloading) { + ovms::MediapipeGraphConfig mgc{"idleGuard", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("idleGuard", mgc, kIdleUnloadDummyPbtxt, nullptr); + // Drive BEGIN -> AVAILABLE -> RELOADING. + def.forceValidationPassedEventForTest(); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + def.forceReloadEventForTest(); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); + + def.insertSidePacketMarkerForTest("marker"); + const void* mapsBefore = def.sidePacketMapsPtrForTest(); + + ASSERT_EQ(def.unload(), ovms::StatusCode::OK); + + // Critical: unload() must NOT have cleared resources while RELOADING. + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); + ASSERT_TRUE(def.hasSidePacketMarkerForTest("marker")); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), mapsBefore); +} + +TEST(MediapipeIdleUnloadGuard, UnloadTransitionsAndTearsDownWhenAvailable) { + ovms::MediapipeGraphConfig mgc{"idleGuard", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("idleGuard", mgc, kIdleUnloadDummyPbtxt, nullptr); + def.forceValidationPassedEventForTest(); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + def.insertSidePacketMarkerForTest("marker"); + const void* mapsBefore = def.sidePacketMapsPtrForTest(); + + ASSERT_EQ(def.unload(), ovms::StatusCode::OK); + + // Now it should have transitioned and cleared (but kept the same object). + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + ASSERT_FALSE(def.hasSidePacketMarkerForTest("marker")); + ASSERT_TRUE(def.sidePacketMapsEmptyForTest()); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), mapsBefore); // clear(), not reset() +} + +TEST(MediapipeIdleUnloadGuard, UnloadSkipsWhenRequestsInFlight) { + ovms::MediapipeGraphConfig mgc{"idleGuard", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("idleGuard", mgc, kIdleUnloadDummyPbtxt, nullptr); + def.forceValidationPassedEventForTest(); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + def.insertSidePacketMarkerForTest("marker"); + const void* mapsBefore = def.sidePacketMapsPtrForTest(); + + { + // Simulate an in-flight request by holding an unload guard (bumps the counter). + ovms::ServableDefinitionUnloadGuard guard(def); + ASSERT_EQ(def.requestsHandlesCounterForTest(), 1u); + + // unload() must skip without tearing down because counter > 0. + ASSERT_EQ(def.unload(), ovms::StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + ASSERT_TRUE(def.hasSidePacketMarkerForTest("marker")); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), mapsBefore); + } + // After the guard releases, unload() now proceeds. + ASSERT_EQ(def.requestsHandlesCounterForTest(), 0u); + ASSERT_EQ(def.unload(), ovms::StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + ASSERT_TRUE(def.sidePacketMapsEmptyForTest()); +} diff --git a/src/test/pipelinedefinitionstatus_test.cpp b/src/test/pipelinedefinitionstatus_test.cpp index 578ad38295..e920af07dd 100644 --- a/src/test/pipelinedefinitionstatus_test.cpp +++ b/src/test/pipelinedefinitionstatus_test.cpp @@ -302,3 +302,133 @@ TEST(PipelineDefinitionStatus, ConvertToModelStatus) { ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RETIRED); ASSERT_EQ((std::tuple(ModelVersionState::END, ModelVersionStatusErrorCode::OK)), pds.convertToModelStatus()); } + +// --------------------------------------------------------------------------- +// Idle unload feature: UNLOADED state transitions (issue #4141) +// --------------------------------------------------------------------------- + +TEST(PipelineDefinitionStatus, AvailableThenUnload) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); +} + +TEST(PipelineDefinitionStatus, UnloadedThenReloadGoesToReloading) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + pds.handle(ReloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); +} + +TEST(PipelineDefinitionStatus, UnloadedThenReloadThenValidationPassGoesToAvailable) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(UnloadEvent()); + pds.handle(ReloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); + pds.handle(ValidationPassedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); +} + +TEST(PipelineDefinitionStatus, UnloadedThenValidationPassDefensiveGoesToAvailable) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + pds.handle(ValidationPassedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); +} + +TEST(PipelineDefinitionStatus, UnloadedThenRetireGoesToRetired) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + pds.handle(RetireEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RETIRED); +} + +TEST(PipelineDefinitionStatus, UnloadedIsNotAvailable) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + ASSERT_TRUE(pds.isAvailable()); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + ASSERT_FALSE(pds.isAvailable()); +} + +TEST(PipelineDefinitionStatus, UnloadedConvertsToModelStatusAvailable) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + // UNLOADED must report AVAILABLE so health checks / routing do not exclude the + // servable (it auto-reloads on the next inference request). + ASSERT_EQ((std::tuple(ModelVersionState::AVAILABLE, ModelVersionStatusErrorCode::OK)), pds.convertToModelStatus()); +} + +TEST(PipelineDefinitionStatus, UnloadEventOnBeginIsNoOp) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::BEGIN); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::BEGIN); +} + +TEST(PipelineDefinitionStatus, UnloadEventOnReloadingIsNoOp) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(ReloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); +} + +TEST(PipelineDefinitionStatus, UnloadEventOnRetiredIsNoOp) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(RetireEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RETIRED); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RETIRED); +} + +TEST(PipelineDefinitionStatus, UnloadEventOnLoadingPreconditionFailedRevertsToUnloaded) { + // A failed wake-up reload (validate -> LOADING_PRECONDITION_FAILED) is reverted + // to UNLOADED by wakeUpIfUnloaded() via UnloadEvent so the next request retries. + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationFailedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); +} + +TEST(PipelineDefinitionStatus, UnloadedAfterFailedWakeIsRetryableViaReload) { + // Full retry path: AVAILABLE -> UNLOADED -> (wake) RELOADING -> (fail) FAILED + // -> (revert) UNLOADED -> (retry wake) RELOADING -> (pass) AVAILABLE. + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + pds.handle(ReloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); + pds.handle(ValidationFailedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED); + pds.handle(UnloadEvent()); // wakeUpIfUnloaded reverts on failure + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + pds.handle(ReloadEvent()); + pds.handle(ValidationPassedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); +} + +TEST(PipelineDefinitionStatus, UnloadEventOnUnloadedIsIdempotent) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + pds.handle(UnloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); +} diff --git a/src/test/schema_test.cpp b/src/test/schema_test.cpp index 2b5aaf11fd..fad87515f2 100644 --- a/src/test/schema_test.cpp +++ b/src/test/schema_test.cpp @@ -2003,6 +2003,66 @@ TEST(SchemaTest, MediapipeConfigInModelConfigPositive) { auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); EXPECT_EQ(result, ovms::StatusCode::OK); } + +TEST(SchemaTest, MediapipeConfigIdleUnloadTimeoutPositive) { + const char* mediapipeConfigPositive = R"( + { + "model_config_list": [], + "mediapipe_config_list": [ + { + "name": "dummy_model", + "graph_path": "graph.pbtxt", + "base_path": "dummy_path_base", + "idle_unload_timeout_seconds": 300 + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(mediapipeConfigPositive); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::OK); +} + +TEST(SchemaTest, MediapipeConfigIdleUnloadTimeoutNegativeValueRejected) { + const char* mediapipeConfigNegative = R"( + { + "model_config_list": [], + "mediapipe_config_list": [ + { + "name": "dummy_model", + "graph_path": "graph.pbtxt", + "base_path": "dummy_path_base", + "idle_unload_timeout_seconds": -5 + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(mediapipeConfigNegative); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} + +TEST(SchemaTest, MediapipeConfigIdleUnloadTimeoutWrongTypeRejected) { + const char* mediapipeConfigNegative = R"( + { + "model_config_list": [], + "mediapipe_config_list": [ + { + "name": "dummy_model", + "graph_path": "graph.pbtxt", + "base_path": "dummy_path_base", + "idle_unload_timeout_seconds": "notAnInteger" + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(mediapipeConfigNegative); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} #endif TEST(SchemaTest, MediapipeConfigNegativeAdditionalMediapipeConfigField) { diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index 2510ca2b60..eeb00f3d73 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -853,6 +853,23 @@ class DummyMediapipeGraphDefinition : public ovms::MediapipeGraphDefinition { ovms::GenAiServableMap& getGenAiServableMap() { return this->sidePacketMaps->genAiServableMap; } + // Test seams for idle-unload concurrency tests. + // Drive the underlying state machine directly. + void forceReloadEventForTest() { this->status.handle(ovms::ReloadEvent()); } + void forceValidationPassedEventForTest() { this->status.handle(ovms::ValidationPassedEvent()); } + // Identity of the sidePacketMaps shared_ptr, so a test can detect whether it + // was reset/swapped (unload uses clear(), not reset(), so the pointer must be stable). + const void* sidePacketMapsPtrForTest() const { return static_cast(this->sidePacketMaps.get()); } + bool sidePacketMapsEmptyForTest() { return this->sidePacketMaps->empty(); } + // Insert a harmless marker into a side-packet map so we can detect teardown. + void insertSidePacketMarkerForTest(const std::string& key) { + this->sidePacketMaps->genAiServableMap.insert({key, nullptr}); + } + bool hasSidePacketMarkerForTest(const std::string& key) { + return this->sidePacketMaps->genAiServableMap.count(key) > 0; + } + uint64_t requestsHandlesCounterForTest() const { return this->requestsHandlesCounter.load(); } + DummyMediapipeGraphDefinition(const std::string name, const ovms::MediapipeGraphConfig& config, std::string inputConfig, From ceb9f4dea6ab7fc7a241f0831e4c1ea56bb9d751 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 20 Aug 2026 17:52:59 +0200 Subject: [PATCH 02/10] extend idle model management to groups --- src/BUILD | 5 +- src/capi_frontend/capi_dag_utils.cpp | 2 + src/capi_frontend/server_settings.hpp | 1 + src/cli_parser.cpp | 5 + src/config.cpp | 1 + src/config.hpp | 7 + src/http_rest_api_handler.cpp | 22 + .../kfs_grpc_inference_service.cpp | 20 + .../mediapipegraphconfig.cpp | 6 + .../mediapipegraphconfig.hpp | 13 + src/model_group_manager.cpp | 396 ++++++++++++++++++ src/model_group_manager.hpp | 86 ++++ src/modelconfig.cpp | 12 + src/modelconfig.hpp | 23 + src/modelmanager.cpp | 63 +++ src/schema.cpp | 6 + src/status.cpp | 3 + src/status.hpp | 4 + src/test/model_group_manager_test.cpp | 182 ++++++++ src/test/schema_test.cpp | 83 +++- src/test/template_test.cpp | 104 +++++ 21 files changed, 1040 insertions(+), 4 deletions(-) create mode 100644 src/model_group_manager.cpp create mode 100644 src/model_group_manager.hpp create mode 100644 src/test/model_group_manager_test.cpp create mode 100644 src/test/template_test.cpp diff --git a/src/BUILD b/src/BUILD index a50b90a904..8c47a9dfe6 100644 --- a/src/BUILD +++ b/src/BUILD @@ -583,8 +583,8 @@ ovms_cc_library( ) ovms_cc_library( name = "modelmanager", - hdrs = ["modelmanager.hpp"], - srcs = ["modelmanager.cpp"], + hdrs = ["modelmanager.hpp", "model_group_manager.hpp"], + srcs = ["modelmanager.cpp", "model_group_manager.cpp"], deps = select({ "//conditions:default": [], "//:not_disable_mediapipe" : [ @@ -2032,6 +2032,7 @@ cc_test( "test/model_version_policy_test.cpp", "test/modelconfig_test.cpp", "test/modelinstance_test.cpp", + "test/model_group_manager_test.cpp", "test/modelmanager_test.cpp", "test/modelversionstatus_test.cpp", "test/node_library_manager_test.cpp", diff --git a/src/capi_frontend/capi_dag_utils.cpp b/src/capi_frontend/capi_dag_utils.cpp index c2b13d4eaa..5b7c867ec6 100644 --- a/src/capi_frontend/capi_dag_utils.cpp +++ b/src/capi_frontend/capi_dag_utils.cpp @@ -41,6 +41,8 @@ OVMS_ServableState convertToServableState(ovms::PipelineDefinitionStateCode code case ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED: case ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION: return OVMS_ServableState::OVMS_STATE_LOADING_FAILED; + case ovms::PipelineDefinitionStateCode::UNLOADED: + return OVMS_ServableState::OVMS_STATE_RETIRED; } throw new std::exception(); } diff --git a/src/capi_frontend/server_settings.hpp b/src/capi_frontend/server_settings.hpp index 8ee00f9869..491ea173bb 100644 --- a/src/capi_frontend/server_settings.hpp +++ b/src/capi_frontend/server_settings.hpp @@ -241,6 +241,7 @@ struct ServerSettingsImpl { std::string grpcChannelArguments; uint32_t filesystemPollWaitMilliseconds = 1000; uint32_t resourcesCleanerPollWaitSeconds = 300; + uint32_t idleUnloadTimeoutSeconds = 0; std::string cacheDir; bool withPython = false; bool startedWithCLI = false; diff --git a/src/cli_parser.cpp b/src/cli_parser.cpp index d7125abc87..38a94900a1 100644 --- a/src/cli_parser.cpp +++ b/src/cli_parser.cpp @@ -137,6 +137,10 @@ std::variant> CLIParser::parse(int argc, char* "Time interval between config and model versions changes detection. Default is 1. Zero or negative value disables changes monitoring.", cxxopts::value()->default_value("1"), "FILE_SYSTEM_POLL_WAIT_SECONDS") + ("idle_unload_timeout_seconds", + "Idle timeout in seconds for model group unloading. When > 0, models not in the 'permanent' group are loaded on demand and unloaded after this idle period. Only effective with config.json multi-model setup. Default is 0 (disabled).", + cxxopts::value()->default_value("0"), + "IDLE_UNLOAD_TIMEOUT_SECONDS") ("custom_node_resources_cleaner_interval_seconds", "Time interval between two consecutive resources cleanup scans. Default is 300. Zero value disables resources cleaner.", cxxopts::value()->default_value("300"), @@ -569,6 +573,7 @@ void CLIParser::prepareServer(ServerSettingsImpl& serverSettings) { serverSettings.filesystemPollWaitMilliseconds = result->operator[]("file_system_poll_wait_seconds").as() * 1000; serverSettings.resourcesCleanerPollWaitSeconds = result->operator[]("custom_node_resources_cleaner_interval_seconds").as(); + serverSettings.idleUnloadTimeoutSeconds = result->operator[]("idle_unload_timeout_seconds").as(); serverSettings.grpcWorkers = result->operator[]("grpc_workers").as(); if (result->count("log_level")) diff --git a/src/config.cpp b/src/config.cpp index cdbd47c6f0..0623b78ad4 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -432,6 +432,7 @@ const std::string& Config::tracePath() const { return this->serverSettings.trace const std::string& Config::grpcChannelArguments() const { return this->serverSettings.grpcChannelArguments; } uint32_t Config::filesystemPollWaitMilliseconds() const { return this->serverSettings.filesystemPollWaitMilliseconds; } uint32_t Config::resourcesCleanerPollWaitSeconds() const { return this->serverSettings.resourcesCleanerPollWaitSeconds; } +uint32_t Config::idleUnloadTimeoutSeconds() const { return this->serverSettings.idleUnloadTimeoutSeconds; } bool Config::allowCredentials() const { return this->serverSettings.allowCredentials; } const std::string& Config::allowedOrigins() const { return this->serverSettings.allowedOrigins; } const std::string& Config::allowedMethods() const { return this->serverSettings.allowedMethods; } diff --git a/src/config.hpp b/src/config.hpp index d710bc4e9a..4f4709c11b 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -312,6 +312,13 @@ class Config { */ uint32_t resourcesCleanerPollWaitSeconds() const; + /** + * @brief Get the idle unload timeout in seconds (0 = disabled) + * + * @return uint32_t + */ + uint32_t idleUnloadTimeoutSeconds() const; + bool allowCredentials() const; const std::string& allowedOrigins() const; const std::string& allowedMethods() const; diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index db544b9130..24680bb0bc 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -73,6 +73,7 @@ #include "mediapipe_internal/mediapipegraphexecutor.hpp" #endif +#include "model_group_manager.hpp" #include "kfs_frontend/kfs_request_utils.hpp" #include "predict_request_validation_utils.hpp" #include "deserialization_main.hpp" @@ -656,6 +657,27 @@ Status HttpRestApiHandler::processListModelsRequest(std::string& response) { for (auto const& graphName : availableMediapipes) { parseModel(writer, graphName, timestamp); } + + // In idle management mode, include unloaded mediapipe graphs from configured groups + auto* groupMgr = modelManager.getGroupManager(); + if (groupMgr && groupMgr->isEnabled()) { + std::set alreadyListed(availableMediapipes.begin(), availableMediapipes.end()); + for (const auto& servableName : groupMgr->getAllConfiguredServableNames()) { + if (alreadyListed.find(servableName) == alreadyListed.end()) { + // Check if it's not already listed as a regular model + bool alreadyAsModel = false; + for (const auto& name : availableModelNames) { + if (name == servableName) { + alreadyAsModel = true; + break; + } + } + if (!alreadyAsModel) { + parseModel(writer, servableName, timestamp); + } + } + } + } #endif writer.EndArray(); writer.String("object"); diff --git a/src/kfs_frontend/kfs_grpc_inference_service.cpp b/src/kfs_frontend/kfs_grpc_inference_service.cpp index e6ebb2bb0d..bc2d26af05 100644 --- a/src/kfs_frontend/kfs_grpc_inference_service.cpp +++ b/src/kfs_frontend/kfs_grpc_inference_service.cpp @@ -45,6 +45,7 @@ #include "../inference_executor.hpp" #include "../modelinstanceunloadguard.hpp" #include "../modelmanager.hpp" +#include "../model_group_manager.hpp" #include "../ovinferrequestsqueue.hpp" #include "../servable_definition.hpp" #include "../servable_definition_unload_guard.hpp" @@ -121,6 +122,16 @@ Status KFSInferenceServiceImpl::getModelReady(const KFSGetModelStatusRequest* re SPDLOG_DEBUG("ModelReady requested name: {}, version: {}", name, versionString); if (model == nullptr) { SPDLOG_DEBUG("ModelReady requested model {} is missing, trying to find definition with such name", name); + // In idle management mode, report configured-but-unloaded models as ready + auto* groupMgr = manager.getGroupManager(); + if (groupMgr && groupMgr->isEnabled()) { + std::string group = groupMgr->getGroupForServable(name); + if (!group.empty()) { + SPDLOG_DEBUG("ModelReady: model {} belongs to group '{}', reporting ready (idle management mode)", name, group); + response->set_ready(true); + return StatusCode::OK; + } + } auto* definition = manager.findServableDefinition(name); if (!definition) { return StatusCode::MODEL_NAME_MISSING; @@ -153,6 +164,15 @@ Status KFSInferenceServiceImpl::getModelReady(const KFSGetModelStatusRequest* re SPDLOG_DEBUG("ModelReady requested model: name {}; default version", name); instance = model->getDefaultModelInstance(); if (instance == nullptr) { + // In idle management mode, report configured models as ready even if no instance is loaded + auto* groupMgr = manager.getGroupManager(); + if (groupMgr && groupMgr->isEnabled()) { + std::string group = groupMgr->getGroupForServable(name); + if (!group.empty()) { + response->set_ready(true); + return StatusCode::OK; + } + } SPDLOG_DEBUG("ModelReady requested model {}; version {} is missing", name, versionString); return Status(StatusCode::MODEL_VERSION_MISSING); } diff --git a/src/mediapipe_internal/mediapipegraphconfig.cpp b/src/mediapipe_internal/mediapipegraphconfig.cpp index 14930f115c..62748784be 100644 --- a/src/mediapipe_internal/mediapipegraphconfig.cpp +++ b/src/mediapipe_internal/mediapipegraphconfig.cpp @@ -128,6 +128,12 @@ Status MediapipeGraphConfig::parseNode(const rapidjson::Value& v) { this->setIdleUnloadTimeoutSeconds(timeoutSeconds); SPDLOG_DEBUG("Mediapipe graph {} idle_unload_timeout_seconds set to {}", this->getGraphName(), timeoutSeconds); } + if (v.HasMember("group_name")) { + this->setGroupName(v["group_name"].GetString()); + } else { + this->setGroupName(this->getGraphName()); + } + SPDLOG_DEBUG("Mediapipe graph {} group_name set to {}", this->getGraphName(), this->getGroupName()); } catch (std::logic_error& e) { SPDLOG_DEBUG("Relative path error: {}", e.what()); return StatusCode::INTERNAL_ERROR; diff --git a/src/mediapipe_internal/mediapipegraphconfig.hpp b/src/mediapipe_internal/mediapipegraphconfig.hpp index cbccd08dc9..c15dc5ed84 100644 --- a/src/mediapipe_internal/mediapipegraphconfig.hpp +++ b/src/mediapipe_internal/mediapipegraphconfig.hpp @@ -73,6 +73,11 @@ class MediapipeGraphConfig { */ int idleUnloadTimeoutSeconds = 0; + /** + * @brief Group name for idle model management. Defaults to graph name. + */ + std::string groupName; + public: MediapipeGraphConfig(const std::string& graphName = "", const std::string& basePath = "", @@ -186,6 +191,14 @@ class MediapipeGraphConfig { this->idleUnloadTimeoutSeconds = seconds; } + const std::string& getGroupName() const { + return this->groupName; + } + + void setGroupName(const std::string& groupName) { + this->groupName = groupName; + } + bool isReloadRequired(const MediapipeGraphConfig& rhs) const; /** diff --git a/src/model_group_manager.cpp b/src/model_group_manager.cpp new file mode 100644 index 0000000000..bd20d636ff --- /dev/null +++ b/src/model_group_manager.cpp @@ -0,0 +1,396 @@ +//***************************************************************************** +// Copyright 2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "model_group_manager.hpp" + +#include +#include + +#include "logging.hpp" +#include "model.hpp" +#include "modelconfig.hpp" +#include "modelinstance.hpp" +#include "modelmanager.hpp" +#if (MEDIAPIPE_DISABLE == 0) +#include "mediapipe_internal/mediapipefactory.hpp" +#include "mediapipe_internal/mediapipegraphdefinition.hpp" +#endif + +namespace ovms { + +ModelGroupManager::ModelGroupManager(uint32_t idleTimeoutSeconds) : + idleTimeoutSeconds_(idleTimeoutSeconds), + lastActivityTimeNs_(std::make_shared>( + std::chrono::steady_clock::now().time_since_epoch().count())) { +} + +void ModelGroupManager::buildGroups(const std::unordered_map& modelConfigs, + ModelManager& mm) { + std::unique_lock lock(groupsMtx_); + groups_.clear(); + servableToGroup_.clear(); + + for (const auto& [name, config] : modelConfigs) { + const std::string& groupName = config.getGroupName(); + groups_[groupName].groupName = groupName; + groups_[groupName].modelNames.insert(name); + servableToGroup_[name] = groupName; + } + +#if (MEDIAPIPE_DISABLE == 0) + // Also process mediapipe graph definitions + for (const auto& graphName : mm.getMediapipeFactory().getMediapipePipelinesNames()) { + MediapipeGraphDefinition* def = mm.getMediapipeFactory().findDefinitionByName(graphName); + if (def == nullptr) { + continue; + } + const std::string& groupName = def->getMediapipeGraphConfig().getGroupName(); + if (groupName.empty()) { + // No group_name set — treat graph name as its own group + groups_[graphName].groupName = graphName; + groups_[graphName].mediapipeNames.insert(graphName); + servableToGroup_[graphName] = graphName; + } else { + groups_[groupName].groupName = groupName; + groups_[groupName].mediapipeNames.insert(graphName); + servableToGroup_[graphName] = groupName; + } + } +#endif + + size_t totalServables = modelConfigs.size(); +#if (MEDIAPIPE_DISABLE == 0) + totalServables += mm.getMediapipeFactory().getMediapipePipelinesNames().size(); +#endif + SPDLOG_INFO("Model group manager built {} groups from {} servables", groups_.size(), totalServables); + for (const auto& [gname, ginfo] : groups_) { + SPDLOG_INFO(" Group '{}': {} models, {} mediapipe graphs{}", + gname, ginfo.modelNames.size(), ginfo.mediapipeNames.size(), + ginfo.isPermanent() ? " (permanent)" : ""); + } +} + +void ModelGroupManager::buildGroups(const std::unordered_map& modelConfigs) { + std::unique_lock lock(groupsMtx_); + groups_.clear(); + servableToGroup_.clear(); + + for (const auto& [name, config] : modelConfigs) { + const std::string& groupName = config.getGroupName(); + groups_[groupName].groupName = groupName; + groups_[groupName].modelNames.insert(name); + servableToGroup_[name] = groupName; + } + + SPDLOG_INFO("Model group manager built {} groups from {} models", groups_.size(), modelConfigs.size()); + for (const auto& [gname, ginfo] : groups_) { + SPDLOG_INFO(" Group '{}': {} models, {} mediapipe graphs{}", + gname, ginfo.modelNames.size(), ginfo.mediapipeNames.size(), + ginfo.isPermanent() ? " (permanent)" : ""); + } +} + +std::string ModelGroupManager::getGroupForServable(const std::string& servableName) const { + std::shared_lock lock(groupsMtx_); + auto it = servableToGroup_.find(servableName); + if (it != servableToGroup_.end()) { + return it->second; + } + return ""; +} + +bool ModelGroupManager::isGroupLoaded(const std::string& groupName) const { + if (groupName.empty()) { + return false; + } + std::shared_lock lock(groupsMtx_); + auto it = groups_.find(groupName); + if (it != groups_.end() && it->second.isPermanent()) { + return true; + } + return activeGroupName_ == groupName; +} + +const std::string& ModelGroupManager::getActiveGroupName() const { + return activeGroupName_; +} + +void ModelGroupManager::recordActivity() { + lastActivityTimeNs_->store( + std::chrono::steady_clock::now().time_since_epoch().count(), + std::memory_order_relaxed); +} + +std::vector ModelGroupManager::getAllConfiguredServableNames() const { + std::shared_lock lock(groupsMtx_); + std::vector names; + for (const auto& [servableName, groupName] : servableToGroup_) { + names.push_back(servableName); + } + return names; +} + +bool ModelGroupManager::canUnloadActiveGroup(ModelManager& mm) const { + std::shared_lock lock(groupsMtx_); + auto it = groups_.find(activeGroupName_); + if (it == groups_.end()) { + return true; + } + const auto& groupInfo = it->second; + + // Check all classic models in the group + for (const auto& modelName : groupInfo.modelNames) { + auto model = mm.findModelByName(modelName); + if (model == nullptr) { + continue; + } + for (const auto& [version, instance] : model->getModelVersions()) { + if (!instance->canUnloadInstance()) { + SPDLOG_DEBUG("Cannot unload group '{}': model {} version {} has active requests", + activeGroupName_, modelName, version); + return false; + } + if (instance->getStatus().getState() == ModelVersionState::LOADING) { + SPDLOG_DEBUG("Cannot unload group '{}': model {} version {} is loading", + activeGroupName_, modelName, version); + return false; + } + } + } + +#if (MEDIAPIPE_DISABLE == 0) + // Check all mediapipe graphs in the group + for (const auto& graphName : groupInfo.mediapipeNames) { + MediapipeGraphDefinition* def = mm.getMediapipeFactory().findDefinitionByName(graphName); + if (def == nullptr) { + continue; + } + auto activeCount = def->getActiveInferenceCount(); + if (activeCount && activeCount->load(std::memory_order_acquire) > 0) { + SPDLOG_DEBUG("Cannot unload group '{}': mediapipe graph {} has active inferences", + activeGroupName_, graphName); + return false; + } + } +#endif + + return true; +} + +Status ModelGroupManager::loadGroup(const std::string& groupName, ModelManager& mm) { + SPDLOG_INFO("Loading model group '{}'", groupName); + + std::shared_lock lock(groupsMtx_); + auto it = groups_.find(groupName); + if (it == groups_.end()) { + SPDLOG_ERROR("Model group '{}' not found", groupName); + return StatusCode::GROUP_LOAD_FAILED; + } + const auto& groupInfo = it->second; + lock.unlock(); + + Status firstError = StatusCode::OK; + + // Load classic models + for (const auto& modelName : groupInfo.modelNames) { + auto configIt = mm.getServedModelConfigs().find(modelName); + if (configIt == mm.getServedModelConfigs().end()) { + SPDLOG_WARN("Model config for '{}' not found during group load", modelName); + continue; + } + ModelConfig config = configIt->second; + auto status = mm.reloadModelWithVersions(config); + if (!status.ok()) { + SPDLOG_ERROR("Failed to load model '{}' in group '{}': {}", modelName, groupName, status.string()); + if (firstError.ok()) { + firstError = status; + } + } else { + SPDLOG_INFO("Loaded model '{}' in group '{}'", modelName, groupName); + } + } + +#if (MEDIAPIPE_DISABLE == 0) + // Wake up mediapipe graphs in this group + for (const auto& graphName : groupInfo.mediapipeNames) { + MediapipeGraphDefinition* def = mm.getMediapipeFactory().findDefinitionByName(graphName); + if (def == nullptr) { + SPDLOG_WARN("Mediapipe graph '{}' not found during group load", graphName); + continue; + } + auto status = def->wakeUpIfUnloaded(mm); + if (!status.ok()) { + SPDLOG_ERROR("Failed to wake mediapipe graph '{}' in group '{}': {}", graphName, groupName, status.string()); + if (firstError.ok()) { + firstError = status; + } + } else { + SPDLOG_INFO("Woke mediapipe graph '{}' in group '{}'", graphName, groupName); + } + } +#endif + + activeGroupName_ = groupName; + recordActivity(); + + if (!firstError.ok()) { + return StatusCode::GROUP_LOAD_FAILED; + } + SPDLOG_INFO("Model group '{}' loaded successfully", groupName); + return StatusCode::OK; +} + +Status ModelGroupManager::unloadGroup(const std::string& groupName, ModelManager& mm) { + SPDLOG_INFO("Unloading model group '{}'", groupName); + + std::shared_lock lock(groupsMtx_); + auto it = groups_.find(groupName); + if (it == groups_.end()) { + return StatusCode::OK; + } + const auto& groupInfo = it->second; + lock.unlock(); + + // Retire classic models + for (const auto& modelName : groupInfo.modelNames) { + auto model = mm.findModelByName(modelName); + if (model == nullptr) { + continue; + } + model->retireAllVersions(); + SPDLOG_INFO("Retired model '{}' in group '{}'", modelName, groupName); + } + +#if (MEDIAPIPE_DISABLE == 0) + // Unload mediapipe graphs (AVAILABLE -> UNLOADED, frees resources but keeps definition) + for (const auto& graphName : groupInfo.mediapipeNames) { + MediapipeGraphDefinition* def = mm.getMediapipeFactory().findDefinitionByName(graphName); + if (def != nullptr) { + def->unload(); + SPDLOG_INFO("Unloaded mediapipe graph '{}' in group '{}'", graphName, groupName); + } + } +#endif + + if (activeGroupName_ == groupName) { + activeGroupName_.clear(); + } + SPDLOG_INFO("Model group '{}' unloaded successfully", groupName); + return StatusCode::OK; +} + +Status ModelGroupManager::ensureGroupLoaded(const std::string& servableName, ModelManager& mm) { + std::string groupName = getGroupForServable(servableName); + if (groupName.empty()) { + // Not managed by group manager — let normal flow handle it + return StatusCode::OK; + } + + // Permanent group is always loaded + { + std::shared_lock lock(groupsMtx_); + auto it = groups_.find(groupName); + if (it != groups_.end() && it->second.isPermanent()) { + recordActivity(); + return StatusCode::OK; + } + } + + // Already the active group + if (activeGroupName_ == groupName) { + recordActivity(); + return StatusCode::OK; + } + + // Serialize group swaps + std::lock_guard swapLock(loadUnloadMtx_); + + // Double-check after acquiring the lock + if (activeGroupName_ == groupName) { + recordActivity(); + return StatusCode::OK; + } + + // Unload the currently active group if any + if (!activeGroupName_.empty()) { + SPDLOG_INFO("Swapping model group from '{}' to '{}'", activeGroupName_, groupName); + // Wait for active requests to drain with bounded retry + constexpr int kMaxRetries = 300; // 30 seconds at 100ms intervals + constexpr int kRetryIntervalMs = 100; + for (int i = 0; i < kMaxRetries; ++i) { + if (canUnloadActiveGroup(mm)) { + break; + } + if (i == kMaxRetries - 1) { + SPDLOG_ERROR("Timed out waiting for group '{}' to drain requests before swap to '{}'", + activeGroupName_, groupName); + return StatusCode::GROUP_UNLOAD_BLOCKED; + } + std::this_thread::sleep_for(std::chrono::milliseconds(kRetryIntervalMs)); + } + auto unloadStatus = unloadGroup(activeGroupName_, mm); + if (!unloadStatus.ok()) { + SPDLOG_ERROR("Failed to unload group '{}': {}", activeGroupName_, unloadStatus.string()); + return unloadStatus; + } + } + + return loadGroup(groupName, mm); +} + +void ModelGroupManager::unloadActiveGroupIfIdle(ModelManager& mm) { + if (!isEnabled()) { + return; + } + if (activeGroupName_.empty()) { + return; + } + + // Check if we have a permanent group as active (should not happen, but safety) + { + std::shared_lock lock(groupsMtx_); + auto it = groups_.find(activeGroupName_); + if (it != groups_.end() && it->second.isPermanent()) { + return; + } + } + + // Check idle timeout + int64_t lastActivity = lastActivityTimeNs_->load(std::memory_order_relaxed); + int64_t nowNs = std::chrono::steady_clock::now().time_since_epoch().count(); + int64_t timeoutNs = static_cast(idleTimeoutSeconds_) * 1'000'000'000LL; + if ((nowNs - lastActivity) < timeoutNs) { + return; + } + + // Check if we can safely unload (no active requests) + if (!canUnloadActiveGroup(mm)) { + SPDLOG_DEBUG("Skipping idle unload of group '{}': active requests in flight", activeGroupName_); + return; + } + + SPDLOG_INFO("Idle unloading model group '{}' after {}s timeout", activeGroupName_, idleTimeoutSeconds_); + std::lock_guard swapLock(loadUnloadMtx_); + // Re-check after acquiring lock + if (activeGroupName_.empty()) { + return; + } + if (!canUnloadActiveGroup(mm)) { + return; + } + unloadGroup(activeGroupName_, mm); +} + +} // namespace ovms diff --git a/src/model_group_manager.hpp b/src/model_group_manager.hpp new file mode 100644 index 0000000000..8eb657f372 --- /dev/null +++ b/src/model_group_manager.hpp @@ -0,0 +1,86 @@ +//***************************************************************************** +// Copyright 2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "status.hpp" + +namespace ovms { + +class ModelConfig; +class ModelManager; + +struct ModelGroupInfo { + std::string groupName; + std::set modelNames; + std::set mediapipeNames; + bool isPermanent() const { return groupName == "permanent"; } +}; + +class ModelGroupManager { +public: + explicit ModelGroupManager(uint32_t idleTimeoutSeconds); + + bool isEnabled() const { return idleTimeoutSeconds_ > 0; } + uint32_t getIdleTimeoutSeconds() const { return idleTimeoutSeconds_; } + + void buildGroups(const std::unordered_map& modelConfigs, + ModelManager& mm); + // Overload for testing without ModelManager (processes only classic models) + void buildGroups(const std::unordered_map& modelConfigs); + + std::string getGroupForServable(const std::string& servableName) const; + + bool isGroupLoaded(const std::string& groupName) const; + + Status ensureGroupLoaded(const std::string& servableName, ModelManager& mm); + + void unloadActiveGroupIfIdle(ModelManager& mm); + + void recordActivity(); + + const std::string& getActiveGroupName() const; + + const std::unordered_map& getGroups() const { return groups_; } + + std::vector getAllConfiguredServableNames() const; + +private: + bool canUnloadActiveGroup(ModelManager& mm) const; + Status loadGroup(const std::string& groupName, ModelManager& mm); + Status unloadGroup(const std::string& groupName, ModelManager& mm); + + uint32_t idleTimeoutSeconds_; + + mutable std::shared_mutex groupsMtx_; + std::unordered_map groups_; + std::unordered_map servableToGroup_; + + mutable std::mutex loadUnloadMtx_; + std::string activeGroupName_; + + std::shared_ptr> lastActivityTimeNs_; +}; + +} // namespace ovms diff --git a/src/modelconfig.cpp b/src/modelconfig.cpp index c02d22c199..8a0aed41ce 100644 --- a/src/modelconfig.cpp +++ b/src/modelconfig.cpp @@ -115,6 +115,10 @@ bool ModelConfig::isReloadRequired(const ModelConfig& rhs) const { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to plugin config mismatch", this->name); return true; } + if (this->groupName != rhs.groupName) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to group name mismatch", this->name); + return true; + } if (!isLayoutConfigurationEqual(rhs)) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to named layout mismatch", this->name); return true; @@ -720,6 +724,14 @@ Status ModelConfig::parseNode(const rapidjson::Value& v) { SPDLOG_DEBUG("allow_cache: {}", v["allow_cache"].GetBool()); } + // Group name for idle model management + if (v.HasMember("group_name")) { + setGroupName(v["group_name"].GetString()); + } else { + setGroupName(getName()); + } + SPDLOG_DEBUG("group_name: {}", getGroupName()); + // if the config has models which require custom loader to be used, then load the same here if (v.HasMember("custom_loader_options")) { if (!parseCustomLoaderOptionsConfig(v["custom_loader_options"]).ok()) { diff --git a/src/modelconfig.hpp b/src/modelconfig.hpp index 353715f651..055750835c 100644 --- a/src/modelconfig.hpp +++ b/src/modelconfig.hpp @@ -199,6 +199,11 @@ class ModelConfig { */ std::optional precision; + /** + * @brief Group name for idle model management. Defaults to model name. + */ + std::string groupName; + public: /** * @brief Construct a new Model Config object @@ -281,6 +286,24 @@ class ModelConfig { this->name = name; } + /** + * @brief Get the group name + * + * @return const std::string& + */ + const std::string& getGroupName() const { + return this->groupName; + } + + /** + * @brief Set the group name + * + * @param groupName + */ + void setGroupName(const std::string& groupName) { + this->groupName = groupName; + } + /** * @brief Get local path to specific model version where .xml and .bin is located for loading * diff --git a/src/modelmanager.cpp b/src/modelmanager.cpp index 75a72e661b..fa396ac2f1 100644 --- a/src/modelmanager.cpp +++ b/src/modelmanager.cpp @@ -58,6 +58,7 @@ #include "filesystem/filesystemfactory.hpp" #include "graph_export/graph_export.hpp" #include "logging.hpp" +#include "model_group_manager.hpp" #if (MEDIAPIPE_DISABLE == 0) #include "mediapipe_internal/mediapipefactory.hpp" #include "mediapipe_internal/mediapipegraphdefinition.hpp" @@ -187,6 +188,13 @@ Status ModelManager::start(const Config& config) { resourcesCleanupIntervalMillisec = config.resourcesCleanerPollWaitSeconds() * 1000; Status status; this->startedWithConfigFile = (config.configPath() != ""); + + // Initialize model group manager if idle unload is enabled and using config file + if (this->startedWithConfigFile && config.idleUnloadTimeoutSeconds() > 0) { + groupManager_ = std::make_unique(config.idleUnloadTimeoutSeconds()); + SPDLOG_INFO("Model group idle management enabled with {}s timeout", config.idleUnloadTimeoutSeconds()); + } + if (isStartedWithConfigFile()) { status = startFromFile(config.configPath()); } else { @@ -956,6 +964,33 @@ Status ModelManager::loadConfig() { IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); } + // Build model groups and unload non-permanent servables for on-demand loading + if (groupManager_ && groupManager_->isEnabled()) { + groupManager_->buildGroups(this->servedModelConfigs, *this); + // Unload all non-permanent servables so they are loaded on demand + for (const auto& [groupName, groupInfo] : groupManager_->getGroups()) { + if (groupInfo.isPermanent()) { + continue; + } + for (const auto& modelName : groupInfo.modelNames) { + auto model = findModelByName(modelName); + if (model != nullptr) { + model->retireAllVersions(); + SPDLOG_INFO("Retired model '{}' (group '{}') for on-demand loading", modelName, groupName); + } + } +#if (MEDIAPIPE_DISABLE == 0) + for (const auto& graphName : groupInfo.mediapipeNames) { + MediapipeGraphDefinition* def = mediapipeFactory->findDefinitionByName(graphName); + if (def != nullptr) { + def->unload(); + SPDLOG_INFO("Unloaded mediapipe graph '{}' (group '{}') for on-demand loading", graphName, groupName); + } + } +#endif + } + } + this->lastLoadConfigStatus = firstErrorStatus; return firstErrorStatus; } @@ -1095,6 +1130,10 @@ void ModelManager::watcher(std::future exitSignal, bool watchConfigFile) { // non-blocking (it skips graphs with in-flight requests rather than // draining). This keeps configMtx hold time minimal. unloadIdleGraphs(); + // Model group idle unload: unload the active non-permanent group if idle + if (groupManager_ && groupManager_->isEnabled()) { + groupManager_->unloadActiveGroupIfIdle(*this); + } SPDLOG_LOGGER_TRACE(modelmanager_logger, "Models configuration and filesystem check cycle end"); } SPDLOG_LOGGER_INFO(modelmanager_logger, "Stopped model manager thread"); @@ -1587,6 +1626,16 @@ Status ModelManager::getModelInstance(const std::string& modelName, std::unique_ptr& modelInstanceUnloadGuardPtr) const { SPDLOG_DEBUG("Requesting model: {}; version: {}.", modelName, modelVersionId); + // On-demand group loading for idle model management + if (groupManager_ && groupManager_->isEnabled()) { + auto status = groupManager_->ensureGroupLoaded(modelName, const_cast(*this)); + if (!status.ok()) { + SPDLOG_ERROR("Failed to load group for model '{}': {}", modelName, status.string()); + return status; + } + groupManager_->recordActivity(); + } + auto model = findModelByName(modelName); if (model == nullptr) { return StatusCode::MODEL_NAME_MISSING; @@ -1611,6 +1660,10 @@ const CustomNodeLibraryManager& ModelManager::getCustomNodeLibraryManager() cons } const std::vector ModelManager::getNamesOfAvailableModels() const { + // In idle management mode, report all configured models as available + if (groupManager_ && groupManager_->isEnabled()) { + return groupManager_->getAllConfiguredServableNames(); + } std::vector names; std::shared_lock lock(modelsMtx); for (auto& [name, model] : models) { @@ -1624,6 +1677,16 @@ const std::vector ModelManager::getNamesOfAvailableModels() const { Status ModelManager::createPipeline(std::unique_ptr& graph, const std::string& name) { #if (MEDIAPIPE_DISABLE == 0) + // On-demand group loading for idle model management + if (groupManager_ && groupManager_->isEnabled()) { + auto status = groupManager_->ensureGroupLoaded(name, *this); + if (!status.ok()) { + SPDLOG_ERROR("Failed to load group for mediapipe graph '{}': {}", name, status.string()); + return status; + } + groupManager_->recordActivity(); + } + // Lazy wake-up with bounded retry. A request can observe state==AVAILABLE here, // then have the watcher flip it to UNLOADED before create()->waitForLoaded() runs, // which returns MEDIAPIPE_DEFINITION_NOT_LOADED_YET. We retry a bounded number of diff --git a/src/schema.cpp b/src/schema.cpp index 0c10a0da93..23da14985f 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -241,6 +241,9 @@ const std::string MODEL_CONFIG_DEFINITION = R"( } }, "minProperties": 1 + }, + "group_name": { + "type": "string" } }, "additionalProperties": false @@ -361,6 +364,9 @@ const std::string MODELS_CONFIG_SCHEMA = R"({ "idle_unload_timeout_seconds": { "type": "integer", "minimum": 0 + }, + "group_name": { + "type": "string" } }, "additionalProperties": false diff --git a/src/status.cpp b/src/status.cpp index 0394192e86..8b2880fe6c 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -346,5 +346,8 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::DEVICE_WRONG_FORMAT, "Device is in wrong format"}, {StatusCode::SHAPE_DYNAMIC_BUT_NPU_USED, "Shape is dynamic but NPU is used"}, {StatusCode::STATIC_RESOLUTION_MISUSE, "Wrong usage of static resolution"}, + + {StatusCode::GROUP_LOAD_FAILED, "Model group failed to load"}, + {StatusCode::GROUP_UNLOAD_BLOCKED, "Cannot unload model group due to active requests"}, }; } // namespace ovms diff --git a/src/status.hpp b/src/status.hpp index 94be7948cb..6c935a9bff 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -359,6 +359,10 @@ enum class StatusCode { SHAPE_DYNAMIC_BUT_NPU_USED, STATIC_RESOLUTION_MISUSE, + // Model Group Management + GROUP_LOAD_FAILED, + GROUP_UNLOAD_BLOCKED, + STATUS_CODE_END }; diff --git a/src/test/model_group_manager_test.cpp b/src/test/model_group_manager_test.cpp new file mode 100644 index 0000000000..8c10e71cea --- /dev/null +++ b/src/test/model_group_manager_test.cpp @@ -0,0 +1,182 @@ +//***************************************************************************** +// Copyright 2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include + +#include +#include + +#include "../model_group_manager.hpp" +#include "../modelconfig.hpp" +#include "../status.hpp" + +using namespace ovms; + +class ModelGroupManagerTest : public ::testing::Test { +protected: + std::unordered_map createModelConfigs( + const std::vector>& nameGroupPairs) { + std::unordered_map configs; + for (const auto& [name, group] : nameGroupPairs) { + ModelConfig config; + config.setName(name); + config.setGroupName(group); + configs.emplace(name, std::move(config)); + } + return configs; + } +}; + +TEST_F(ModelGroupManagerTest, DisabledByDefault) { + ModelGroupManager mgr(0); + ASSERT_FALSE(mgr.isEnabled()); +} + +TEST_F(ModelGroupManagerTest, EnabledWithPositiveTimeout) { + ModelGroupManager mgr(30); + ASSERT_TRUE(mgr.isEnabled()); + ASSERT_EQ(mgr.getIdleTimeoutSeconds(), 30u); +} + +TEST_F(ModelGroupManagerTest, BuildGroups_DefaultGroupNames) { + ModelGroupManager mgr(30); + auto configs = createModelConfigs({ + {"model_a", "model_a"}, + {"model_b", "model_b"}, + {"model_c", "model_c"}, + }); + mgr.buildGroups(configs); + const auto& groups = mgr.getGroups(); + ASSERT_EQ(groups.size(), 3u); + EXPECT_TRUE(groups.count("model_a")); + EXPECT_TRUE(groups.count("model_b")); + EXPECT_TRUE(groups.count("model_c")); +} + +TEST_F(ModelGroupManagerTest, BuildGroups_ExplicitGroupNames) { + ModelGroupManager mgr(30); + auto configs = createModelConfigs({ + {"model_a", "rag"}, + {"model_b", "rag"}, + {"model_c", "rag"}, + }); + mgr.buildGroups(configs); + const auto& groups = mgr.getGroups(); + ASSERT_EQ(groups.size(), 1u); + EXPECT_TRUE(groups.count("rag")); + EXPECT_EQ(groups.at("rag").modelNames.size(), 3u); +} + +TEST_F(ModelGroupManagerTest, BuildGroups_PermanentGroup) { + ModelGroupManager mgr(30); + auto configs = createModelConfigs({ + {"model_a", "permanent"}, + {"model_b", "permanent"}, + {"model_c", "rag"}, + }); + mgr.buildGroups(configs); + const auto& groups = mgr.getGroups(); + ASSERT_EQ(groups.size(), 2u); + EXPECT_TRUE(groups.at("permanent").isPermanent()); + EXPECT_FALSE(groups.at("rag").isPermanent()); + EXPECT_EQ(groups.at("permanent").modelNames.size(), 2u); +} + +TEST_F(ModelGroupManagerTest, BuildGroups_MixedGroups) { + ModelGroupManager mgr(30); + auto configs = createModelConfigs({ + {"model_a", "rag"}, + {"model_b", "rag"}, + {"model_c", "audio"}, + {"model_d", "audio"}, + {"model_e", "permanent"}, + }); + mgr.buildGroups(configs); + const auto& groups = mgr.getGroups(); + ASSERT_EQ(groups.size(), 3u); + EXPECT_EQ(groups.at("rag").modelNames.size(), 2u); + EXPECT_EQ(groups.at("audio").modelNames.size(), 2u); + EXPECT_EQ(groups.at("permanent").modelNames.size(), 1u); +} + +TEST_F(ModelGroupManagerTest, GetGroupForServable) { + ModelGroupManager mgr(30); + auto configs = createModelConfigs({ + {"model_a", "rag"}, + {"model_b", "audio"}, + }); + mgr.buildGroups(configs); + EXPECT_EQ(mgr.getGroupForServable("model_a"), "rag"); + EXPECT_EQ(mgr.getGroupForServable("model_b"), "audio"); + EXPECT_EQ(mgr.getGroupForServable("nonexistent"), ""); +} + +TEST_F(ModelGroupManagerTest, IsGroupLoaded_PermanentAlwaysTrue) { + ModelGroupManager mgr(30); + auto configs = createModelConfigs({ + {"model_a", "permanent"}, + {"model_b", "rag"}, + }); + mgr.buildGroups(configs); + EXPECT_TRUE(mgr.isGroupLoaded("permanent")); + EXPECT_FALSE(mgr.isGroupLoaded("rag")); + EXPECT_FALSE(mgr.isGroupLoaded("nonexistent")); +} + +TEST_F(ModelGroupManagerTest, GetAllConfiguredServableNames) { + ModelGroupManager mgr(30); + auto configs = createModelConfigs({ + {"model_a", "rag"}, + {"model_b", "rag"}, + {"model_c", "permanent"}, + }); + mgr.buildGroups(configs); + auto names = mgr.getAllConfiguredServableNames(); + ASSERT_EQ(names.size(), 3u); + std::set nameSet(names.begin(), names.end()); + EXPECT_TRUE(nameSet.count("model_a")); + EXPECT_TRUE(nameSet.count("model_b")); + EXPECT_TRUE(nameSet.count("model_c")); +} + +TEST_F(ModelGroupManagerTest, RecordActivityUpdatesTimestamp) { + ModelGroupManager mgr(30); + auto configs = createModelConfigs({{"model_a", "rag"}}); + mgr.buildGroups(configs); + + // Record activity and verify no crash + mgr.recordActivity(); +} + +TEST_F(ModelGroupManagerTest, ActiveGroupNameInitiallyEmpty) { + ModelGroupManager mgr(30); + EXPECT_TRUE(mgr.getActiveGroupName().empty()); +} + +// Schema validation test for group_name in model config +TEST(SchemaValidation, GroupNameInModelConfig) { + ModelConfig config; + config.setName("test_model"); + config.setGroupName("my_group"); + EXPECT_EQ(config.getGroupName(), "my_group"); + + // Default group name should be model name + ModelConfig config2; + config2.setName("test_model_2"); + config2.setGroupName(config2.getName()); + EXPECT_EQ(config2.getGroupName(), "test_model_2"); +} diff --git a/src/test/schema_test.cpp b/src/test/schema_test.cpp index fad87515f2..b56cb7fb00 100644 --- a/src/test/schema_test.cpp +++ b/src/test/schema_test.cpp @@ -2063,9 +2063,88 @@ TEST(SchemaTest, MediapipeConfigIdleUnloadTimeoutWrongTypeRejected) { auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); } -#endif -TEST(SchemaTest, MediapipeConfigNegativeAdditionalMediapipeConfigField) { +TEST(SchemaTest, ModelConfigGroupNameValidString) { + const char* config = R"( + { + "model_config_list": [ + { + "config": { + "name": "dummy_model", + "base_path": "dummy_path", + "group_name": "rag" + } + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(config); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::OK); +} + +TEST(SchemaTest, ModelConfigGroupNameInvalidType) { + const char* config = R"( + { + "model_config_list": [ + { + "config": { + "name": "dummy_model", + "base_path": "dummy_path", + "group_name": 123 + } + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(config); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} + +#if (MEDIAPIPE_DISABLE == 0) +TEST(SchemaTest, MediapipeConfigGroupNameValidString) { + const char* config = R"( + { + "model_config_list": [], + "mediapipe_config_list": [ + { + "name": "dummy_graph", + "graph_path": "graph.pbtxt", + "base_path": "dummy_path", + "group_name": "llm_group" + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(config); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::OK); +} + +TEST(SchemaTest, MediapipeConfigGroupNameInvalidType) { + const char* config = R"( + { + "model_config_list": [], + "mediapipe_config_list": [ + { + "name": "dummy_graph", + "graph_path": "graph.pbtxt", + "base_path": "dummy_path", + "group_name": 42 + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(config); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} +#endif const char* mediapipeConfigNegative = R"( { "model_config_list": [], diff --git a/src/test/template_test.cpp b/src/test/template_test.cpp new file mode 100644 index 0000000000..648110754b --- /dev/null +++ b/src/test/template_test.cpp @@ -0,0 +1,104 @@ +#include +#include +#include +#include + +int main() { + using chat_t = std::vector>; + + chat_t chat1{{{"role", "user"}, {"content", "hello"}}}; + chat_t chat2{{{"role", "system"}, {"content", "You are assistant."}, {"role", "user"}, {"content", "hello"}}}; + chat_t chat3{{{"role", "system"}, {"content", "You are assistant."}, {"role", "user"}, {"content", "hello"}, + {"role", "assistant"}, {"content", "how can I help you?"}, {"role", "user"}, {"content", "how much is 2+2?"}}}; + /* +curl http://ov-spr-19.sclab.intel.com:8000/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"meta-llama/Llama-2-7b-chat-hf","messages":[{"role":"user","content":"hello"}], "max_tokens":30}' +curl http://ov-spr-19.sclab.intel.com:8000/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"meta-llama/Llama-2-7b-chat-hf","messages":[{"role":"system","content":"You are assistant."},{"role":"user","content":"hello"}], "max_tokens":30}' +curl http://ov-spr-19.sclab.intel.com:8000/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"meta-llama/Llama-2-7b-chat-hf","messages":[{"role":"system","content":"You are assistant."},{"role":"user","content":"hello"},{"role":"assistant","content":"how can I help you?"}, {"role":"user","content":"how much is 2+2?"}], "max_tokens":30}' +*/ + + using expected_prompt = std::string; + using expected_tokens = std::vector; + std::unordered_map> test_prompts; + std::unordered_map> test_tokens; + + //////// + std::string model_name{"meta-llama/Llama-2-7b-chat-hf"}; + std::vector prompts{ + "[INST] hello [/INST]", + "[INST] <>\nYou are assistant.\n<>\n\nhello [/INST]", + "[INST] <>\nYou are assistant.\n<>\n\nhello [/INST] how can I help you? [INST] how much is 2+2? [/INST]"}; + std::vector tokens{ + {1, 518, 25580, 29962, 22172, 518, 29914, 25580, 29962}, + {1, 518, 25580, 29962, 3532, 14816, 29903, 6778, 13, 3492, 526, 20255, 29889, 13, 29966, 829, 14816, 29903, 6778, 13, 13, 12199, 518, 29914, 25580, 29962}, + {1, 518, 25580, 29962, 3532, 14816, 29903, 6778, 13, 3492, 526, 20255, 29889, 13, 29966, 829, 14816, 29903, 6778, 13, 13, 12199, 518, 29914, 25580, 29962, 920, 508, 306, 1371, 366, 29973, 29871, 2, 1, 518, 25580, 29962, 920, 1568, 338, 29871, 29906, 29974, 29906, 29973, 518, 29914, 25580, 29962}}; + test_prompts.insert({model_name, prompts}); + test_tokens.insert({model_name, tokens}); + + //////// + std::string model_name{"meta-llama/Meta-Llama-3-8B-Instruct"}; + std::vector prompts{ + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\nhello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nhello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nhello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nhow can I help you?<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nhow much is 2+2?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"}; + std::vector tokens{ + {128000, 128006, 882, 128007, 271, 15339, 128009, 128006, 78191, 128007, 271}, + {128000, 128006, 9125, 128007, 271, 2675, 527, 18328, 13, 128009, 128006, 882, 128007, 271, 15339, 128009, 128006, 78191, 128007, 271}, + {128000, 128006, 9125, 128007, 271, 2675, 527, 18328, 13, 128009, 128006, 882, 128007, 271, 15339, 128009, 128006, 78191, 128007, 271, 5269, 649, 358, 1520, 499, 30, 128009, 128006, 882, 128007, 271, 5269, 1790, 374, 220, 17, 10, 17, 30, 128009, 128006, 78191, 128007, 271}}; + test_prompts.insert({model_name, prompts}); + test_tokens.insert({model_name, tokens}); + + //////// + std::string model_name{"TinyLlama/TinyLlama-1.1B-Chat-v0.6"}; + std::vector prompts{ + "<|user|>\nhello\n<|assistant|>\n", + "<|system|>\nYou are assistant.\n<|user|>\nhello\n<|assistant|>\n", + "<|system|>\nYou are assistant.\n<|user|>\nhello\n<|assistant|>\nhow can I help you?\n<|user|>\nhow much is 2+2?\n<|assistant|>\n"}; + std::vector tokens{ + {529, 29989, 1792, 29989, 29958, 13, 12199, 2, 29871, 13, 29966, 29989, 465, 22137, 29989, 29958, 13}, + {529, 29989, 5205, 29989, 29958, 13, 3492, 526, 20255, 29889, 2, 29871, 13, 29966, 29989, 1792, 29989, 29958, 13, 12199, 2, 29871, 13, 29966, 29989, 465, 22137, 29989, 29958, 13}, + {529, 29989, 5205, 29989, 29958, 13, 3492, 526, 20255, 29889, 2, 29871, 13, 29966, 29989, 1792, 29989, 29958, 13, 12199, 2, 29871, 13, 29966, 29989, 465, 22137, 29989, 29958, 13, 3525, 508, 306, 1371, 366, 29973, 2, 29871, 13, 29966, 29989, 1792, 29989, 29958, 13, 3525, 1568, 338, 29871, 29906, 29974, 29906, 29973, 2, 29871, 13, 29966, 29989, 465, 22137, 29989, 29958, 13}}; + test_prompts.insert({model_name, prompts}); + test_tokens.insert({model_name, tokens}); + + //////// + std::string model_name{"Qwen/Qwen-7B-Chat"}; + std::vector prompts{ + "<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n", + "<|im_start|>system\nYou are assistant.<|im_end|>\n<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n", + "<|im_start|>system\nYou are assistant.<|im_end|>\n<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\nhow can I help you?<|im_end|>\n<|im_start|>user\nhow much is 2+2?<|im_end|>\n<|im_start|>assistant\n"}; + std::vector tokens{ + {151644, 872, 198, 14990, 151645, 198, 151644, 77091, 198}, + {151644, 8948, 198, 2610, 525, 17847, 13, 151645, 198, 151644, 872, 198, 14990, 151645, 198, 151644, 77091, 198}, + {151644, 8948, 198, 2610, 525, 17847, 13, 151645, 198, 151644, 872, 198, 14990, 151645, 198, 151644, 77091, 198, 5158, 646, 358, 1492, 498, 30, 151645, 198, 151644, 872, 198, 5158, 1753, 374, 220, 17, 10, 17, 30, 151645, 198, 151644, 77091, 198}}; + test_prompts.insert({model_name, prompts}); + test_tokens.insert({model_name, tokens}); + + //////// + std::string model_name{"mistralai/Mistral-7B-Instruct-v0.2"}; + std::vector prompts{ + "[INST] hello [/INST]", + "error", + "error"}; + std::vector tokens{ + {1, 733, 16289, 28793, 6312, 28709, 733, 28748, 16289, 28793}, + {}, + {}}; + test_prompts.insert({model_name, prompts}); + test_tokens.insert({model_name, tokens}); + + //////// + std::string model_name{"THUDM/glm-4-9b-chat"}; + std::vector prompts{ + "[gMASK]<|user|>\nhello<|assistant|>", + "[gMASK]<|system|>\nYou are assistant.<|user|>\nhello<|assistant|>", + "[gMASK]<|system|>\nYou are assistant.<|user|>\nhello<|assistant|>\nhow can I help you?<|user|>\nhow much is 2+2?<|assistant|>"}; + std::vector tokens{ + {151331, 151333, 151336, 198, 14978, 151337}, + {151331, 151333, 151335, 198, 2610, 525, 17821, 13, 151336, 198, 14978, 151337}, + {151331, 151333, 151335, 198, 2610, 525, 17821, 13, 151336, 198, 14978, 151337, 198, 5158, 646, 358, 1492, 498, 30, 151336, 198, 5158, 1753, 374, 220, 17, 10, 17, 30, 151337}}; + test_prompts.insert({model_name, prompts}); + test_tokens.insert({model_name, tokens}); + + std::cout << chat1[0].at("role") << ": " << chat1[0].at("content") << std::endl; + std::cout << prompts[0] << std::endl; +}; From f9b978e546d19d90a75c60c9c4b3f03c0ed92001 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 20 Aug 2026 18:52:59 +0200 Subject: [PATCH 03/10] spelling --- spelling-whitelist.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index 175473be80..d7d92b3c44 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -41,3 +41,12 @@ src/test/llm/output_parsers/gemma4_output_parser_test.cpp src/test/llm/output_parsers/qwen3_output_parser_test.cpp:719: thi ==> the, this extras/chat_template_examples/chat_template_onyx.jinja src/test/llm/chat_templates/chat_template_onyx.jinja +src/mediapipe_internal/mediapipegraphdefinition.cpp:592: nowNs ==> knowns, nouns +src/mediapipe_internal/mediapipegraphdefinition.cpp:594: nowNs ==> knowns, nouns +src/mediapipe_internal/mediapipegraphdefinition.cpp:664: Re-use ==> Reuse +src/mediapipe_internal/mediapipegraphdefinition.hpp:96: nowNs ==> knowns, nouns +src/mediapipe_internal/mediapipegraphdefinition.hpp:97: nowNs ==> knowns, nouns +src/model_group_manager.cpp:372: nowNs ==> knowns, nouns +src/model_group_manager.cpp:374: nowNs ==> knowns, nouns +src/test/llm/llmnode_test.cpp:6168: nowNs ==> knowns, nouns +src/test/llm/llmnode_test.cpp:6170: nowNs ==> knowns, nouns From f9aad584fbfb812095f1267f9a1f938cb70e0222 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 20 Aug 2026 19:12:57 +0200 Subject: [PATCH 04/10] stype --- src/capi_frontend/server_settings.hpp | 1 + src/cli_parser.cpp | 16 ++- src/config.cpp | 2 +- src/config_export_module/config_export.cpp | 2 + .../mediapipegraphexecutor.cpp | 8 +- src/modelmanager.hpp | 18 +++ src/test/llm/llmnode_test.cpp | 2 +- src/test/schema_test.cpp | 10 +- src/test/template_test.cpp | 104 ------------------ 9 files changed, 46 insertions(+), 117 deletions(-) delete mode 100644 src/test/template_test.cpp diff --git a/src/capi_frontend/server_settings.hpp b/src/capi_frontend/server_settings.hpp index 491ea173bb..a6c9fd5c1e 100644 --- a/src/capi_frontend/server_settings.hpp +++ b/src/capi_frontend/server_settings.hpp @@ -264,6 +264,7 @@ struct ModelsSettingsImpl { uint32_t nireq = 0; std::string targetDevice; std::string pluginConfig; + std::optional groupName; std::vector userSetSingleModelArguments; std::string configPath; diff --git a/src/cli_parser.cpp b/src/cli_parser.cpp index 38a94900a1..131cf3277b 100644 --- a/src/cli_parser.cpp +++ b/src/cli_parser.cpp @@ -72,7 +72,7 @@ std::variant> CLIParser::parse(int argc, char* std::stringstream ss; try { options = std::make_unique(argv[0], "OpenVINO Model Server"); - auto configOptions = std::make_unique("ovms --add_to_config --config_path --model_name --model_repository_path \n ovms --add_to_config --config_path --model_path --model_name \n ovms --remove_from_config --config_path --model_name ", "config management commands:"); + auto configOptions = std::make_unique("ovms --add_to_config --config_path --model_name --model_repository_path \n ovms --add_to_config --config_path --model_path --model_name --group_name \n ovms --remove_from_config --config_path --model_name ", "config management commands:"); // Adding this option to parse unrecognised options in another parser options->allow_unrecognised_options(); @@ -213,7 +213,11 @@ std::variant> CLIParser::parse(int argc, char* ("remove_from_config", "Directive to remove a model from configuration file. This parameter should be executed with --config_path and --model_name to specify which model to remove.", cxxopts::value()->default_value("false"), - "REMOVE_FROM_CONFIG"); + "REMOVE_FROM_CONFIG") + ("group_name", + "Optional group name for idle model group management. Used with --add_to_config.", + cxxopts::value(), + "GROUP_NAME"); // Set default value for model_repository_path from environment variable if it exists and is not empty std::string defaultModelRepoPath = ""; @@ -347,6 +351,10 @@ std::variant> CLIParser::parse(int argc, char* "Name of the model", cxxopts::value(), "MODEL_NAME") + ("group_name", + "Optional group name for idle model group management", + cxxopts::value(), + "GROUP_NAME") ("config_path", "Path to json configuration file", cxxopts::value()->default_value(defaultConfigPath), @@ -926,6 +934,10 @@ void CLIParser::prepareConfigExport(ModelsSettingsImpl& modelsSettings) { } else if (!result->operator[]("model_repository_path").as().empty() && result->count("model_name")) { modelsSettings.modelPath = FileSystem::joinPath({result->operator[]("model_repository_path").as(), modelsSettings.modelName}); } + if (result->count("group_name")) { + modelsSettings.groupName = result->operator[]("group_name").as(); + modelsSettings.userSetSingleModelArguments.push_back("group_name"); + } std::string defaultConfigPath = ""; const char* envModelRepoPath = std::getenv("OVMS_MODEL_REPOSITORY_PATH"); if (envModelRepoPath != nullptr && std::string(envModelRepoPath).length() > 0) { diff --git a/src/config.cpp b/src/config.cpp index 0623b78ad4..986ea7bb94 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -141,7 +141,7 @@ bool Config::validateUserSettingsInConfigAddRemoveModel(const ModelsSettingsImpl static const std::vector allowedForRemove = {"model_name", "config_path"}; static const std::vector allowedForAdd = {"model_name", "model_path", "config_path", "batch_size", "shape", "layout", "mean", "scale", "color_format", "precision", - "model_version_policy", "nireq", "target_device", "plugin_config"}; + "model_version_policy", "nireq", "target_device", "plugin_config", "group_name"}; const auto& allowedUserSettings = (exportType == ENABLE_MODEL) ? allowedForAdd : allowedForRemove; std::vector usedButDisallowedUserSettings; diff --git a/src/config_export_module/config_export.cpp b/src/config_export_module/config_export.cpp index f2f5ecc39c..27474ba587 100644 --- a/src/config_export_module/config_export.cpp +++ b/src/config_export_module/config_export.cpp @@ -56,6 +56,8 @@ static void addOptionalModelFields(rapidjson::Value& configObj, const ModelsSett configObj.AddMember("target_device", rapidjson::Value(modelSettings.targetDevice.c_str(), alloc), alloc); if (!modelSettings.pluginConfig.empty()) addJsonOrStringMember(configObj, "plugin_config", modelSettings.pluginConfig, alloc); + if (modelSettings.groupName.has_value()) + configObj.AddMember("group_name", rapidjson::Value(modelSettings.groupName.value().c_str(), alloc), alloc); } Status loadJsonConfig(const std::string& jsonFilename, rapidjson::Document& configJson) { diff --git a/src/mediapipe_internal/mediapipegraphexecutor.cpp b/src/mediapipe_internal/mediapipegraphexecutor.cpp index 0ae0b5f4e9..330f9b2705 100644 --- a/src/mediapipe_internal/mediapipegraphexecutor.cpp +++ b/src/mediapipe_internal/mediapipegraphexecutor.cpp @@ -63,8 +63,8 @@ MediapipeGraphExecutor::MediapipeGraphExecutor( mediapipeServableMetricReporter(mediapipeServableMetricReporter), guard(std::move(guard)), activeInferenceGuard(activeInferenceCount - ? std::optional(ActiveInferenceGuard(std::move(activeInferenceCount), std::move(lastActivityTimeNs))) - : std::nullopt) {} + ? std::optional(ActiveInferenceGuard(std::move(activeInferenceCount), std::move(lastActivityTimeNs))) + : std::nullopt) {} MediapipeGraphExecutor::MediapipeGraphExecutor( const std::string& name, const std::string& version, @@ -90,7 +90,7 @@ MediapipeGraphExecutor::MediapipeGraphExecutor( currentStreamTimestamp(::mediapipe::Timestamp(STARTING_TIMESTAMP_VALUE)), mediapipeServableMetricReporter(mediapipeServableMetricReporter), activeInferenceGuard(activeInferenceCount - ? std::optional(ActiveInferenceGuard(std::move(activeInferenceCount), std::move(lastActivityTimeNs))) - : std::nullopt) {} + ? std::optional(ActiveInferenceGuard(std::move(activeInferenceCount), std::move(lastActivityTimeNs))) + : std::nullopt) {} } // namespace ovms diff --git a/src/modelmanager.hpp b/src/modelmanager.hpp index c093700e6c..1eff875282 100644 --- a/src/modelmanager.hpp +++ b/src/modelmanager.hpp @@ -57,6 +57,7 @@ class MediapipeFactory; class MediapipeGraphConfig; class MediapipeGraphExecutor; class ModelInstance; +class ModelGroupManager; class ServableDefinition; class ModelInstanceUnloadGuard; class Pipeline; @@ -237,6 +238,12 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M */ std::string rootDirectoryPath; bool startedWithConfigFile = false; + + /** + * @brief Model group manager for idle load/unload + */ + std::unique_ptr groupManager_; + /** * @brief Set json config directory path * @@ -307,6 +314,14 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M return models; } + const std::unordered_map& getServedModelConfigs() const { + return servedModelConfigs; + } + + ModelGroupManager* getGroupManager() const { + return groupManager_.get(); + } + const std::vector getNamesOfAvailableModels() const; /** @@ -320,6 +335,9 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M const MediapipeFactory& getMediapipeFactory() const { return *mediapipeFactory; } + MediapipeFactory& getMediapipeFactory() { + return *mediapipeFactory; + } #endif const CustomNodeLibraryManager& getCustomNodeLibraryManager() const; diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index 8db89cd73f..b6b1aeb8c6 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -5848,7 +5848,7 @@ class LLMIdleUnloadTest : public ::testing::Test { node_options: { [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { models_path: ")" + - modelsPath + R"(" + modelsPath + R"(" cache_size: 1 } } diff --git a/src/test/schema_test.cpp b/src/test/schema_test.cpp index b56cb7fb00..541e5b755a 100644 --- a/src/test/schema_test.cpp +++ b/src/test/schema_test.cpp @@ -2145,7 +2145,7 @@ TEST(SchemaTest, MediapipeConfigGroupNameInvalidType) { EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); } #endif - const char* mediapipeConfigNegative = R"( +const char* mediapipeConfigNegative = R"( { "model_config_list": [], "mediapipe_config_list": [ @@ -2157,8 +2157,8 @@ TEST(SchemaTest, MediapipeConfigGroupNameInvalidType) { ] })"; - rapidjson::Document configDoc; - configDoc.Parse(mediapipeConfigNegative); - auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); - EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +rapidjson::Document configDoc; +configDoc.Parse(mediapipeConfigNegative); +auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); +EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); } diff --git a/src/test/template_test.cpp b/src/test/template_test.cpp deleted file mode 100644 index 648110754b..0000000000 --- a/src/test/template_test.cpp +++ /dev/null @@ -1,104 +0,0 @@ -#include -#include -#include -#include - -int main() { - using chat_t = std::vector>; - - chat_t chat1{{{"role", "user"}, {"content", "hello"}}}; - chat_t chat2{{{"role", "system"}, {"content", "You are assistant."}, {"role", "user"}, {"content", "hello"}}}; - chat_t chat3{{{"role", "system"}, {"content", "You are assistant."}, {"role", "user"}, {"content", "hello"}, - {"role", "assistant"}, {"content", "how can I help you?"}, {"role", "user"}, {"content", "how much is 2+2?"}}}; - /* -curl http://ov-spr-19.sclab.intel.com:8000/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"meta-llama/Llama-2-7b-chat-hf","messages":[{"role":"user","content":"hello"}], "max_tokens":30}' -curl http://ov-spr-19.sclab.intel.com:8000/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"meta-llama/Llama-2-7b-chat-hf","messages":[{"role":"system","content":"You are assistant."},{"role":"user","content":"hello"}], "max_tokens":30}' -curl http://ov-spr-19.sclab.intel.com:8000/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"meta-llama/Llama-2-7b-chat-hf","messages":[{"role":"system","content":"You are assistant."},{"role":"user","content":"hello"},{"role":"assistant","content":"how can I help you?"}, {"role":"user","content":"how much is 2+2?"}], "max_tokens":30}' -*/ - - using expected_prompt = std::string; - using expected_tokens = std::vector; - std::unordered_map> test_prompts; - std::unordered_map> test_tokens; - - //////// - std::string model_name{"meta-llama/Llama-2-7b-chat-hf"}; - std::vector prompts{ - "[INST] hello [/INST]", - "[INST] <>\nYou are assistant.\n<>\n\nhello [/INST]", - "[INST] <>\nYou are assistant.\n<>\n\nhello [/INST] how can I help you? [INST] how much is 2+2? [/INST]"}; - std::vector tokens{ - {1, 518, 25580, 29962, 22172, 518, 29914, 25580, 29962}, - {1, 518, 25580, 29962, 3532, 14816, 29903, 6778, 13, 3492, 526, 20255, 29889, 13, 29966, 829, 14816, 29903, 6778, 13, 13, 12199, 518, 29914, 25580, 29962}, - {1, 518, 25580, 29962, 3532, 14816, 29903, 6778, 13, 3492, 526, 20255, 29889, 13, 29966, 829, 14816, 29903, 6778, 13, 13, 12199, 518, 29914, 25580, 29962, 920, 508, 306, 1371, 366, 29973, 29871, 2, 1, 518, 25580, 29962, 920, 1568, 338, 29871, 29906, 29974, 29906, 29973, 518, 29914, 25580, 29962}}; - test_prompts.insert({model_name, prompts}); - test_tokens.insert({model_name, tokens}); - - //////// - std::string model_name{"meta-llama/Meta-Llama-3-8B-Instruct"}; - std::vector prompts{ - "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\nhello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", - "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nhello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", - "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nhello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nhow can I help you?<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nhow much is 2+2?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"}; - std::vector tokens{ - {128000, 128006, 882, 128007, 271, 15339, 128009, 128006, 78191, 128007, 271}, - {128000, 128006, 9125, 128007, 271, 2675, 527, 18328, 13, 128009, 128006, 882, 128007, 271, 15339, 128009, 128006, 78191, 128007, 271}, - {128000, 128006, 9125, 128007, 271, 2675, 527, 18328, 13, 128009, 128006, 882, 128007, 271, 15339, 128009, 128006, 78191, 128007, 271, 5269, 649, 358, 1520, 499, 30, 128009, 128006, 882, 128007, 271, 5269, 1790, 374, 220, 17, 10, 17, 30, 128009, 128006, 78191, 128007, 271}}; - test_prompts.insert({model_name, prompts}); - test_tokens.insert({model_name, tokens}); - - //////// - std::string model_name{"TinyLlama/TinyLlama-1.1B-Chat-v0.6"}; - std::vector prompts{ - "<|user|>\nhello\n<|assistant|>\n", - "<|system|>\nYou are assistant.\n<|user|>\nhello\n<|assistant|>\n", - "<|system|>\nYou are assistant.\n<|user|>\nhello\n<|assistant|>\nhow can I help you?\n<|user|>\nhow much is 2+2?\n<|assistant|>\n"}; - std::vector tokens{ - {529, 29989, 1792, 29989, 29958, 13, 12199, 2, 29871, 13, 29966, 29989, 465, 22137, 29989, 29958, 13}, - {529, 29989, 5205, 29989, 29958, 13, 3492, 526, 20255, 29889, 2, 29871, 13, 29966, 29989, 1792, 29989, 29958, 13, 12199, 2, 29871, 13, 29966, 29989, 465, 22137, 29989, 29958, 13}, - {529, 29989, 5205, 29989, 29958, 13, 3492, 526, 20255, 29889, 2, 29871, 13, 29966, 29989, 1792, 29989, 29958, 13, 12199, 2, 29871, 13, 29966, 29989, 465, 22137, 29989, 29958, 13, 3525, 508, 306, 1371, 366, 29973, 2, 29871, 13, 29966, 29989, 1792, 29989, 29958, 13, 3525, 1568, 338, 29871, 29906, 29974, 29906, 29973, 2, 29871, 13, 29966, 29989, 465, 22137, 29989, 29958, 13}}; - test_prompts.insert({model_name, prompts}); - test_tokens.insert({model_name, tokens}); - - //////// - std::string model_name{"Qwen/Qwen-7B-Chat"}; - std::vector prompts{ - "<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n", - "<|im_start|>system\nYou are assistant.<|im_end|>\n<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n", - "<|im_start|>system\nYou are assistant.<|im_end|>\n<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\nhow can I help you?<|im_end|>\n<|im_start|>user\nhow much is 2+2?<|im_end|>\n<|im_start|>assistant\n"}; - std::vector tokens{ - {151644, 872, 198, 14990, 151645, 198, 151644, 77091, 198}, - {151644, 8948, 198, 2610, 525, 17847, 13, 151645, 198, 151644, 872, 198, 14990, 151645, 198, 151644, 77091, 198}, - {151644, 8948, 198, 2610, 525, 17847, 13, 151645, 198, 151644, 872, 198, 14990, 151645, 198, 151644, 77091, 198, 5158, 646, 358, 1492, 498, 30, 151645, 198, 151644, 872, 198, 5158, 1753, 374, 220, 17, 10, 17, 30, 151645, 198, 151644, 77091, 198}}; - test_prompts.insert({model_name, prompts}); - test_tokens.insert({model_name, tokens}); - - //////// - std::string model_name{"mistralai/Mistral-7B-Instruct-v0.2"}; - std::vector prompts{ - "[INST] hello [/INST]", - "error", - "error"}; - std::vector tokens{ - {1, 733, 16289, 28793, 6312, 28709, 733, 28748, 16289, 28793}, - {}, - {}}; - test_prompts.insert({model_name, prompts}); - test_tokens.insert({model_name, tokens}); - - //////// - std::string model_name{"THUDM/glm-4-9b-chat"}; - std::vector prompts{ - "[gMASK]<|user|>\nhello<|assistant|>", - "[gMASK]<|system|>\nYou are assistant.<|user|>\nhello<|assistant|>", - "[gMASK]<|system|>\nYou are assistant.<|user|>\nhello<|assistant|>\nhow can I help you?<|user|>\nhow much is 2+2?<|assistant|>"}; - std::vector tokens{ - {151331, 151333, 151336, 198, 14978, 151337}, - {151331, 151333, 151335, 198, 2610, 525, 17821, 13, 151336, 198, 14978, 151337}, - {151331, 151333, 151335, 198, 2610, 525, 17821, 13, 151336, 198, 14978, 151337, 198, 5158, 646, 358, 1492, 498, 30, 151336, 198, 5158, 1753, 374, 220, 17, 10, 17, 30, 151337}}; - test_prompts.insert({model_name, prompts}); - test_tokens.insert({model_name, tokens}); - - std::cout << chat1[0].at("role") << ": " << chat1[0].at("content") << std::endl; - std::cout << prompts[0] << std::endl; -}; From 4554fb63c2592a43b31cec80fa63197f5d17108c Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 20 Aug 2026 20:15:55 +0200 Subject: [PATCH 05/10] style --- src/model_group_manager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/model_group_manager.cpp b/src/model_group_manager.cpp index bd20d636ff..6c8c3501ec 100644 --- a/src/model_group_manager.cpp +++ b/src/model_group_manager.cpp @@ -16,6 +16,7 @@ #include "model_group_manager.hpp" #include +#include #include #include "logging.hpp" From f4c12ec262022a161c66ff45ab04d2b1ac3152ec Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 20 Aug 2026 20:25:56 +0200 Subject: [PATCH 06/10] fix --- spelling-whitelist.txt | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index d7d92b3c44..dde4556cb9 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -41,12 +41,6 @@ src/test/llm/output_parsers/gemma4_output_parser_test.cpp src/test/llm/output_parsers/qwen3_output_parser_test.cpp:719: thi ==> the, this extras/chat_template_examples/chat_template_onyx.jinja src/test/llm/chat_templates/chat_template_onyx.jinja -src/mediapipe_internal/mediapipegraphdefinition.cpp:592: nowNs ==> knowns, nouns -src/mediapipe_internal/mediapipegraphdefinition.cpp:594: nowNs ==> knowns, nouns -src/mediapipe_internal/mediapipegraphdefinition.cpp:664: Re-use ==> Reuse -src/mediapipe_internal/mediapipegraphdefinition.hpp:96: nowNs ==> knowns, nouns -src/mediapipe_internal/mediapipegraphdefinition.hpp:97: nowNs ==> knowns, nouns -src/model_group_manager.cpp:372: nowNs ==> knowns, nouns -src/model_group_manager.cpp:374: nowNs ==> knowns, nouns -src/test/llm/llmnode_test.cpp:6168: nowNs ==> knowns, nouns -src/test/llm/llmnode_test.cpp:6170: nowNs ==> knowns, nouns +src/mediapipe_internal/mediapipegraphdefinition.cpp +src/model_group_manager.cpp +src/test/llm/llmnode_test.cpp From 8abf884ba050178eabe1bd5f41b96dac73c65230 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 20 Aug 2026 23:18:57 +0200 Subject: [PATCH 07/10] fix --- spelling-whitelist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index dde4556cb9..205828e72e 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -42,5 +42,6 @@ src/test/llm/output_parsers/qwen3_output_parser_test.cpp:719: thi ==> the, this extras/chat_template_examples/chat_template_onyx.jinja src/test/llm/chat_templates/chat_template_onyx.jinja src/mediapipe_internal/mediapipegraphdefinition.cpp +src/mediapipe_internal/mediapipegraphdefinition.hpp src/model_group_manager.cpp src/test/llm/llmnode_test.cpp From 5b4dffaca68cb43f5ee009209a50966466b57494 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 20 Aug 2026 23:28:41 +0200 Subject: [PATCH 08/10] style --- src/model_group_manager.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/model_group_manager.hpp b/src/model_group_manager.hpp index 8eb657f372..9c2d85b630 100644 --- a/src/model_group_manager.hpp +++ b/src/model_group_manager.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include From ff586f1769676172637fe60dc2b7161deaf963ff Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 20 Aug 2026 23:59:55 +0200 Subject: [PATCH 09/10] build fix --- src/test/schema_test.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/test/schema_test.cpp b/src/test/schema_test.cpp index 541e5b755a..e4b0020bfb 100644 --- a/src/test/schema_test.cpp +++ b/src/test/schema_test.cpp @@ -2063,6 +2063,7 @@ TEST(SchemaTest, MediapipeConfigIdleUnloadTimeoutWrongTypeRejected) { auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); } +#endif TEST(SchemaTest, ModelConfigGroupNameValidString) { const char* config = R"( @@ -2145,7 +2146,9 @@ TEST(SchemaTest, MediapipeConfigGroupNameInvalidType) { EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); } #endif -const char* mediapipeConfigNegative = R"( + +TEST(SchemaTest, MediapipeConfigNegativeAdditionalMediapipeConfigField) { + const char* mediapipeConfigNegative = R"( { "model_config_list": [], "mediapipe_config_list": [ @@ -2157,8 +2160,8 @@ const char* mediapipeConfigNegative = R"( ] })"; -rapidjson::Document configDoc; -configDoc.Parse(mediapipeConfigNegative); -auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); -EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); + rapidjson::Document configDoc; + configDoc.Parse(mediapipeConfigNegative); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); } From d835e7aa44a2d76fd0753caa15a064611d0c589d Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Fri, 21 Aug 2026 01:01:54 +0200 Subject: [PATCH 10/10] initialization fix --- src/mediapipe_internal/mediapipefactory.cpp | 15 ++++ src/mediapipe_internal/mediapipefactory.hpp | 4 + .../mediapipegraphdefinition.cpp | 11 +++ .../mediapipegraphdefinition.hpp | 5 ++ src/modelmanager.cpp | 23 ++--- src/test/mediapipeflow_test.cpp | 85 +++++++++++++++++++ 6 files changed, 133 insertions(+), 10 deletions(-) diff --git a/src/mediapipe_internal/mediapipefactory.cpp b/src/mediapipe_internal/mediapipefactory.cpp index 5f8deebd6b..4f7e546875 100644 --- a/src/mediapipe_internal/mediapipefactory.cpp +++ b/src/mediapipe_internal/mediapipefactory.cpp @@ -78,6 +78,21 @@ Status MediapipeFactory::createDefinition(const std::string& pipelineName, return stat; } +Status MediapipeFactory::createDefinitionAsUnloaded(const std::string& pipelineName, + const MediapipeGraphConfig& config, + MetricProvider& metrics) { + if (definitionExists(pipelineName)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Mediapipe graph definition: {} is already created", pipelineName); + return StatusCode::PIPELINE_DEFINITION_ALREADY_EXIST; + } + std::shared_ptr graphDefinition = std::make_shared( + pipelineName, config, metrics.getMetricRegistry(), &metrics.getMetricConfig(), pythonBackend); + graphDefinition->setAsUnloaded(); + std::unique_lock lock(definitionsMtx); + definitions.insert({pipelineName, std::move(graphDefinition)}); + return StatusCode::OK; +} + bool MediapipeFactory::definitionExists(const std::string& name) const { std::shared_lock lock(definitionsMtx); if (this->definitions.find(name) != this->definitions.end()) { diff --git a/src/mediapipe_internal/mediapipefactory.hpp b/src/mediapipe_internal/mediapipefactory.hpp index a9ac8ae9b0..974532e1a5 100644 --- a/src/mediapipe_internal/mediapipefactory.hpp +++ b/src/mediapipe_internal/mediapipefactory.hpp @@ -49,6 +49,10 @@ class MediapipeFactory { MetricProvider& metrics, const ServableNameChecker& checker); + Status createDefinitionAsUnloaded(const std::string& pipelineName, + const MediapipeGraphConfig& config, + MetricProvider& metrics); + bool definitionExists(const std::string& name) const; public: diff --git a/src/mediapipe_internal/mediapipegraphdefinition.cpp b/src/mediapipe_internal/mediapipegraphdefinition.cpp index 14289ec24a..b690d921b3 100644 --- a/src/mediapipe_internal/mediapipegraphdefinition.cpp +++ b/src/mediapipe_internal/mediapipegraphdefinition.cpp @@ -594,6 +594,17 @@ bool MediapipeGraphDefinition::shouldUnloadDueToIdle() const { return (nowNs - lastActivity) >= timeoutNs; } +void MediapipeGraphDefinition::setAsUnloaded() { + // Transition from BEGIN → AVAILABLE → UNLOADED without loading any resources. + // Used during initialization with idle group management to create definitions + // that exist in the factory but skip the expensive validate()/initializeNodes() + // path. wakeUpIfUnloaded() will later perform the full load on demand. + this->status.handle(ValidationPassedEvent()); + this->status.handle(UnloadEvent()); + SPDLOG_LOGGER_INFO(modelmanager_logger, + "Mediapipe graph {} created in UNLOADED state (idle group management)", getName()); +} + Status MediapipeGraphDefinition::unload() { // Serialize against wakeUpIfUnloaded()/reload()/retire() using the SAME lock so // all lifecycle mutations are mutually exclusive. This prevents the watcher thread diff --git a/src/mediapipe_internal/mediapipegraphdefinition.hpp b/src/mediapipe_internal/mediapipegraphdefinition.hpp index f05e66fc4f..5c13c32a2f 100644 --- a/src/mediapipe_internal/mediapipegraphdefinition.hpp +++ b/src/mediapipe_internal/mediapipegraphdefinition.hpp @@ -90,6 +90,11 @@ class MediapipeGraphDefinition : public SingleVersionServableDefinition { bool isIdleUnloadEnabled() const; bool shouldUnloadDueToIdle() const; + // Create definition in UNLOADED state without loading any resources. + // Used during initialization with idle group management to avoid loading + // non-permanent graphs that would be immediately unloaded. + void setAsUnloaded(); + // Test-only: backdate the last-activity timestamp by the given number of seconds // so idle-timeout behavior can be exercised deterministically without sleeping. void backdateLastActivityForTest(int64_t seconds) { diff --git a/src/modelmanager.cpp b/src/modelmanager.cpp index fa396ac2f1..9df4ce2c6b 100644 --- a/src/modelmanager.cpp +++ b/src/modelmanager.cpp @@ -465,6 +465,16 @@ Status ModelManager::processMediapipeConfig(const MediapipeGraphConfig& config, mediapipesInConfigFile.insert(config.getGraphName()); MediapipeGraphDefinition* mediapipeGraphDefinition = factory.findDefinitionByName(config.getGraphName()); if (mediapipeGraphDefinition == nullptr) { + // When idle group management is enabled and the graph belongs to a non-permanent + // group, create it in UNLOADED state to avoid expensive validate()/initializeNodes() + // that would load LLM models into GPU only to immediately unload them. + if (groupManager_ && groupManager_->isEnabled() && + !config.getGroupName().empty() && config.getGroupName() != "permanent") { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, + "Mediapipe graph:{} belongs to non-permanent group '{}'; creating as UNLOADED", + config.getGraphName(), config.getGroupName()); + return factory.createDefinitionAsUnloaded(config.getGraphName(), config, *this); + } SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Mediapipe graph:{} was not loaded so far. Triggering load", config.getGraphName()); auto status = factory.createDefinition(config.getGraphName(), config, *this, *this); return status; @@ -967,7 +977,9 @@ Status ModelManager::loadConfig() { // Build model groups and unload non-permanent servables for on-demand loading if (groupManager_ && groupManager_->isEnabled()) { groupManager_->buildGroups(this->servedModelConfigs, *this); - // Unload all non-permanent servables so they are loaded on demand + // Unload all non-permanent models so they are loaded on demand. + // Mediapipe graphs in non-permanent groups are already created in UNLOADED + // state by processMediapipeConfig, so only models need retirement here. for (const auto& [groupName, groupInfo] : groupManager_->getGroups()) { if (groupInfo.isPermanent()) { continue; @@ -979,15 +991,6 @@ Status ModelManager::loadConfig() { SPDLOG_INFO("Retired model '{}' (group '{}') for on-demand loading", modelName, groupName); } } -#if (MEDIAPIPE_DISABLE == 0) - for (const auto& graphName : groupInfo.mediapipeNames) { - MediapipeGraphDefinition* def = mediapipeFactory->findDefinitionByName(graphName); - if (def != nullptr) { - def->unload(); - SPDLOG_INFO("Unloaded mediapipe graph '{}' (group '{}') for on-demand loading", graphName, groupName); - } - } -#endif } } diff --git a/src/test/mediapipeflow_test.cpp b/src/test/mediapipeflow_test.cpp index 34402403f5..08e17e129a 100644 --- a/src/test/mediapipeflow_test.cpp +++ b/src/test/mediapipeflow_test.cpp @@ -4597,3 +4597,88 @@ TEST(MediapipeIdleUnloadGuard, UnloadSkipsWhenRequestsInFlight) { ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); ASSERT_TRUE(def.sidePacketMapsEmptyForTest()); } + +// --------------------------------------------------------------------------- +// setAsUnloaded() — skip initial loading for idle group management +// --------------------------------------------------------------------------- + +TEST(MediapipeIdleUnloadGuard, SetAsUnloadedTransitionsFromBeginToUnloaded) { + ovms::MediapipeGraphConfig mgc{"skipLoad", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("skipLoad", mgc, kIdleUnloadDummyPbtxt, nullptr); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::BEGIN); + + def.setAsUnloaded(); + + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); +} + +TEST(MediapipeIdleUnloadGuard, SetAsUnloadedDoesNotClearResources) { + ovms::MediapipeGraphConfig mgc{"skipLoad", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("skipLoad", mgc, kIdleUnloadDummyPbtxt, nullptr); + // Insert a marker into side packet maps before setAsUnloaded. + def.insertSidePacketMarkerForTest("marker"); + const void* mapsBefore = def.sidePacketMapsPtrForTest(); + + def.setAsUnloaded(); + + // setAsUnloaded only transitions the state machine — it does not clear resources + // (there are none to clear since validate() was never called). + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + ASSERT_TRUE(def.hasSidePacketMarkerForTest("marker")); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), mapsBefore); +} + +TEST(MediapipeIdleUnloadGuard, SetAsUnloadedThenUnloadIsNoOp) { + ovms::MediapipeGraphConfig mgc{"skipLoad", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("skipLoad", mgc, kIdleUnloadDummyPbtxt, nullptr); + def.setAsUnloaded(); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); + + // A second unload() on an already-UNLOADED graph should be a no-op. + ASSERT_EQ(def.unload(), ovms::StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); +} + +// --------------------------------------------------------------------------- +// createDefinitionAsUnloaded() — factory-level test +// --------------------------------------------------------------------------- + +namespace { +class StubMetricProvider : public ovms::MetricProvider { +public: + ovms::MetricRegistry* getMetricRegistry() const override { return nullptr; } + const ovms::MetricConfig& getMetricConfig() const override { return config_; } + +private: + ovms::MetricConfig config_; +}; +} // namespace + +TEST(MediapipeIdleUnloadGuard, CreateDefinitionAsUnloaded) { + ovms::MediapipeFactory factory(nullptr); + ovms::MediapipeGraphConfig mgc{"unloadedGraph", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + StubMetricProvider metrics; + + auto status = factory.createDefinitionAsUnloaded("unloadedGraph", mgc, metrics); + ASSERT_EQ(status, ovms::StatusCode::OK); + + auto* def = factory.findDefinitionByName("unloadedGraph"); + ASSERT_NE(def, nullptr); + ASSERT_EQ(def->getStateCode(), ovms::PipelineDefinitionStateCode::UNLOADED); +} + +TEST(MediapipeIdleUnloadGuard, CreateDefinitionAsUnloadedRejectsDuplicate) { + ovms::MediapipeFactory factory(nullptr); + ovms::MediapipeGraphConfig mgc{"dupGraph", "", ""}; + StubMetricProvider metrics; + + auto status1 = factory.createDefinitionAsUnloaded("dupGraph", mgc, metrics); + ASSERT_EQ(status1, ovms::StatusCode::OK); + + auto status2 = factory.createDefinitionAsUnloaded("dupGraph", mgc, metrics); + ASSERT_EQ(status2, ovms::StatusCode::PIPELINE_DEFINITION_ALREADY_EXIST); +}