Idle models

docs/llm/reference.md

@@ -112,6 +112,16 @@ The calculator supports the following `node_options` for tuning the pipeline con
 -    `optional string tool_parser` - name of the parser to use for tool calls extraction from model output before creating a response;
 -    `optional bool enable_tool_guided_generation` - enable enforcing tool schema during generation. Requires setting response parser. [default = false];
 -    `optional SparseAttentionConfig sparse_attention_config` - Sparse attention configuration. Disabled if not specified.
+-    `optional int64 idle_unload_timeout_seconds` - unload the graph's model resources after this many seconds with no inference requests, freeing GPU/CPU memory; the model is reloaded automatically on the next request. `0` disables the feature [default = 0]. See [Idle model unload](#idle-model-unload).
+
+### Idle model unload
+When `idle_unload_timeout_seconds` is set to a positive value, the model server unloads the LLM graph's heavy resources (the continuous batching pipeline, freeing GPU VRAM / host memory) after the configured period without any inference requests. The first request after an unload transparently reloads the model and is served once it is ready, so the GPU can be used by other workloads while a model is idle.
+
+Notes:
+- Only inference requests reset the idle timer; status/metrics/health endpoints do not keep a model loaded.
+- The first request after an idle unload pays the reload latency. Combine with [model caching](../model_cache.md) (`--cache_dir`) so the reload is a fast cache import rather than a full recompile.
+- The graph reports as `AVAILABLE` while idle-unloaded (it auto-reloads on demand). The `ovms_graph_loaded` metric reports `1` when loaded and `0` when idle-unloaded.
+- Supported for LLM continuous-batching graphs. Graphs containing Python nodes are not supported with this setting.
 
 ### Caching settings
 The value of `cache_size` might have performance and stability implications. It is used for storing LLM model KV cache data. Adjust it based on your environment capabilities, model size and expected level of concurrency.

spelling-whitelist.txt

@@ -41,3 +41,7 @@ src/test/llm/output_parsers/gemma4_output_parser_test.cpp
 src/test/llm/output_parsers/qwen3_output_parser_test.cpp:719: thi ==> the, this
 extras/chat_template_examples/chat_template_onyx.jinja
 src/test/llm/chat_templates/chat_template_onyx.jinja
+src/mediapipe_internal/mediapipegraphdefinition.cpp
+src/mediapipe_internal/mediapipegraphdefinition.hpp
+src/model_group_manager.cpp
+src/test/llm/llmnode_test.cpp

src/BUILD

@@ -583,8 +583,8 @@ ovms_cc_library(
 )
 ovms_cc_library(
     name = "modelmanager",
-    hdrs = ["modelmanager.hpp"],
-    srcs = ["modelmanager.cpp"],
+    hdrs = ["modelmanager.hpp", "model_group_manager.hpp"],
+    srcs = ["modelmanager.cpp", "model_group_manager.cpp"],
     deps = select({
         "//conditions:default": [],
         "//:not_disable_mediapipe" : [
@@ -2032,6 +2032,7 @@ cc_test(
         "test/model_version_policy_test.cpp",
         "test/modelconfig_test.cpp",
         "test/modelinstance_test.cpp",
+        "test/model_group_manager_test.cpp",
         "test/modelmanager_test.cpp",
         "test/modelversionstatus_test.cpp",
         "test/node_library_manager_test.cpp",

src/capi_frontend/capi_dag_utils.cpp

@@ -41,6 +41,8 @@ OVMS_ServableState convertToServableState(ovms::PipelineDefinitionStateCode code
     case ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED:
     case ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION:
         return OVMS_ServableState::OVMS_STATE_LOADING_FAILED;
+    case ovms::PipelineDefinitionStateCode::UNLOADED:
+        return OVMS_ServableState::OVMS_STATE_RETIRED;
     }
     throw new std::exception();
 }

src/capi_frontend/server_settings.hpp

@@ -241,6 +241,7 @@ struct ServerSettingsImpl {
     std::string grpcChannelArguments;
     uint32_t filesystemPollWaitMilliseconds = 1000;
     uint32_t resourcesCleanerPollWaitSeconds = 300;
+    uint32_t idleUnloadTimeoutSeconds = 0;
     std::string cacheDir;
     bool withPython = false;
     bool startedWithCLI = false;
@@ -263,6 +264,7 @@ struct ModelsSettingsImpl {
     uint32_t nireq = 0;
     std::string targetDevice;
     std::string pluginConfig;
+    std::optional<std::string> groupName;
     std::vector<std::string> userSetSingleModelArguments;
 
     std::string configPath;

src/cli_parser.cpp

@@ -72,7 +72,7 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
     std::stringstream ss;
     try {
         options = std::make_unique<cxxopts::Options>(argv[0], "OpenVINO Model Server");
-        auto configOptions = std::make_unique<cxxopts::Options>("ovms --add_to_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME> --model_repository_path <MODEL_REPO_PATH> \n  ovms --add_to_config --config_path <CONFIG_PATH> --model_path <MODEL_PATH> --model_name <MODEL_NAME>  \n  ovms --remove_from_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME>", "config management commands:");
+        auto configOptions = std::make_unique<cxxopts::Options>("ovms --add_to_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME> --model_repository_path <MODEL_REPO_PATH> \n  ovms --add_to_config --config_path <CONFIG_PATH> --model_path <MODEL_PATH> --model_name <MODEL_NAME> --group_name <GROUP> \n  ovms --remove_from_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME>", "config management commands:");
         // Adding this option to parse unrecognised options in another parser
         options->allow_unrecognised_options();
 
@@ -137,6 +137,10 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
                 "Time interval between config and model versions changes detection. Default is 1. Zero or negative value disables changes monitoring.",
                 cxxopts::value<uint32_t>()->default_value("1"),
                 "FILE_SYSTEM_POLL_WAIT_SECONDS")
+            ("idle_unload_timeout_seconds",
+                "Idle timeout in seconds for model group unloading. When > 0, models not in the 'permanent' group are loaded on demand and unloaded after this idle period. Only effective with config.json multi-model setup. Default is 0 (disabled).",
+                cxxopts::value<uint32_t>()->default_value("0"),
+                "IDLE_UNLOAD_TIMEOUT_SECONDS")
             ("custom_node_resources_cleaner_interval_seconds",
                 "Time interval between two consecutive resources cleanup scans. Default is 300. Zero value disables resources cleaner.",
                 cxxopts::value<uint32_t>()->default_value("300"),
@@ -209,7 +213,11 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
             ("remove_from_config",
                 "Directive to remove a model from configuration file. This parameter should be executed with --config_path and --model_name to specify which model to remove.",
                 cxxopts::value<bool>()->default_value("false"),
-                "REMOVE_FROM_CONFIG");
+                "REMOVE_FROM_CONFIG")
+            ("group_name",
+                "Optional group name for idle model group management. Used with --add_to_config.",
+                cxxopts::value<std::string>(),
+                "GROUP_NAME");
 
         // Set default value for model_repository_path from environment variable if it exists and is not empty
         std::string defaultModelRepoPath = "";
@@ -343,6 +351,10 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
                 "Name of the model",
                 cxxopts::value<std::string>(),
                 "MODEL_NAME")
+            ("group_name",
+                "Optional group name for idle model group management",
+                cxxopts::value<std::string>(),
+                "GROUP_NAME")
             ("config_path",
                 "Path to json configuration file",
                 cxxopts::value<std::string>()->default_value(defaultConfigPath),
@@ -569,6 +581,7 @@ void CLIParser::prepareServer(ServerSettingsImpl& serverSettings) {
     serverSettings.filesystemPollWaitMilliseconds = result->operator[]("file_system_poll_wait_seconds").as<uint32_t>() * 1000;
 
     serverSettings.resourcesCleanerPollWaitSeconds = result->operator[]("custom_node_resources_cleaner_interval_seconds").as<uint32_t>();
+    serverSettings.idleUnloadTimeoutSeconds = result->operator[]("idle_unload_timeout_seconds").as<uint32_t>();
     serverSettings.grpcWorkers = result->operator[]("grpc_workers").as<uint32_t>();
 
     if (result->count("log_level"))
@@ -921,6 +934,10 @@ void CLIParser::prepareConfigExport(ModelsSettingsImpl& modelsSettings) {
     } else if (!result->operator[]("model_repository_path").as<std::string>().empty() && result->count("model_name")) {
         modelsSettings.modelPath = FileSystem::joinPath({result->operator[]("model_repository_path").as<std::string>(), modelsSettings.modelName});
     }
+    if (result->count("group_name")) {
+        modelsSettings.groupName = result->operator[]("group_name").as<std::string>();
+        modelsSettings.userSetSingleModelArguments.push_back("group_name");
+    }
     std::string defaultConfigPath = "";
     const char* envModelRepoPath = std::getenv("OVMS_MODEL_REPOSITORY_PATH");
     if (envModelRepoPath != nullptr && std::string(envModelRepoPath).length() > 0) {

src/config.cpp

@@ -141,7 +141,7 @@ bool Config::validateUserSettingsInConfigAddRemoveModel(const ModelsSettingsImpl
     static const std::vector<std::string> allowedForRemove = {"model_name", "config_path"};
     static const std::vector<std::string> allowedForAdd = {"model_name", "model_path", "config_path",
         "batch_size", "shape", "layout", "mean", "scale", "color_format", "precision",
-        "model_version_policy", "nireq", "target_device", "plugin_config"};
+        "model_version_policy", "nireq", "target_device", "plugin_config", "group_name"};
 
     const auto& allowedUserSettings = (exportType == ENABLE_MODEL) ? allowedForAdd : allowedForRemove;
     std::vector<std::string> usedButDisallowedUserSettings;
@@ -432,6 +432,7 @@ const std::string& Config::tracePath() const { return this->serverSettings.trace
 const std::string& Config::grpcChannelArguments() const { return this->serverSettings.grpcChannelArguments; }
 uint32_t Config::filesystemPollWaitMilliseconds() const { return this->serverSettings.filesystemPollWaitMilliseconds; }
 uint32_t Config::resourcesCleanerPollWaitSeconds() const { return this->serverSettings.resourcesCleanerPollWaitSeconds; }
+uint32_t Config::idleUnloadTimeoutSeconds() const { return this->serverSettings.idleUnloadTimeoutSeconds; }
 bool Config::allowCredentials() const { return this->serverSettings.allowCredentials; }
 const std::string& Config::allowedOrigins() const { return this->serverSettings.allowedOrigins; }
 const std::string& Config::allowedMethods() const { return this->serverSettings.allowedMethods; }

src/config.hpp

@@ -312,6 +312,13 @@ class Config {
      */
     uint32_t resourcesCleanerPollWaitSeconds() const;
 
+    /**
+     * @brief Get the idle unload timeout in seconds (0 = disabled)
+     *
+     * @return uint32_t
+     */
+    uint32_t idleUnloadTimeoutSeconds() const;
+
     bool allowCredentials() const;
     const std::string& allowedOrigins() const;
     const std::string& allowedMethods() const;

src/config_export_module/config_export.cpp

@@ -56,6 +56,8 @@ static void addOptionalModelFields(rapidjson::Value& configObj, const ModelsSett
         configObj.AddMember("target_device", rapidjson::Value(modelSettings.targetDevice.c_str(), alloc), alloc);
     if (!modelSettings.pluginConfig.empty())
         addJsonOrStringMember(configObj, "plugin_config", modelSettings.pluginConfig, alloc);
+    if (modelSettings.groupName.has_value())
+        configObj.AddMember("group_name", rapidjson::Value(modelSettings.groupName.value().c_str(), alloc), alloc);
 }
 
 Status loadJsonConfig(const std::string& jsonFilename, rapidjson::Document& configJson) {

src/dags/pipelinedefinitionstatus.cpp

@@ -35,7 +35,8 @@ const std::string& pipelineDefinitionStateCodeToString(PipelineDefinitionStateCo
         {PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION, "LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION"},
         {PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION, "AVAILABLE_REQUIRED_REVALIDATION"},
         {PipelineDefinitionStateCode::AVAILABLE, "AVAILABLE"},
-        {PipelineDefinitionStateCode::RETIRED, "RETIRED"}};
+        {PipelineDefinitionStateCode::RETIRED, "RETIRED"},
+        {PipelineDefinitionStateCode::UNLOADED, "UNLOADED"}};
     return names.at(code);
 }
 
@@ -62,6 +63,9 @@ StateKeeper BeginState::handle(const RetireEvent& e) const {
     throw std::logic_error(INVALID_TRANSITION_MESSAGE);
     return {};
 }
+StateKeeper BeginState::handle(const UnloadEvent& e) const {
+    return {};  // unload is a no-op when not yet loaded
+}
 
 PipelineDefinitionStateCode ReloadState::getStateCode() const {
     return code;
@@ -84,6 +88,9 @@ StateKeeper ReloadState::handle(const RetireEvent& e) const {
     throw std::logic_error(INVALID_TRANSITION_MESSAGE);
     return {};
 }
+StateKeeper ReloadState::handle(const UnloadEvent& e) const {
+    return {};  // unload is a no-op while reloading
+}
 
 PipelineDefinitionStateCode AvailableState::getStateCode() const {
     return code;
@@ -105,6 +112,9 @@ StateChanger<AvailableRequiredRevalidation> AvailableState::handle(const UsedMod
 StateChanger<RetiredState> AvailableState::handle(const RetireEvent& e) const {
     return {};
 }
+StateChanger<UnloadedState> AvailableState::handle(const UnloadEvent& e) const {
+    return {};
+}
 
 PipelineDefinitionStateCode AvailableRequiredRevalidation::getStateCode() const {
     return code;
@@ -124,6 +134,9 @@ StateKeeper AvailableRequiredRevalidation::handle(const UsedModelChangedEvent& e
 StateChanger<RetiredState> AvailableRequiredRevalidation::handle(const RetireEvent& e) const {
     return {};
 }
+StateKeeper AvailableRequiredRevalidation::handle(const UnloadEvent& e) const {
+    return {};  // unload is a no-op in AVAILABLE_REQUIRED_REVALIDATION
+}
 
 PipelineDefinitionStateCode LoadingPreconditionFailedState::getStateCode() const {
     return code;
@@ -145,6 +158,10 @@ StateChanger<LoadingFailedLastValidationRequiredRevalidation> LoadingPreconditio
 StateChanger<RetiredState> LoadingPreconditionFailedState::handle(const RetireEvent& e) const {
     return {};
 }
+StateChanger<UnloadedState> LoadingPreconditionFailedState::handle(const UnloadEvent& e) const {
+    // Revert a failed wake-up reload back to UNLOADED so the next request retries.
+    return {};
+}
 
 PipelineDefinitionStateCode LoadingFailedLastValidationRequiredRevalidation::getStateCode() const {
     return code;
@@ -164,6 +181,9 @@ StateKeeper LoadingFailedLastValidationRequiredRevalidation::handle(const UsedMo
 StateChanger<RetiredState> LoadingFailedLastValidationRequiredRevalidation::handle(const RetireEvent& e) const {
     return {};
 }
+StateKeeper LoadingFailedLastValidationRequiredRevalidation::handle(const UnloadEvent& e) const {
+    return {};  // unload is a no-op when loading already failed
+}
 
 PipelineDefinitionStateCode RetiredState::getStateCode() const {
     return code;
@@ -187,6 +207,31 @@ StateKeeper RetiredState::handle(const RetireEvent& e) const {
     throw std::logic_error(INVALID_TRANSITION_MESSAGE);
     return {};
 }
+StateKeeper RetiredState::handle(const UnloadEvent& e) const {
+    return {};  // unload is a no-op when already retired
+}
+
+PipelineDefinitionStateCode UnloadedState::getStateCode() const {
+    return code;
+}
+StateChanger<ReloadState> UnloadedState::handle(const ReloadEvent& e) const {
+    return {};  // wake-up: transition through reload path
+}
+StateChanger<RetiredState> UnloadedState::handle(const RetireEvent& e) const {
+    return {};  // config removal while unloaded
+}
+StateChanger<AvailableState> UnloadedState::handle(const ValidationPassedEvent& e) const {
+    return {};  // defensive: if validation passes directly, go available
+}
+StateKeeper UnloadedState::handle(const ValidationFailedEvent& e) const {
+    return {};
+}
+StateKeeper UnloadedState::handle(const UsedModelChangedEvent& e) const {
+    return {};
+}
+StateKeeper UnloadedState::handle(const UnloadEvent& e) const {
+    return {};  // already unloaded, idempotent
+}
 
 PipelineDefinitionStatus::PipelineDefinitionStatus(const std::string& type, const std::string& name) :
     MachineState(type, name) {}
@@ -233,6 +278,15 @@ std::tuple<ModelVersionState, ModelVersionStatusErrorCode> PipelineDefinitionSta
             ModelVersionState::END,
             ModelVersionStatusErrorCode::OK};
 
+    case PipelineDefinitionStateCode::UNLOADED:
+        // Report AVAILABLE: the graph auto-reloads on the next inference request,
+        // so health checks and routing should treat it as available. Reporting END
+        // or UNLOADING would cause clients and load-balancers to permanently
+        // exclude this servable from their pools.
+        return {
+            ModelVersionState::AVAILABLE,
+            ModelVersionStatusErrorCode::OK};
+
     default:
         return {};
     }