diff --git a/src/BUILD b/src/BUILD index 5cbf3b01e6..601f2ff830 100644 --- a/src/BUILD +++ b/src/BUILD @@ -797,6 +797,7 @@ ovms_cc_library( "//src/kfserving_api:kfserving_api_cpp", "capimodule", "//src/pull_module:hf_pull_model_module", + "//src/graph_export:graph_export", "//src/servables_config_manager_module:servablesconfigmanagermodule", "predict_request_validation_utils", # to be removed when capi has its own lib and added there @atobisze "kfs_backend_impl", diff --git a/src/cli_parser.cpp b/src/cli_parser.cpp index 9437fd9755..19787d8f33 100644 --- a/src/cli_parser.cpp +++ b/src/cli_parser.cpp @@ -712,7 +712,9 @@ void CLIParser::prepareGraph(ServerSettingsImpl& serverSettings, HFSettingsImpl& } if (result->count("source_model")) { hfSettings.sourceModel = result->operator[]("source_model").as(); - } else if (result->count("model_name")) { + } else if (result->count("model_name") && !result->count("model_path")) { + // Only use model_name as source_model when model_path is not set + // (when model_path is set, user wants to use local model without HF pull) hfSettings.sourceModel = result->operator[]("model_name").as(); } if ((result->count("weight-format") || result->count("extra_quantization_params")) && isOptimumCliDownload(hfSettings.sourceModel, hfSettings.ggufFilename)) { @@ -732,6 +734,11 @@ void CLIParser::prepareGraph(ServerSettingsImpl& serverSettings, HFSettingsImpl& if (result->count("vocoder")) hfSettings.exportSettings.vocoder = result->operator[]("vocoder").as(); hfSettings.downloadPath = result->operator[]("model_repository_path").as(); + // When --task is used with --model_path but without --pull/--source_model, + // use model_path as the model location (no HF download needed) + if (!result->count("pull") && !result->count("source_model") && result->count("model_path")) { + hfSettings.exportSettings.modelPath = result->operator[]("model_path").as(); + } if (result->count("task")) { hfSettings.task = stringToEnum(result->operator[]("task").as()); switch (hfSettings.task) { @@ -840,11 +847,14 @@ void CLIParser::prepareGraphStart(HFSettingsImpl& hfSettings, ModelsSettingsImpl // Model settings if (result->count("model_name")) { modelsSettings.modelName = result->operator[]("model_name").as(); - } else { + } else if (!hfSettings.sourceModel.empty()) { modelsSettings.modelName = hfSettings.sourceModel; } - modelsSettings.modelPath = FileSystem::joinPath({hfSettings.downloadPath, hfSettings.sourceModel}); + // Only override modelPath if it wasn't already set via --model_path + if (!result->count("model_path")) { + modelsSettings.modelPath = FileSystem::joinPath({hfSettings.downloadPath, hfSettings.sourceModel}); + } } void CLIParser::prepare(ServerSettingsImpl* serverSettings, ModelsSettingsImpl* modelsSettings) { diff --git a/src/config.cpp b/src/config.cpp index 3222e775b1..880db200be 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -149,13 +149,17 @@ bool Config::validateUserSettingsInConfigAddRemoveModel(const ModelsSettingsImpl bool Config::validate() { if (this->serverSettings.serverMode == HF_PULL_MODE || this->serverSettings.serverMode == HF_PULL_AND_START_MODE) { - if (!serverSettings.hfSettings.sourceModel.size()) { - std::cerr << "source_model parameter is required for pull mode"; - return false; - } - if (!serverSettings.hfSettings.downloadPath.size()) { - std::cerr << "model_repository_path parameter is required for pull mode"; - return false; + // When --task is used with --model_path (no HF pulling), sourceModel and downloadPath are not required + bool taskWithModelPath = this->serverSettings.serverMode == HF_PULL_AND_START_MODE && !this->modelsSettings.modelPath.empty(); + if (!taskWithModelPath) { + if (!serverSettings.hfSettings.sourceModel.size()) { + std::cerr << "source_model parameter is required for pull mode"; + return false; + } + if (!serverSettings.hfSettings.downloadPath.size()) { + std::cerr << "model_repository_path parameter is required for pull mode"; + return false; + } } if (this->serverSettings.hfSettings.task == UNKNOWN_GRAPH) { std::cerr << "Error: --task parameter not set." << std::endl; diff --git a/src/graph_export/graph_export.cpp b/src/graph_export/graph_export.cpp index b98a18a966..aa4a5a9aba 100644 --- a/src/graph_export/graph_export.cpp +++ b/src/graph_export/graph_export.cpp @@ -52,6 +52,20 @@ #endif namespace ovms { +static std::string s_inMemoryGraphContent; + +bool GraphExport::hasInMemoryGraphContent() { + return !s_inMemoryGraphContent.empty(); +} + +const std::string& GraphExport::getInMemoryGraphContent() { + return s_inMemoryGraphContent; +} + +void GraphExport::clearInMemoryGraphContent() { + s_inMemoryGraphContent.clear(); +} + static const std::string OVMS_VERSION_GRAPH_LINE = std::string("# File created with: ") + PROJECT_NAME + std::string(" ") + PROJECT_VERSION + std::string("\n"); static std::string constructModelsPath(const std::string& modelPath, const std::optional& ggufFilenameOpt) { @@ -91,7 +105,7 @@ std::string GraphExport::getDraftModelDirectoryPath(const std::string& directory } \ auto pluginConfigOpt = std::get>(pluginConfigOrStatus) -static Status createPbtxtFile(const std::string& directoryPath, const std::string& pbtxtContent) { +static Status createPbtxtFile(const std::string& directoryPath, const std::string& pbtxtContent, bool writeToFile) { #if (MEDIAPIPE_DISABLE == 0) ::mediapipe::CalculatorGraphConfig config; SPDLOG_TRACE("Generated pbtxt: {}", pbtxtContent); @@ -101,12 +115,16 @@ static Status createPbtxtFile(const std::string& directoryPath, const std::strin return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; } #endif + if (!writeToFile) { + s_inMemoryGraphContent = pbtxtContent; + return StatusCode::OK; + } // clang-format on std::string fullPath = FileSystem::joinPath({directoryPath, "graph.pbtxt"}); return FileSystem::createFileOverwrite(fullPath, pbtxtContent); } -static Status createTextGenerationGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) { +static Status createTextGenerationGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) { if (!std::holds_alternative(hfSettings.graphSettings)) { SPDLOG_ERROR("Graph options not initialized for text generation."); return StatusCode::INTERNAL_ERROR; @@ -198,10 +216,10 @@ static Status createTextGenerationGraphTemplate(const std::string& directoryPath } } })"; - return createPbtxtFile(directoryPath, oss.str()); + return createPbtxtFile(directoryPath, oss.str(), writeToFile); } -static Status createRerankGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) { +static Status createRerankGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) { if (!std::holds_alternative(hfSettings.graphSettings)) { SPDLOG_ERROR("Graph options not initialized for reranking."); return StatusCode::INTERNAL_ERROR; @@ -242,10 +260,10 @@ node { } } })"; - return createPbtxtFile(directoryPath, oss.str()); + return createPbtxtFile(directoryPath, oss.str(), writeToFile); } -static Status createEmbeddingsGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) { +static Status createEmbeddingsGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) { if (!std::holds_alternative(hfSettings.graphSettings)) { SPDLOG_ERROR("Graph options not initialized for embeddings."); return StatusCode::INTERNAL_ERROR; @@ -289,10 +307,10 @@ node { oss << R"(} } })"; - return createPbtxtFile(directoryPath, oss.str()); + return createPbtxtFile(directoryPath, oss.str(), writeToFile); } -static Status createTextToSpeechGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) { +static Status createTextToSpeechGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) { if (!std::holds_alternative(hfSettings.graphSettings)) { SPDLOG_ERROR("Graph options not initialized for speech generation."); return StatusCode::INTERNAL_ERROR; @@ -339,11 +357,15 @@ node { } #endif // clang-format on + if (!writeToFile) { + s_inMemoryGraphContent = oss.str(); + return StatusCode::OK; + } std::string fullPath = FileSystem::joinPath({directoryPath, "graph.pbtxt"}); return FileSystem::createFileOverwrite(fullPath, oss.str()); } -static Status createSpeechToTextGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) { +static Status createSpeechToTextGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) { if (!std::holds_alternative(hfSettings.graphSettings)) { SPDLOG_ERROR("Graph options not initialized for speech to text."); return StatusCode::INTERNAL_ERROR; @@ -389,11 +411,15 @@ node { } #endif // clang-format on + if (!writeToFile) { + s_inMemoryGraphContent = oss.str(); + return StatusCode::OK; + } std::string fullPath = FileSystem::joinPath({directoryPath, "graph.pbtxt"}); return FileSystem::createFileOverwrite(fullPath, oss.str()); } -static Status createImageGenerationGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) { +static Status createImageGenerationGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) { if (!std::holds_alternative(hfSettings.graphSettings)) { SPDLOG_ERROR("Graph options not initialized for image generation."); return StatusCode::INTERNAL_ERROR; @@ -473,13 +499,13 @@ node: { } )"; // clang-format on - return createPbtxtFile(directoryPath, oss.str()); + return createPbtxtFile(directoryPath, oss.str(), writeToFile); } GraphExport::GraphExport() { } -Status GraphExport::createServableConfig(const std::string& directoryPath, const HFSettingsImpl& hfSettings) { +Status GraphExport::createServableConfig(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) { if (directoryPath.empty()) { SPDLOG_ERROR("Directory path empty: {}", directoryPath); return StatusCode::PATH_INVALID; @@ -502,17 +528,17 @@ Status GraphExport::createServableConfig(const std::string& directoryPath, const } } if (hfSettings.task == TEXT_GENERATION_GRAPH) { - return createTextGenerationGraphTemplate(directoryPath, hfSettings); + return createTextGenerationGraphTemplate(directoryPath, hfSettings, writeToFile); } else if (hfSettings.task == EMBEDDINGS_GRAPH) { - return createEmbeddingsGraphTemplate(directoryPath, hfSettings); + return createEmbeddingsGraphTemplate(directoryPath, hfSettings, writeToFile); } else if (hfSettings.task == RERANK_GRAPH) { - return createRerankGraphTemplate(directoryPath, hfSettings); + return createRerankGraphTemplate(directoryPath, hfSettings, writeToFile); } else if (hfSettings.task == IMAGE_GENERATION_GRAPH) { - return createImageGenerationGraphTemplate(directoryPath, hfSettings); + return createImageGenerationGraphTemplate(directoryPath, hfSettings, writeToFile); } else if (hfSettings.task == TEXT_TO_SPEECH_GRAPH) { - return createTextToSpeechGraphTemplate(directoryPath, hfSettings); + return createTextToSpeechGraphTemplate(directoryPath, hfSettings, writeToFile); } else if (hfSettings.task == SPEECH_TO_TEXT_GRAPH) { - return createSpeechToTextGraphTemplate(directoryPath, hfSettings); + return createSpeechToTextGraphTemplate(directoryPath, hfSettings, writeToFile); } else if (hfSettings.task == UNKNOWN_GRAPH) { SPDLOG_ERROR("Graph options not initialized."); return StatusCode::INTERNAL_ERROR; diff --git a/src/graph_export/graph_export.hpp b/src/graph_export/graph_export.hpp index e6f9fdcbef..4b44193610 100644 --- a/src/graph_export/graph_export.hpp +++ b/src/graph_export/graph_export.hpp @@ -27,9 +27,13 @@ class Status; class GraphExport { public: GraphExport(); - Status createServableConfig(const std::string& directoryPath, const HFSettingsImpl& graphSettings); + Status createServableConfig(const std::string& directoryPath, const HFSettingsImpl& graphSettings, bool writeToFile = true); static std::variant, Status> createPluginString(const ExportSettings& exportSettings); static std::string getDraftModelDirectoryName(std::string draftModel); static std::string getDraftModelDirectoryPath(const std::string& directoryPath, const std::string& draftModel); + + static bool hasInMemoryGraphContent(); + static const std::string& getInMemoryGraphContent(); + static void clearInMemoryGraphContent(); }; } // namespace ovms diff --git a/src/mediapipe_internal/BUILD b/src/mediapipe_internal/BUILD index a01a96b807..35456ec281 100644 --- a/src/mediapipe_internal/BUILD +++ b/src/mediapipe_internal/BUILD @@ -79,6 +79,7 @@ ovms_cc_library( "//src:libovms_servable_name_checker", "//src/metrics:libovms_metric_provider", "//src/filesystem:libovmsfilesystem", + "//src/graph_export:graph_export", "//src:libovms_version", "//src:libovms_execution_context", "//src:libovmstimer", diff --git a/src/mediapipe_internal/mediapipegraphconfig.cpp b/src/mediapipe_internal/mediapipegraphconfig.cpp index d767942535..200de9c289 100644 --- a/src/mediapipe_internal/mediapipegraphconfig.cpp +++ b/src/mediapipe_internal/mediapipegraphconfig.cpp @@ -25,6 +25,7 @@ #include #include "src/filesystem/filesystem.hpp" +#include "src/graph_export/graph_export.hpp" #include "../status.hpp" namespace ovms { @@ -129,6 +130,10 @@ Status MediapipeGraphConfig::parseNode(const rapidjson::Value& v) { } void MediapipeGraphConfig::logGraphConfigContent() const { + if (GraphExport::hasInMemoryGraphContent()) { + SPDLOG_DEBUG("Content of in-memory graph config:\n{}", GraphExport::getInMemoryGraphContent()); + return; + } std::ifstream fileStream(this->graphPath); if (!fileStream.is_open()) { SPDLOG_ERROR("Failed to open file: {}", this->graphPath); diff --git a/src/mediapipe_internal/mediapipegraphdefinition.cpp b/src/mediapipe_internal/mediapipegraphdefinition.cpp index d26806edab..b66c55e7ad 100644 --- a/src/mediapipe_internal/mediapipegraphdefinition.cpp +++ b/src/mediapipe_internal/mediapipegraphdefinition.cpp @@ -26,6 +26,7 @@ #include "../execution_context.hpp" #include "src/filesystem/filesystem.hpp" +#include "src/graph_export/graph_export.hpp" #include "src/metrics/metric.hpp" #include "../model_metric_reporter.hpp" #include "../ov_utils.hpp" @@ -60,6 +61,13 @@ const tensor_map_t MediapipeGraphDefinition::getOutputsInfo() const { } Status MediapipeGraphDefinition::validateForConfigFileExistence() { + if (GraphExport::hasInMemoryGraphContent()) { + const std::string& content = GraphExport::getInMemoryGraphContent(); + this->chosenConfig = content; + this->mgconfig.setCurrentGraphPbTxtMD5(ovms::FileSystem::getStringMD5(content)); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Using in-memory graph content for mediapipe graph definition: {}", this->getName()); + return StatusCode::OK; + } std::ifstream ifs(this->mgconfig.getGraphPath()); if (!ifs.is_open()) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to open mediapipe graph definition: {}, file: {}\n", this->getName(), this->mgconfig.getGraphPath()); diff --git a/src/modelmanager.cpp b/src/modelmanager.cpp index e57f29eb3a..f94f0ba60a 100644 --- a/src/modelmanager.cpp +++ b/src/modelmanager.cpp @@ -58,6 +58,7 @@ #include "dags/pipelinedefinition.hpp" #include "filesystem/filesystem.hpp" #include "filesystem/filesystemfactory.hpp" +#include "graph_export/graph_export.hpp" #include "logging.hpp" #if (MEDIAPIPE_DISABLE == 0) #include "mediapipe_internal/mediapipefactory.hpp" @@ -229,7 +230,8 @@ Status ModelManager::startFromConfig() { std::vector mediapipesInConfigFile; std::ifstream ifs(mpConfig.getGraphPath()); - if (ifs.is_open()) { + bool graphAvailable = ifs.is_open() || GraphExport::hasInMemoryGraphContent(); + if (graphAvailable) { // Single model with graph.pbtxt, check if user passed model unsupported model parameters in cmd arguments status = ModelManager::validateUserSettingsInSingleModelCliGraphStart(config.getModelSettings()); if (!status.ok()) @@ -473,10 +475,13 @@ bool ModelManager::CheckStartFromGraph(std::string inputPath, MediapipeGraphConf if (ifs.is_open()) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Graph: {} path: {} exists", mpConfig.getGraphName(), mpConfig.getGraphPath()); return true; - } else { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Graph: {} path: {} does not exist", mpConfig.getGraphName(), mpConfig.getGraphPath()); - return false; } + if (GraphExport::hasInMemoryGraphContent()) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Graph: {} using in-memory graph content", mpConfig.getGraphName()); + return true; + } + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Graph: {} path: {} does not exist", mpConfig.getGraphName(), mpConfig.getGraphPath()); + return false; } Status ModelManager::validateUserSettingsInSingleModelCliGraphStart(const ModelsSettingsImpl& modelsSettings) { diff --git a/src/pull_module/hf_pull_model_module.cpp b/src/pull_module/hf_pull_model_module.cpp index b73cad6638..9a3cedb7ec 100644 --- a/src/pull_module/hf_pull_model_module.cpp +++ b/src/pull_module/hf_pull_model_module.cpp @@ -151,11 +151,16 @@ Status HfPullModelModule::clone() const { } GraphExport graphExporter; - status = graphExporter.createServableConfig(graphDirectory, this->hfSettings); + bool writeToFile = (ovms::Config::instance().getServerSettings().serverMode == HF_PULL_MODE); + status = graphExporter.createServableConfig(graphDirectory, this->hfSettings, writeToFile); if (!status.ok()) { return status; } - std::cout << "Graph: graph.pbtxt created in: " << graphDirectory << std::endl; + if (writeToFile) { + std::cout << "Graph: graph.pbtxt created in: " << graphDirectory << std::endl; + } else { + std::cout << "Graph: graph.pbtxt content stored in memory" << std::endl; + } return StatusCode::OK; } diff --git a/src/server.cpp b/src/server.cpp index 4a89a304b9..0148eb3732 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -53,6 +53,7 @@ #include "capi_frontend/server_settings.hpp" #include "cli_parser.hpp" #include "config.hpp" +#include "graph_export/graph_export.hpp" #include "grpcservermodule.hpp" #include "http_server.hpp" #include "httpservermodule.hpp" @@ -378,16 +379,29 @@ Status Server::startModules(ovms::Config& config) { return status; } if (config.getServerSettings().serverMode == HF_PULL_MODE || config.getServerSettings().serverMode == HF_PULL_AND_START_MODE) { - INSERT_MODULE(HF_MODEL_PULL_MODULE_NAME, it); - START_MODULE(it); - if (!status.ok()) { - return status; + bool needsHfPull = !config.getServerSettings().hfSettings.sourceModel.empty(); + if (needsHfPull) { + INSERT_MODULE(HF_MODEL_PULL_MODULE_NAME, it); + START_MODULE(it); + if (!status.ok()) { + return status; + } + auto hfModule = dynamic_cast(it->second.get()); + status = hfModule->clone(); + // Return from modules only in --pull mode or error, otherwise start the rest of modules + if (config.getServerSettings().serverMode == HF_PULL_MODE || !status.ok()) + return status; + } else { + // --task with --model_path: create graph in memory without HF download + GraphExport graphExporter; + const auto& hfSettings = config.getServerSettings().hfSettings; + status = graphExporter.createServableConfig(config.modelPath(), hfSettings, false); + if (!status.ok()) { + SPDLOG_ERROR("Failed to create in-memory graph config: {}", status.string()); + return status; + } + SPDLOG_INFO("Graph config created in memory from model_path: {}", config.modelPath()); } - auto hfModule = dynamic_cast(it->second.get()); - status = hfModule->clone(); - // Return from modules only in --pull mode or error, otherwise start the rest of modules - if (config.getServerSettings().serverMode == HF_PULL_MODE || !status.ok()) - return status; } #if (PYTHON_DISABLE == 0) diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index 2d3f80f2e2..3f3082fd1e 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -4532,3 +4532,46 @@ TEST_F(IsolatedServableTests, PromtSizeBetweenDefaultAndNonDefaultMaxPromptLenNP } // TODO: Add missing tests for reading max prompt len property from configuration + +class LLMStartWithTaskParameter : public ::testing::Test { +protected: + static std::unique_ptr t; + std::string modelDir = getGenericFullPathForSrcTest("/ovms/src/test/llm_testing/HuggingFaceTB/SmolLM2-360M-Instruct"); + std::string graphPath = modelDir + "/graph.pbtxt"; + std::string graphPathRenamed = modelDir + "/graph.pbtxt.bak"; + + void SetUp() override { + // Rename graph.pbtxt so it's not used from file + if (std::filesystem::exists(graphPath)) { + std::filesystem::rename(graphPath, graphPathRenamed); + } + // Set model directory to readonly to ensure no file writes happen + SetReadonlyFileAttributeFromDir(modelDir); + } + void TearDown() override { + ovms::Server& server = ovms::Server::instance(); + server.setShutdownRequest(1); + if (t && t->joinable()) + t->join(); + server.setShutdownRequest(0); + // Restore write permissions and rename graph.pbtxt back + RemoveReadonlyFileAttributeFromDir(modelDir); + if (std::filesystem::exists(graphPathRenamed)) { + std::filesystem::rename(graphPathRenamed, graphPath); + } + } +}; + +std::unique_ptr LLMStartWithTaskParameter::t = nullptr; + +TEST_F(LLMStartWithTaskParameter, StartWithModelPathAndTaskWithoutGraphFile) { + std::string port = "9173"; + ovms::Server& server = ovms::Server::instance(); + ::SetUpServer(t, server, port, + "/ovms/src/test/llm_testing/HuggingFaceTB/SmolLM2-360M-Instruct", + "SmolLM2", + 60, + "text_generation"); + ASSERT_EQ(server.getModuleState(ovms::SERVABLE_MANAGER_MODULE_NAME), ovms::ModuleState::INITIALIZED); + ASSERT_FALSE(std::filesystem::exists(graphPath)) << "graph.pbtxt should not be created when using --task with --model_path"; +} diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index d622f9882d..3c0a3b0bc8 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -33,6 +33,7 @@ #include "src/test/test_with_temp_dir.hpp" #include "src/filesystem/filesystem.hpp" #include "src/pull_module/hf_pull_model_module.hpp" +#include "src/graph_export/graph_export.hpp" #include "src/pull_module/libgit2.hpp" #include "src/pull_module/optimum_export.hpp" #include "src/servables_config_manager_module/listmodels.hpp" @@ -342,7 +343,6 @@ TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStart) { // EnvGuard guard; // guard.set("HF_ENDPOINT", "https://modelscope.cn"); // guard.set("HF_ENDPOINT", "https://hf-mirror.com"); - this->filesToPrintInCaseOfFailure.emplace_back("graph.pbtxt"); this->filesToPrintInCaseOfFailure.emplace_back("config.json"); std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); @@ -354,10 +354,11 @@ TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStart) { std::string graphPath = ovms::FileSystem::appendSlash(basePath) + "graph.pbtxt"; ASSERT_EQ(std::filesystem::exists(modelPath), true) << modelPath; - ASSERT_EQ(std::filesystem::exists(graphPath), true) << graphPath; + // In HF_PULL_AND_START_MODE, graph.pbtxt is stored in memory, not written to file + ASSERT_EQ(std::filesystem::exists(graphPath), false) << "graph.pbtxt should not be created in pull-and-start mode"; ASSERT_EQ(std::filesystem::file_size(modelPath), 52417240); - std::string graphContents = GetFileContents(graphPath); - + ASSERT_TRUE(ovms::GraphExport::hasInMemoryGraphContent()); + std::string graphContents = ovms::GraphExport::getInMemoryGraphContent(); ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; } @@ -407,7 +408,6 @@ TEST_F(HfDownloaderPullHfModel, ModelOutOfOvOrg) { TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStartModelOutsideOvOrg) { SKIP_AND_EXIT_IF_NOT_RUNNING_UNSTABLE(); // CVS-180127 - this->filesToPrintInCaseOfFailure.emplace_back("graph.pbtxt"); this->filesToPrintInCaseOfFailure.emplace_back("config.json"); std::string modelName = "AIFunOver/SmolLM2-360M-Instruct-openvino-4bit"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); @@ -419,9 +419,10 @@ TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStartModelOutsideOvOrg) { std::string graphPath = ovms::FileSystem::appendSlash(basePath) + "graph.pbtxt"; ASSERT_EQ(std::filesystem::exists(modelPath), true) << modelPath; - ASSERT_EQ(std::filesystem::exists(graphPath), true) << graphPath; - std::string graphContents = GetFileContents(graphPath); - + // In HF_PULL_AND_START_MODE, graph.pbtxt is stored in memory, not written to file + ASSERT_EQ(std::filesystem::exists(graphPath), false) << "graph.pbtxt should not be created in pull-and-start mode"; + ASSERT_TRUE(ovms::GraphExport::hasInMemoryGraphContent()); + std::string graphContents = ovms::GraphExport::getInMemoryGraphContent(); ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; } @@ -1028,12 +1029,12 @@ TEST(ServerModulesBehaviorTests, PullAndStartModeErrorAndExpectFailAndNoOtherMod DefaultEmptyValuesConfig config; config.getServerSettings().serverMode = ovms::HF_PULL_AND_START_MODE; auto retCode = server.startModules(config); - // Empty config.getServerSettings().hfSettings.downloadPath - // [error][libit2.cpp:336] Libgit2 clone error: 6 message: cannot pick working directory for non-bare repository that isn't a '.git' directory + // Empty sourceModel: takes task+model_path path, but model_path is empty + // -> GraphExport::createServableConfig fails with PATH_INVALID EXPECT_TRUE(!retCode.ok()) << retCode.string(); serverGuard = std::make_unique(server); - EXPECT_TRUE(server.getModule(ovms::HF_MODEL_PULL_MODULE_NAME) != nullptr); - ASSERT_EQ(server.getModule(ovms::HF_MODEL_PULL_MODULE_NAME)->getState(), ovms::ModuleState::INITIALIZED); + // When sourceModel is empty, HF pull module is not inserted + ASSERT_EQ(server.getModule(ovms::HF_MODEL_PULL_MODULE_NAME), nullptr); ASSERT_EQ(server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME), nullptr); ASSERT_EQ(server.getModule(ovms::SERVABLES_CONFIG_MANAGER_MODULE_NAME), nullptr); } diff --git a/src/test/test_utils.cpp b/src/test/test_utils.cpp index 5c21e15158..0e35bc028f 100644 --- a/src/test/test_utils.cpp +++ b/src/test/test_utils.cpp @@ -889,6 +889,25 @@ void SetUpServer(std::unique_ptr& t, ovms::Server& server, std::str EnsureServerStartedWithTimeout(server, timeoutSeconds); } +void SetUpServer(std::unique_ptr& t, ovms::Server& server, std::string& port, const char* modelPath, const char* modelName, int timeoutSeconds, const char* task) { + server.setShutdownRequest(0); + randomizeAndEnsureFree(port); + char* argv[] = {(char*)"ovms", + (char*)"--model_name", + (char*)modelName, + (char*)"--model_path", + (char*)getGenericFullPathForSrcTest(modelPath).c_str(), + (char*)"--port", + (char*)port.c_str(), + (char*)"--task", + (char*)task}; + int argc = 9; + t.reset(new std::thread([&argc, &argv, &server]() { + EXPECT_EQ(EXIT_SUCCESS, server.start(argc, argv)); + })); + EnsureServerStartedWithTimeout(server, timeoutSeconds); +} + std::shared_ptr createTensorInfoCopyWithPrecision(std::shared_ptr src, ovms::Precision newPrecision) { return std::make_shared( src->getName(), diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index 7f5c80202d..8fb0885cda 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -796,6 +796,7 @@ void SetUpServerForDownloadAndStart(std::unique_ptr& t, ovms::Serve */ void SetUpServer(std::unique_ptr& t, ovms::Server& server, std::string& port, const char* configPath, int timeoutSeconds = SERVER_START_FROM_CONFIG_TIMEOUT_SECONDS, std::string apiKeyFile = ""); void SetUpServer(std::unique_ptr& t, ovms::Server& server, std::string& port, const char* modelPath, const char* modelName, int timeoutSeconds = SERVER_START_FROM_CONFIG_TIMEOUT_SECONDS); +void SetUpServer(std::unique_ptr& t, ovms::Server& server, std::string& port, const char* modelPath, const char* modelName, int timeoutSeconds, const char* task); class ConstructorEnabledConfig : public ovms::Config { public: