diff --git a/src/BUILD b/src/BUILD index b9810a0e07..54f8605c06 100644 --- a/src/BUILD +++ b/src/BUILD @@ -206,6 +206,12 @@ ovms_cc_library( deps = ["libovmsstring_utils"], visibility = ["//visibility:public",], ) +ovms_cc_library( + name = "libovms_shutdown_state", + hdrs = ["shutdown_state.hpp"], + srcs = ["shutdown_state.cpp"], + visibility = ["//visibility:public",], +) ovms_cc_library( name = "prediction_service_utils", hdrs = ["prediction_service_utils.hpp"], @@ -817,6 +823,7 @@ ovms_cc_library( ], }) + [ "cpp_headers", + "libovms_shutdown_state", "ovms_header", "capi_backend_impl", "libovms_capi_servable_metadata", diff --git a/src/pull_module/BUILD b/src/pull_module/BUILD index 538e5d318d..cf8fd5eeba 100644 --- a/src/pull_module/BUILD +++ b/src/pull_module/BUILD @@ -45,6 +45,7 @@ ovms_cc_library( deps = [ "@libgit2_engine//:libgit2_engine", "@ovms//src:libovmslogging", + "@ovms//src:libovms_shutdown_state", "@ovms//src/filesystem:libovmsfilesystem", "@ovms//src/filesystem:libovmslocalfilesystem", "@ovms//src:libovmsstatus", diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 8d5cbf8d4a..d3d7743ae1 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -16,11 +16,14 @@ #include "libgit2.hpp" #include +#include #include #include +#include #include #include #include +#include #include #include #include @@ -39,6 +42,7 @@ #include "src/filesystem/filesystem.hpp" #include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" +#include "../shutdown_state.hpp" #include "../stringutils.hpp" #include "../status.hpp" @@ -51,15 +55,147 @@ #endif #endif +/* Exported cancellation accessors in libgit2 patch – used to abort + * ongoing LFS downloads from OVMS. */ +extern "C" { +void git_lfs_cancel_set(int value); +int git_lfs_cancel_get(void); +} + namespace ovms { namespace fs = std::filesystem; +namespace libgit2 { + +/** + * Builds the side-marker path used to track interrupted LFS operations. + * The marker lives next to the repository directory as .lfswip + * (i.e. it is a SIBLING of the repo directory, not a file inside it). + * Example: repositoryPath "/opt/models/OV/model1" produces + * "/opt/models/OV/model1.lfswip". + */ +fs::path getLfsWipMarkerPath(const std::string& repositoryPath) { + const fs::path repository = fs::path(repositoryPath); + return repository.parent_path() / (repository.filename().string() + ".lfswip"); +} + +/** + * Creates or truncates the .lfswip marker file for the target repository. + * Marker presence indicates clone/download interruption handling is in progress. + * Returns true on success, false on filesystem error. + */ +bool createLfsWipMarker(const std::string& repositoryPath) { + const fs::path markerPath = getLfsWipMarkerPath(repositoryPath); + std::error_code ec; + if (!markerPath.parent_path().empty()) { + fs::create_directories(markerPath.parent_path(), ec); + } + std::ofstream marker(markerPath, std::ios::trunc); + return static_cast(marker); +} + +bool hasLfsWipMarker(const std::string& repositoryPath) { + const fs::path markerPath = getLfsWipMarkerPath(repositoryPath); + std::error_code ec; + return fs::exists(markerPath, ec) && fs::is_regular_file(markerPath, ec); +} + +void removeLfsWipMarker(const std::string& repositoryPath) { + const fs::path markerPath = getLfsWipMarkerPath(repositoryPath); + std::error_code ec; + if (!fs::remove(markerPath, ec) && ec) { + SPDLOG_WARN("Failed to remove .lfswip marker {}: {}", markerPath.string(), ec.message()); + } +} + +} // namespace libgit2 + namespace { std::atomic g_activeLibgit2Guards{0}; + +/** + * Sets the LFS (Large File Storage) cancellation flag. + * This signal is checked by the patched libgit2 library to abort ongoing LFS downloads. + */ +static void setLfsCancelRequested(int value) { + git_lfs_cancel_set(value != 0 ? 1 : 0); +} + +#define RETURN_IF_OVMS_CLONE_CANCELLED() \ + do { \ + if (libgit2::isCloneCancellationRequestedFromServer()) { \ + setLfsCancelRequested(1); \ + return -1; \ + } \ + } while (0) + +int checkCloneCancellationTransferProgressCallback(const git_indexer_progress* stats, void* payload) { + (void)stats; + (void)payload; + RETURN_IF_OVMS_CLONE_CANCELLED(); + return 0; +} + +int checkCloneCancellationSidebandProgressCallback(const char* str, int len, void* payload) { + (void)str; + (void)len; + (void)payload; + RETURN_IF_OVMS_CLONE_CANCELLED(); + return 0; +} + +int checkCloneCancellationUpdateTipsCallback(const char* refname, const git_oid* a, const git_oid* b, void* payload) { + (void)refname; + (void)a; + (void)b; + (void)payload; + RETURN_IF_OVMS_CLONE_CANCELLED(); + return 0; +} + +/** + * Callback fired during git checkout to notify about files being modified in the working tree. + * Raised when files are created, updated, or deleted during clone or checkout operations. + * + * @param why Notification type (GIT_CHECKOUT_NOTIFY_*) indicating what changed. + * @param path Path of the file being checked out in the working directory. + * @param baseline Diff file info at HEAD (unused). + * @param target Diff file info being checked out (unused). + * @param workdir Current working directory file state (unused). + * @param payload User-supplied payload pointer (unused). + * @return -1 if cancellation requested (aborts checkout), 0 otherwise. + * @note Works on the git repository working directory; modifies local filesystem. + */ +int checkCloneCancellationCheckoutNotifyCallback(git_checkout_notify_t why, + const char* path, + const git_diff_file* baseline, + const git_diff_file* target, + const git_diff_file* workdir, + void* payload) { + (void)why; + (void)path; + (void)baseline; + (void)target; + (void)workdir; + (void)payload; + RETURN_IF_OVMS_CLONE_CANCELLED(); + return 0; +} } // namespace -// Callback for clone authentication - will be used when password is not set in repo_url -// Does not work with LFS download as it requires additional authentication when password is not set in repository url +/** + * Callback for acquiring authentication credentials during git clone operations. + * Attempts to use HF_TOKEN environment variable for Hugging Face authentication. + * + * @param out [out] Pointer to git_credential structure to fill with credentials. + * @param url The repository URL requiring authentication. + * @param username_from_url Username parsed from URL (if any). + * @param allowed_types Bitmask of credential types supported by the remote. + * @param payload User-supplied payload pointer (unused). + * @return 0 on success, -1 on error, 1 if credential type not supported. + * @note Connects to internet for authentication; reads HF_TOKEN environment variable. + * @note Limited LFS support: LFS downloads may fail if HF_TOKEN is not set. + */ int cred_acquire_cb(git_credential** out, const char* url, const char* username_from_url, @@ -112,6 +248,15 @@ int cred_acquire_cb(git_credential** out, } \ } while (0) +/** + * Initializes the libgit2 library with server connection settings. + * Sets timeouts for server connections and LFS operations, and configures SSL certificate locations. + * Maintains a reference count of active guards to ensure single initialization. + * + * @param opts Configuration options including server timeouts and SSL certificate path. + * @note Thread-safe initialization; uses atomic counter to track active guards. + * @note Connects to internet: configures timeouts for remote git/LFS operations. + */ Libgt2InitGuard::Libgt2InitGuard(const Libgit2Options& opts) { SPDLOG_DEBUG("Initializing libgit2"); this->status = git_libgit2_init(); @@ -202,6 +347,12 @@ class GitRepositoryGuard { git_repository* repo = nullptr; int git_error_class = 0; + /** + * Opens a git repository at the specified filesystem path. + * + * @param path Absolute or relative path to the git repository directory. + * @note Works on specific git repository location (searches for .git directory). + */ GitRepositoryGuard(const std::string& path) { int error = git_repository_open_ext(&repo, path.c_str(), 0, nullptr); if (error < 0) { @@ -219,6 +370,9 @@ class GitRepositoryGuard { } } + /** + * Destructor: safely releases the git repository resource. + */ ~GitRepositoryGuard() { if (repo) { git_repository_free(repo); @@ -249,6 +403,16 @@ class GitRepositoryGuard { } }; +/** + * Verifies the cleanliness of a git repository after clone or download operations. + * Checks for staged changes, unstaged modifications, untracked files, and merge conflicts. + * + * @param checkUntracked If true, also check for untracked files (more expensive on large repos). + * @return StatusCode::OK if repository is clean, StatusCode::HF_GIT_STATUS_UNCLEAN if changes exist, + * or error status if repository check fails. + * @note Works on specific git repository (at downloadPath);scans the entire working directory. + * @note Part of the git domain: analyzes HEAD state, index, and working directory. + */ Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { if (g_activeLibgit2Guards.load(std::memory_order_relaxed) <= 0) { return StatusCode::HF_GIT_LIBGIT2_NOT_INITIALIZED; @@ -335,19 +499,22 @@ Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { return StatusCode::OK; } -#define CHECK(call) \ - do { \ - int _err = (call); \ - if (_err < 0) { \ - const git_error* e = git_error_last(); \ - fprintf(stderr, "[ERROR] %d: %s (%s:%d)\n", _err, e && e->message ? e->message : "no message", __FILE__, __LINE__); \ - return; \ - } \ - } while (0) - namespace libgit2 { -// Trim ASCII leading/trailing whitespace in a locale-independent way. -// This keeps non-ASCII bytes (e.g. UTF-8 continuation bytes) untouched. + +bool isCloneCancellationRequestedFromServer() { + if (isSignalShutdownRequested()) { + processSignalShutdownRequest(); + setLfsCancelRequested(getShutdownRequestValue()); + } + return getShutdownRequestValue() != 0; +} + +bool hasLfsErrorFile(const std::string& repositoryRootPath) { + const fs::path errorFilePath = fs::path(repositoryRootPath) / "lfs_error.txt"; + std::error_code ec; + return fs::exists(errorFilePath, ec) && fs::is_regular_file(errorFilePath, ec); +} + void rtrimCrLfWhitespace(std::string& s) { auto isAsciiWhitespace = [](unsigned char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'; @@ -362,7 +529,6 @@ void rtrimCrLfWhitespace(std::string& s) { s.erase(0, i); } -// Case-insensitive substring search: returns true if 'needle' is found in 'hay' bool containsCaseInsensitive(const std::string& hay, const std::string& needle) { auto toLower = [](std::string v) { std::transform(v.begin(), v.end(), v.begin(), @@ -374,8 +540,6 @@ bool containsCaseInsensitive(const std::string& hay, const std::string& needle) return hayLower.find(needleLower) != std::string::npos; } -// Read at most the first 3 lines of a file, with a per-line cap to avoid huge reads. -// Returns true if successful (even if <3 lines exist; vector will just be shorter). bool readFirstThreeLines(const std::filesystem::path& p, std::vector& out, size_t maxLineBytes) { out.clear(); if (maxLineBytes == 0) @@ -423,8 +587,6 @@ bool readFirstThreeLines(const std::filesystem::path& p, std::vector "version", line2 -> "oid", line3 -> "size" (case-insensitive). bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { std::error_code ec; if (!fs::is_regular_file(p, ec)) @@ -442,7 +604,43 @@ bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { containsCaseInsensitive(lines[2], "size"); } -// Helper: make path relative to base (best-effort, non-throwing). +bool hasLfsKeywordsFirst3Positional(std::string_view content) { + std::array lines; + size_t lineIndex = 0; + size_t position = 0; + + while (lineIndex < lines.size() && position < content.size()) { + size_t next = content.find_first_of("\r\n", position); + auto line = content.substr(position, next == std::string_view::npos ? content.size() - position : next - position); + lines[lineIndex] = std::string(line); + rtrimCrLfWhitespace(lines[lineIndex]); + ++lineIndex; + + if (next == std::string_view::npos) + break; + position = next + 1; + if (next + 1 < content.size() && content[next] == '\r' && content[next + 1] == '\n') + ++position; + } + + if (lineIndex < lines.size()) + return false; + + return containsCaseInsensitive(lines[0], "version") && + containsCaseInsensitive(lines[1], "oid") && + containsCaseInsensitive(lines[2], "size"); +} + +bool blobHasLfsKeywordsFirst3Positional(git_blob* blob) { + if (!blob) + return false; + const void* raw = git_blob_rawcontent(blob); + if (!raw) + return false; + size_t rawSize = git_blob_rawsize(blob); + return hasLfsKeywordsFirst3Positional(std::string_view(static_cast(raw), rawSize)); +} + fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { // Root-like paths (e.g. "/" or "C:\\") have no filename component. // Keep them unchanged instead of converting to a cwd/base-dependent ".." chain. @@ -466,7 +664,16 @@ fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { return path; } -// Find all files under 'directory' that satisfy the first-3-lines LFS keyword check. Default: bool recursive = true +/** + * Recursively/non-recursively searches a directory for files matching the LFS pointer format. + * Skips .git directories to avoid searching the repository metadata. + * + * @param directory Root directory to search for LFS-like files. + * @param recursive If true, search subdirectories; if false, only search directory root. + * @return Vector of relative paths (relative to directory) of files matching LFS format. + * @note Works on local filesystem; scans directory tree (may be expensive on large repos). + * @note Part of the git LFS domain: identifies LFS pointer files in the working directory. + */ std::vector findLfsLikeFiles(const std::string& directory, bool recursive) { std::vector matches; std::error_code ec; @@ -497,230 +704,722 @@ std::vector findLfsLikeFiles(const std::string& directory, bool recurs } return matches; } -} // namespace libgit2 -// pick the right entry pointer type for your libgit2 -#if defined(GIT_LIBGIT2_VER_MAJOR) -// libgit2 ≥ 1.0 generally has const-correct free() (accepts const*) -using git_tree_entry_ptr = const git_tree_entry*; -#else -using git_tree_entry_ptr = git_tree_entry*; -#endif +struct MissingLfsPathsWalkPayload { + git_repository* repo; + fs::path worktreeRoot; + std::set* matches; +}; -// Single guard that owns all temporaries used in resumeLfsDownloadForFile -struct GitScope { - git_object* tree_obj = nullptr; // owns the tree as a generic git_object - git_tree_entry_ptr entry = nullptr; // owns the entry - git_blob* blob = nullptr; // owns the blob - git_buf out = GIT_BUF_INIT; // owns the buffer - - GitScope() = default; - ~GitScope() { cleanup(); } - - GitScope(const GitScope&) = delete; - GitScope& operator=(const GitScope&) = delete; - - GitScope(GitScope&& other) noexcept : - tree_obj(other.tree_obj), - entry(other.entry), - blob(other.blob), - out(other.out) { - other.tree_obj = nullptr; - other.entry = nullptr; - other.blob = nullptr; - other.out = GIT_BUF_INIT; - } - GitScope& operator=(GitScope&& other) noexcept { - if (this != &other) { - cleanup(); - tree_obj = other.tree_obj; - entry = other.entry; - blob = other.blob; - out = other.out; - other.tree_obj = nullptr; - other.entry = nullptr; - other.blob = nullptr; - other.out = GIT_BUF_INIT; +struct MissingNonLfsPathsWalkPayload { + git_repository* repo; + fs::path worktreeRoot; + std::set* matches; +}; + +/** + * Tree-walk callback that collects paths from HEAD tree that have LFS pointers but missing worktree files. + * Used to identify LFS files that need to be downloaded/resumed during clone recovery. + * + * @param root Current directory path in the tree traversal (accumulates parent directories). + * @param entry Current git_tree_entry being visited. + * @param payloadPtr Pointer to MissingLfsPathsWalkPayload containing repo, worktree path, and output set. + * @return 0 to continue tree walk; non-zero aborts the walk. + * @note Works on git repository object database (ODB) and working directory. + * @note Part of the git domain: walks the HEAD commit tree to find tracked LFS files. + */ +int collectMissingLfsPathsFromHeadTreeCb(const char* root, const git_tree_entry* entry, void* payloadPtr) { + auto* payload = reinterpret_cast(payloadPtr); + if (!payload || git_tree_entry_type(entry) != GIT_OBJECT_BLOB) + return 0; + + fs::path relativePath = fs::path(root ? root : "") / git_tree_entry_name(entry); + std::error_code ec; + if (fs::exists(payload->worktreeRoot / relativePath, ec)) + return 0; + + git_blob* blob = nullptr; + if (git_blob_lookup(&blob, payload->repo, git_tree_entry_id(entry)) != 0) + return 0; + std::unique_ptr blobGuard(blob, git_blob_free); + + if (blobHasLfsKeywordsFirst3Positional(blob)) + payload->matches->insert(relativePath); + + return 0; +} + +/** + * Tree-walk callback that collects missing tracked non-LFS files from HEAD tree. + * Paths are included only when missing in worktree and blob is not an LFS pointer. + * + * @param root Current directory path in the tree traversal. + * @param entry Current git_tree_entry being visited. + * @param payloadPtr Pointer to MissingNonLfsPathsWalkPayload with output set. + * @return 0 to continue walk; non-zero aborts traversal. + * @note Works on git object database and local worktree metadata. + */ +int collectMissingNonLfsPathsFromHeadTreeCb(const char* root, const git_tree_entry* entry, void* payloadPtr) { + auto* payload = reinterpret_cast(payloadPtr); + if (!payload || git_tree_entry_type(entry) != GIT_OBJECT_BLOB) + return 0; + + fs::path relativePath = fs::path(root ? root : "") / git_tree_entry_name(entry); + std::error_code ec; + if (fs::exists(payload->worktreeRoot / relativePath, ec)) + return 0; + + git_blob* blob = nullptr; + if (git_blob_lookup(&blob, payload->repo, git_tree_entry_id(entry)) != 0) + return 0; + std::unique_ptr blobGuard(blob, git_blob_free); + + if (!blobHasLfsKeywordsFirst3Positional(blob)) + payload->matches->insert(relativePath); + + return 0; +} + +/** + * Identifies LFS files that are safe to resume downloading after interruption. + * Combines working directory LFS pointers with missing LFS blobs from the HEAD tree. + * Useful for recovery after abrupt clone termination. + * + * @param repo Pointer to the git repository object. + * @param directory Root directory of the git working tree to search. + * @param parseHeadTreeIfInterrupted If true, also scan HEAD tree for missing tracked LFS blobs. + * @return Vector of relative paths to all resumable LFS files needing download completion. + * @note Works on specific git repository; scans both working directory and object database. + * @note Part of the git LFS domain: identifies candidates for partial-clone recovery. + */ +std::vector findResumableLfsFiles(git_repository* repo, const std::string& directory, bool parseHeadTreeIfInterrupted) { + std::set uniqueMatches; + auto existingMatches = findLfsLikeFiles(directory, true); + uniqueMatches.insert(existingMatches.begin(), existingMatches.end()); + + if (parseHeadTreeIfInterrupted) { + git_object* headTreeObject = nullptr; + if (git_revparse_single(&headTreeObject, repo, "HEAD^{tree}") == 0) { + std::unique_ptr headTreeGuard(headTreeObject, git_object_free); + MissingLfsPathsWalkPayload payload{repo, fs::path(directory), &uniqueMatches}; + (void)git_tree_walk(reinterpret_cast(headTreeObject), GIT_TREEWALK_PRE, collectMissingLfsPathsFromHeadTreeCb, &payload); } - return *this; } - git_tree* tree() const { return reinterpret_cast(tree_obj); } + return std::vector(uniqueMatches.begin(), uniqueMatches.end()); +} + +/** + * Finds tracked non-LFS files present in HEAD but missing in worktree. + * Used during resume flow to restore required metadata/config artifacts. + * + * @param repo Pointer to the git repository object. + * @param directory Repository worktree root directory. + * @return Vector of relative paths for missing tracked non-LFS files. + * @note Works on specific git repository; scans HEAD tree and filesystem. + */ +std::vector findMissingTrackedNonLfsFiles(git_repository* repo, const std::string& directory) { + std::set uniqueMatches; + + git_object* headTreeObject = nullptr; + if (git_revparse_single(&headTreeObject, repo, "HEAD^{tree}") == 0) { + std::unique_ptr headTreeGuard(headTreeObject, git_object_free); + MissingNonLfsPathsWalkPayload payload{repo, fs::path(directory), &uniqueMatches}; + (void)git_tree_walk(reinterpret_cast(headTreeObject), GIT_TREEWALK_PRE, collectMissingNonLfsPathsFromHeadTreeCb, &payload); + } + + return std::vector(uniqueMatches.begin(), uniqueMatches.end()); +} + +/** + * Checks for LFS download errors recorded by libgit2 patch in the repository root. + * Reads and logs error messages from lfs_error.txt, then removes the file for cleanup. + * + * @param repositoryRootPath Absolute path to the git repository root directory. + * @return true if error file exists (even if unreadable); false if no error file found. + * @note Works on specific git repository location; reads and deletes lfs_error.txt. + * @note Part of the git LFS domain: detects LFS download failures recorded by patched libgit2. + */ +bool ifHasLfsErrorFileLogContentAndRemove(const std::string& repositoryRootPath) { + const fs::path errorFilePath = fs::path(repositoryRootPath) / "lfs_error.txt"; + std::error_code ec; + if (!fs::exists(errorFilePath, ec) || !fs::is_regular_file(errorFilePath, ec)) { + return false; + } + + std::ifstream errorFile(errorFilePath, std::ios::binary); + if (!errorFile) { + SPDLOG_ERROR("Detected lfs_error.txt but failed to open file: {}", errorFilePath.string()); + // Still attempt to remove the stale error file for clean state + fs::remove(errorFilePath, ec); + return true; + } + + std::ostringstream content; + content << errorFile.rdbuf(); + errorFile.close(); + SPDLOG_ERROR("{}", content.str()); + + // Remove the error file to ensure clean state for subsequent download/resume attempts + std::error_code removeEc; + if (fs::remove(errorFilePath, removeEc)) { + SPDLOG_DEBUG("Removed lfs_error.txt from repository root"); + } else if (removeEc) { + SPDLOG_WARN("Failed to remove lfs_error.txt: {}", removeEc.message()); + } + + return true; +} +} // namespace libgit2 + +/** + * Restores missing tracked non-LFS files after interrupted clone/resume. + * Rebuilds index entries from HEAD and checks out selected paths into worktree. + * + * @param repo Pointer to the git repository object. + * @param missingPaths Relative paths of tracked non-LFS files missing in worktree. + * @return StatusCode::OK on success, cancellation/error status otherwise. + * @note Works on repository index and working directory; does not download data from remote. + */ +Status restoreMissingTrackedNonLfsFiles(git_repository* repo, const std::vector& missingPaths) { + if (missingPaths.empty()) + return StatusCode::OK; -private: - void cleanup() noexcept { - git_buf_dispose(&out); - if (blob) { - git_blob_free(blob); - blob = nullptr; + auto runGitCall = [&](int rc, const char* callName) -> Status { + if (rc >= 0) { + return StatusCode::OK; } - if (entry) { - git_tree_entry_free(entry); - entry = nullptr; + if (libgit2::isCloneCancellationRequestedFromServer() || rc == GIT_EUSER) { + setLfsCancelRequested(1); + SPDLOG_DEBUG("Pull restore cancelled for non-lfs filesin {}.", callName); + return StatusCode::HF_GIT_CLONE_CANCELLED; } - if (tree_obj) { - git_object_free(tree_obj); - tree_obj = nullptr; + const git_error* e = git_error_last(); + SPDLOG_ERROR("Pull restore failed for non-lfs files in {} rc:{} msg:{}", callName, rc, e && e->message ? e->message : "no message"); + return StatusCode::HF_GIT_STATUS_FAILED; + }; + + // Repair index entries from HEAD for missing tracked non-LFS files before checkout. + // Interrupted clone may leave these paths absent in index, which later surfaces as staged changes. + { + git_object* headObj = nullptr; + if (auto s = runGitCall(git_revparse_single(&headObj, repo, "HEAD^{tree}"), "git_revparse_single"); !s.ok()) + return s; + auto headGuard = std::unique_ptr{headObj, git_object_free}; + git_tree* headTree = reinterpret_cast(headObj); + + git_index* index = nullptr; + if (auto s = runGitCall(git_repository_index(&index, repo), "git_repository_index"); !s.ok()) + return s; + auto indexGuard = std::unique_ptr{index, git_index_free}; + + std::vector repairedPaths; + repairedPaths.reserve(missingPaths.size()); + for (const auto& p : missingPaths) { + std::string path = p.generic_string(); + git_tree_entry* treeEntry = nullptr; + if (auto s = runGitCall(git_tree_entry_bypath(&treeEntry, headTree, path.c_str()), "git_tree_entry_bypath"); !s.ok()) + return s; + auto treeEntryGuard = std::unique_ptr{treeEntry, git_tree_entry_free}; + + git_index_entry idxEntry = {}; + idxEntry.id = *git_tree_entry_id(treeEntry); + idxEntry.mode = git_tree_entry_filemode_raw(treeEntry); + idxEntry.path = path.c_str(); + if (auto s = runGitCall(git_index_add(index, &idxEntry), "git_index_add"); !s.ok()) + return s; + repairedPaths.emplace_back(std::move(path)); } + + if (auto s = runGitCall(git_index_write(index), "git_index_write"); !s.ok()) + return s; + } + + std::vector ownedPaths; + ownedPaths.reserve(missingPaths.size()); + for (const auto& p : missingPaths) { + ownedPaths.emplace_back(p.generic_string()); + } + + std::vector checkoutPaths; + checkoutPaths.reserve(ownedPaths.size()); + for (auto& p : ownedPaths) { + checkoutPaths.push_back(const_cast(p.c_str())); + } + + git_strarray pathspec = {checkoutPaths.data(), checkoutPaths.size()}; + git_checkout_options checkoutOptions = GIT_CHECKOUT_OPTIONS_INIT; + checkoutOptions.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_RECREATE_MISSING; + checkoutOptions.notify_flags = GIT_CHECKOUT_NOTIFY_ALL; + checkoutOptions.notify_cb = checkCloneCancellationCheckoutNotifyCallback; + checkoutOptions.paths = pathspec; + + int rc = git_checkout_head(repo, &checkoutOptions); + if (rc >= 0) + return StatusCode::OK; + + if (libgit2::isCloneCancellationRequestedFromServer() || rc == GIT_EUSER) { + setLfsCancelRequested(1); + SPDLOG_INFO("Model download resume cancelled by shutdown request during non-LFS file restore. Re-run the same command to resume."); + return StatusCode::HF_GIT_CLONE_CANCELLED; } -}; -void resumeLfsDownloadForFile(git_repository* repo, const char* filePathInRepo) { - GitScope g; + const git_error* e = git_error_last(); + SPDLOG_ERROR("Pull restore failed for non-LFS file rc:{} msg:{}", rc, e && e->message ? e->message : "no message"); + return StatusCode::HF_GIT_STATUS_FAILED; +} - // Resolve HEAD tree (HEAD^{tree}) - CHECK(git_revparse_single(&g.tree_obj, repo, "HEAD^{tree}")); +/** + * Resumes the download of a single LFS file after an interrupted clone. + * Repairs the index entry and forces git checkout to re-trigger the LFS smudge filter. + * Removes the stale worktree file to ensure libgit2 re-downloads it. + * + * @param repo Pointer to the git repository object. + * @param filePathInRepo Path to the LFS file relative to repository root (e.g., "model.bin"). + * @return StatusCode::OK on successful resume, StatusCode::HF_GIT_CLONE_CANCELLED if shutdown + * requested, StatusCode::HF_GIT_LIBGIT2_LFS_DOWNLOAD_FAILED on download errors. + * @note Connects to internet: downloads actual LFS file content from remote. + * @note Works on specific git repository; modifies index, working directory, and accesses ODB. + * @note Part of the git LFS domain: repairs and completes interrupted LFS transfers. + */ +Status resumeLfsDownloadForFile(git_repository* repo, const char* filePathInRepo) { + setLfsCancelRequested(0); + if (libgit2::isCloneCancellationRequestedFromServer()) { + setLfsCancelRequested(1); + return StatusCode::HF_GIT_CLONE_CANCELLED; + } + + auto runGitCall = [&](int rc, const char* callName) -> Status { + if (rc >= 0) { + return StatusCode::OK; + } + if (libgit2::isCloneCancellationRequestedFromServer() || rc == GIT_EUSER) { + setLfsCancelRequested(1); + SPDLOG_DEBUG("Pull resume cancelled in {} for path: {}", callName, filePathInRepo); + return StatusCode::HF_GIT_CLONE_CANCELLED; + } + const git_error* e = git_error_last(); + SPDLOG_ERROR("Pull resume failed in {} for path: {} rc:{} msg:{}", callName, filePathInRepo, rc, e && e->message ? e->message : "no message"); + return StatusCode::HF_GIT_LIBGIT2_LFS_DOWNLOAD_FAILED; + }; - // Find the tree entry by path - CHECK(git_tree_entry_bypath(&g.entry, g.tree(), filePathInRepo)); + // Repair the index entry for this path. + // After a Ctrl+C mid-clone the index can be partially written and the + // entry for the aborted file may be missing entirely. + // git_checkout_head silently skips any path that has no index entry, so + // the LFS smudge filter is never reached. + // We re-add the correct entry (pointer blob SHA from the HEAD tree) so + // that the subsequent git_checkout_head always finds it and triggers the + // LFS download. + { + git_object* headObj = nullptr; + if (auto s = runGitCall(git_revparse_single(&headObj, repo, "HEAD^{tree}"), "git_revparse_single"); !s.ok()) + return s; + auto headGuard = std::unique_ptr{headObj, git_object_free}; + git_tree* headTree = reinterpret_cast(headObj); + + git_tree_entry* treeEntry = nullptr; + if (auto s = runGitCall(git_tree_entry_bypath(&treeEntry, headTree, filePathInRepo), "git_tree_entry_bypath"); !s.ok()) + return s; + auto treeEntryGuard = std::unique_ptr{treeEntry, git_tree_entry_free}; + + git_index* index = nullptr; + if (auto s = runGitCall(git_repository_index(&index, repo), "git_repository_index"); !s.ok()) + return s; + auto indexGuard = std::unique_ptr{index, git_index_free}; + + git_index_entry idxEntry = {}; + idxEntry.id = *git_tree_entry_id(treeEntry); + idxEntry.mode = git_tree_entry_filemode_raw(treeEntry); + idxEntry.path = filePathInRepo; + if (auto s = runGitCall(git_index_add(index, &idxEntry), "git_index_add"); !s.ok()) + return s; + if (auto s = runGitCall(git_index_write(index), "git_index_write"); !s.ok()) + return s; + } - // Ensure it's a blob - if (git_tree_entry_type(g.entry) != GIT_OBJECT_BLOB) { - fprintf(stderr, "[ERROR] Path is not a blob: %s\n", filePathInRepo); - return; // Guard cleans up + // Remove the worktree file before calling git_checkout_head. + // After an aborted clone the worktree contains the LFS pointer text and + // the index blob is also that pointer text, so git_checkout_head sees + // worktree == ODB blob and skips the entry entirely (no smudge filter + // is invoked). Deleting the file first forces libgit2 to recreate it + // through the normal checkout/smudge pipeline, which triggers the LFS + // filter and downloads the actual binary. + const char* workdir = git_repository_workdir(repo); + if (workdir) { + std::error_code ec; + fs::remove(fs::path(workdir) / filePathInRepo, ec); + // Deliberately ignore the error: if the file is already absent the + // checkout will still recreate it via GIT_CHECKOUT_RECREATE_MISSING. } - // Lookup the blob - CHECK(git_blob_lookup(&g.blob, repo, git_tree_entry_id(g.entry))); + std::array checkoutPaths = {const_cast(filePathInRepo)}; + git_strarray pathspec = {checkoutPaths.data(), checkoutPaths.size()}; + git_checkout_options checkoutOptions = GIT_CHECKOUT_OPTIONS_INIT; + checkoutOptions.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_RECREATE_MISSING; + checkoutOptions.notify_flags = GIT_CHECKOUT_NOTIFY_ALL; + checkoutOptions.notify_cb = checkCloneCancellationCheckoutNotifyCallback; + checkoutOptions.paths = pathspec; - // Configure filter behavior - git_blob_filter_options opts = GIT_BLOB_FILTER_OPTIONS_INIT; - // Choose direction: - // GIT_FILTER_TO_WORKTREE : apply smudge (as if writing to working tree) - // GIT_FILTER_TO_ODB : apply clean (as if writing to ODB) - opts.flags |= GIT_FILTER_TO_WORKTREE; + auto status = runGitCall(git_checkout_head(repo, &checkoutOptions), "git_checkout_head"); + if (!status.ok()) + return status; - // Apply filters based on .gitattributes for this path (triggers LFS smudge/clean) - CHECK(git_blob_filter(&g.out, g.blob, filePathInRepo, &opts)); + return StatusCode::OK; +} - // We don't need the buffer contents; the filter side-effects are enough. - // All resources (out, blob, entry, tree_obj) will be freed automatically here. +namespace { + +struct ResumeCandidates { + bool hasWipMarker = false; + bool hasLfsErrorFile = false; + bool interruptionLikely = false; + std::vector lfsMatches; + std::vector missingNonLfsMatches; +}; + +Status mapRepositoryOpenFailureToStatus(const GitRepositoryGuard& repoGuard) { + if (repoGuard.git_error_class == GIT_ERROR_OS) + return StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH; + if (repoGuard.git_error_class == GIT_ERROR_INVALID) + return StatusCode::HF_GIT_LIBGIT2_NOT_INITIALIZED; + return StatusCode::HF_GIT_STATUS_FAILED; } -Status HfDownloader::downloadModel() { - if (FileSystem::isPathEscaped(this->downloadPath)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", this->downloadPath); - return StatusCode::PATH_INVALID; +/** + * Builds resume candidate lists based on interruption markers and repository scan. + * + * @param repo Pointer to the git repository object. + * @param downloadPath Repository worktree root path. + * @return ResumeCandidates containing LFS and non-LFS recovery targets. + * @note Works on local repository metadata and filesystem; no network operations. + */ +ResumeCandidates buildResumeCandidates(git_repository* repo, const std::string& downloadPath) { + ResumeCandidates candidates; + candidates.hasWipMarker = libgit2::hasLfsWipMarker(downloadPath); + candidates.hasLfsErrorFile = libgit2::hasLfsErrorFile(downloadPath); + + // Checking if the download was partially finished for any files in repository, + // including tracked LFS pointer blobs missing from the worktree after abrupt termination. + candidates.lfsMatches = libgit2::findResumableLfsFiles(repo, downloadPath, candidates.hasWipMarker || candidates.hasLfsErrorFile); + if (candidates.hasWipMarker) { + candidates.missingNonLfsMatches = libgit2::findMissingTrackedNonLfsFiles(repo, downloadPath); } - // Repository exists and we do not want to overwrite - if (std::filesystem::is_directory(this->downloadPath) && !this->overwriteModels) { - // Checking if the download was partially finished for any files in repository - auto matches = libgit2::findLfsLikeFiles(this->downloadPath, true); + candidates.interruptionLikely = candidates.hasWipMarker || candidates.hasLfsErrorFile || !candidates.lfsMatches.empty(); + return candidates; +} - if (matches.empty()) { - SPDLOG_DEBUG("No files to resume download found."); - std::cout << "Path already exists on local filesystem. Skipping download to path: " << this->downloadPath << std::endl; - return StatusCode::OK; - } else { - std::cout << "Found " << matches.size() << " file(s) to resume partial download:\n"; - for (const auto& p : matches) { - std::cout << " " << p.string() << "\n"; - } +void printResumeCandidates(const ResumeCandidates& candidates) { + if (!candidates.lfsMatches.empty()) { + std::cout << "Found " << candidates.lfsMatches.size() << " LFS file(s) to resume partial download:\n"; + for (const auto& p : candidates.lfsMatches) { + std::cout << " " << p.string() << "\n"; } + } - GitRepositoryGuard repoGuard(this->downloadPath); - if (!repoGuard.get()) { - std::cout << "Path already exists on local filesystem. Cannot download model to: " << this->downloadPath << std::endl; - std::cout << "Use --override to start download from scratch." << std::endl; - if (repoGuard.git_error_class == GIT_ERROR_OS) - return StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH; - else if (repoGuard.git_error_class == GIT_ERROR_INVALID) - return StatusCode::HF_GIT_LIBGIT2_NOT_INITIALIZED; - else - return StatusCode::HF_GIT_STATUS_FAILED; + if (!candidates.missingNonLfsMatches.empty()) { + std::cout << "Found " << candidates.missingNonLfsMatches.size() << " missing tracked non-LFS file(s) to restore:\n"; + for (const auto& p : candidates.missingNonLfsMatches) { + std::cout << " " << p.string() << "\n"; } + } +} - // Set repository url - std::string passRepoUrl = GetRepositoryUrlWithPassword(); - const char* url = passRepoUrl.c_str(); - int error = git_repository_set_url(repoGuard.get(), url); - if (error < 0) { - const git_error* err = git_error_last(); - if (err) - SPDLOG_ERROR("Repository set url failed: {} {}", err->klass, err->message); - else - SPDLOG_ERROR("Repository set url failed: {}", error); - std::cout << "Path already exists on local filesystem. And set git repository url failed: " << this->downloadPath << std::endl; - std::cout << "Consider --override to start download from scratch." << std::endl; - return StatusCode::HF_GIT_CLONE_FAILED; +/** + * Reconciles index with HEAD tree after crash recovery operations. + * This removes staged index artifacts before strict cleanliness validation. + * + * @param repo Pointer to the git repository object. + * @return StatusCode::OK on success, cancellation/error status otherwise. + * @note Works on local git index; does not modify tracked file contents in worktree. + */ +Status restoreGitIndexToHead(git_repository* repo) { + auto runGitCall = [&](int rc, const char* callName) -> Status { + if (rc >= 0) { + return StatusCode::OK; + } + if (libgit2::isCloneCancellationRequestedFromServer() || rc == GIT_EUSER) { + setLfsCancelRequested(1); + SPDLOG_DEBUG("Pull resume cancelled for index normalization in {}.", callName); + return StatusCode::HF_GIT_CLONE_CANCELLED; } + const git_error* e = git_error_last(); + SPDLOG_ERROR("Pull resume failed for index normalization in {} rc:{} msg:{}", callName, rc, e && e->message ? e->message : "no message"); + return StatusCode::HF_GIT_STATUS_FAILED; + }; + + git_object* headObj = nullptr; + if (auto s = runGitCall(git_revparse_single(&headObj, repo, "HEAD^{tree}"), "git_revparse_single"); !s.ok()) + return s; + auto headGuard = std::unique_ptr{headObj, git_object_free}; + git_tree* headTree = reinterpret_cast(headObj); + + git_index* index = nullptr; + if (auto s = runGitCall(git_repository_index(&index, repo), "git_repository_index"); !s.ok()) + return s; + auto indexGuard = std::unique_ptr{index, git_index_free}; + + if (auto s = runGitCall(git_index_read_tree(index, headTree), "git_index_read_tree"); !s.ok()) + return s; + if (auto s = runGitCall(git_index_write(index), "git_index_write"); !s.ok()) + return s; + + return StatusCode::OK; +} + +Status resumeExistingRepository(git_repository* repo, + const std::string& downloadPath, + const ResumeCandidates& candidates, + const std::function& checkRepositoryStatusFn) { + if (!libgit2::createLfsWipMarker(downloadPath)) { + SPDLOG_WARN("Failed to create .lfswip marker before pull resume at {}", libgit2::getLfsWipMarkerPath(downloadPath).string()); + } + + printResumeCandidates(candidates); - for (const auto& p : matches) { - std::cout << " Resuming " << p.string() << "...\n"; - std::string path = p.string(); - resumeLfsDownloadForFile(repoGuard.get(), path.c_str()); + for (const auto& p : candidates.lfsMatches) { + if (libgit2::isCloneCancellationRequestedFromServer()) { + setLfsCancelRequested(1); + SPDLOG_INFO("Model download resume cancelled by shutdown request before resuming LFS file: {}. Re-run the same command to resume.", p.string()); + return StatusCode::HF_GIT_CLONE_CANCELLED; } + if (!libgit2::createLfsWipMarker(downloadPath)) { + SPDLOG_WARN("Failed to refresh .lfswip marker before pull resume at {}", libgit2::getLfsWipMarkerPath(downloadPath).string()); + } + std::cout << " Resuming " << p.string() << "...\n"; + std::string path = p.string(); + auto resumeStatus = resumeLfsDownloadForFile(repo, path.c_str()); + if (!resumeStatus.ok()) { + return resumeStatus; + } + } - // Non blocking check - SPDLOG_DEBUG("Checking repository status."); - auto status = CheckRepositoryStatus(false); - if (!status.ok()) { - SPDLOG_DEBUG("[WARNING] Model repository status check failed after resuming download. Status: {}", status.string()); - SPDLOG_DEBUG("Consider --override to start download from scratch."); - } else { - SPDLOG_DEBUG("Model repository status check passed after resuming download."); + if (candidates.hasWipMarker) { + auto restoreMissingStatus = restoreMissingTrackedNonLfsFiles(repo, candidates.missingNonLfsMatches); + if (!restoreMissingStatus.ok()) { + return restoreMissingStatus; } - return StatusCode::OK; + } else if (!candidates.missingNonLfsMatches.empty()) { + SPDLOG_DEBUG("Skipping non-LFS missing tracked files restoration because .lfswip marker was not found."); } - auto status = IModelDownloader::checkIfOverwriteAndRemove(); + auto restoredGitIndexStatus = restoreGitIndexToHead(repo); + if (!restoredGitIndexStatus.ok()) { + return restoredGitIndexStatus; + } + + // Checking if git status is ok but we are left with LFS errors recorded by libgit2 patch in repository root. + if (libgit2::ifHasLfsErrorFileLogContentAndRemove(downloadPath)) { + SPDLOG_ERROR("Model download resume failed: LFS errors recorded above. Re-run the same command to retry, or use --override to restart from scratch."); + return StatusCode::HF_GIT_LIBGIT2_LFS_DOWNLOAD_FAILED; + } + + // Blocking check + SPDLOG_DEBUG("Checking repository status."); + auto status = checkRepositoryStatusFn(false); if (!status.ok()) { + SPDLOG_ERROR("Model repository status check failed after resuming download. Status: {}", status.string()); + SPDLOG_ERROR("Consider --override to start download from scratch."); return status; } + SPDLOG_DEBUG("Model repository status check passed after resuming download."); - SPDLOG_DEBUG("Downloading to path: {}", this->downloadPath); + libgit2::removeLfsWipMarker(downloadPath); + return StatusCode::OK; +} - git_repository* cloned_repo = NULL; +Status handleExistingRepositoryWithoutOverwrite(const std::string& downloadPath, + const std::function& checkRepositoryStatusFn) { + GitRepositoryGuard repoGuard(downloadPath); + if (!repoGuard.get()) { + std::cout << "Path already exists on local filesystem. Cannot download model to: " << downloadPath << std::endl; + std::cout << "Use --override to start download from scratch." << std::endl; + return mapRepositoryOpenFailureToStatus(repoGuard); + } + + auto candidates = buildResumeCandidates(repoGuard.get(), downloadPath); + if (!candidates.interruptionLikely) { + SPDLOG_DEBUG("Model pull operation found no interruption signals for this path: {}", downloadPath); + std::cout << "Path already exists on local filesystem. Skipping download to path: " << downloadPath << std::endl; + return StatusCode::OK; + } + + return resumeExistingRepository(repoGuard.get(), downloadPath, candidates, checkRepositoryStatusFn); +} + +/** + * Configures libgit2 clone options with callbacks, credentials, and optional proxy. + * + * @param cloneOptions [in/out] libgit2 clone options structure to populate. + * @param useProxy Whether to route git operations through proxyUrl. + * @param proxyUrl Proxy URL to pass into libgit2 when useProxy is true. + * @note Configures clone behavior only; no network call is performed by this function itself. + */ +void configureCloneOptions(git_clone_options& cloneOptions, bool useProxy, const std::string& proxyUrl) { // clone_opts for progress reporting set in libgit2 lib by patch - git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; + cloneOptions.fetch_opts.callbacks.transfer_progress = checkCloneCancellationTransferProgressCallback; + cloneOptions.fetch_opts.callbacks.sideband_progress = checkCloneCancellationSidebandProgressCallback; + cloneOptions.fetch_opts.callbacks.update_tips = checkCloneCancellationUpdateTipsCallback; + + cloneOptions.checkout_opts.notify_flags = GIT_CHECKOUT_NOTIFY_ALL; + cloneOptions.checkout_opts.notify_cb = checkCloneCancellationCheckoutNotifyCallback; + // Credential check function - clone_opts.fetch_opts.callbacks.credentials = cred_acquire_cb; + cloneOptions.fetch_opts.callbacks.credentials = cred_acquire_cb; // Use proxy - if (CheckIfProxySet()) { - clone_opts.fetch_opts.proxy_opts.type = GIT_PROXY_SPECIFIED; - clone_opts.fetch_opts.proxy_opts.url = this->httpProxy.c_str(); + if (useProxy) { + cloneOptions.fetch_opts.proxy_opts.type = GIT_PROXY_SPECIFIED; + cloneOptions.fetch_opts.proxy_opts.url = proxyUrl.c_str(); SPDLOG_DEBUG("Download using https_proxy settings"); } else { SPDLOG_DEBUG("Download with https_proxy not set"); } +} - std::string repoUrl = GetRepoUrl(); - SPDLOG_DEBUG("Downloading from url: {}", repoUrl.c_str()); - std::string passRepoUrl = GetRepositoryUrlWithPassword(); +/** + * Executes git clone for a model repository and handles cancellation/error mapping. + * + * @param downloadPath Destination repository path on local filesystem. + * @param passRepoUrl Source repository URL (possibly with embedded credentials). + * @param cloneOptions Prepared libgit2 clone options. + * @return StatusCode::OK on success, cancellation or clone failure status otherwise. + * @note Connects to remote git endpoint and writes repository data to local filesystem. + */ +Status executeClone(const std::string& downloadPath, const std::string& passRepoUrl, git_clone_options& cloneOptions) { + git_repository* clonedRepo = nullptr; const char* url = passRepoUrl.c_str(); - const char* path = this->downloadPath.c_str(); + const char* path = downloadPath.c_str(); SPDLOG_TRACE("Starting git clone to: {}", path); - int error = git_clone(&cloned_repo, url, path, &clone_opts); + if (!libgit2::createLfsWipMarker(downloadPath)) { + SPDLOG_WARN("Failed to create .lfswip marker before clone at {}", libgit2::getLfsWipMarkerPath(downloadPath).string()); + } + setLfsCancelRequested(0); /* reset for this operation */ + int error = git_clone(&clonedRepo, url, path, &cloneOptions); SPDLOG_TRACE("Ended git clone"); if (error != 0) { + if (libgit2::isCloneCancellationRequestedFromServer() || error == GIT_EUSER) { + SPDLOG_INFO("Model download cancelled by shutdown request. Re-run the same command to resume."); + return StatusCode::HF_GIT_CLONE_CANCELLED; + } const git_error* err = git_error_last(); if (err) SPDLOG_ERROR("Libgit2 clone error: {} message: {}", err->klass, err->message); else SPDLOG_ERROR("Libgit2 clone error: {}", error); return StatusCode::HF_GIT_CLONE_FAILED; - } else if (cloned_repo) { - git_repository_free(cloned_repo); + } + if (clonedRepo) { + git_repository_free(clonedRepo); + } + return StatusCode::OK; +} + +Status finalizeAfterClone(const std::string& downloadPath, + const std::function& checkRepositoryStatusFn, + const std::function& removeReadonlyFn) { + // Checking if git status is ok but we are left with LFS errors recorded by libgit2 patch in repository root. + if (libgit2::ifHasLfsErrorFileLogContentAndRemove(downloadPath)) { + SPDLOG_ERROR("Model download failed: LFS errors recorded above. Re-run the same command to resume, or use --override to restart from scratch."); + return StatusCode::HF_GIT_LIBGIT2_LFS_DOWNLOAD_FAILED; } SPDLOG_DEBUG("Checking repository status."); - status = CheckRepositoryStatus(true); + auto status = checkRepositoryStatusFn(true); if (!status.ok()) { SPDLOG_ERROR("Model repository status check failed after model download. Status: {}", status.string()); SPDLOG_ERROR("Consider rerunning the command to resume the download after network issues."); SPDLOG_ERROR("Consider --override flag to start download from scratch."); return status; - } else { - SPDLOG_DEBUG("Model repository status check passed after model download."); } + SPDLOG_DEBUG("Model repository status check passed after model download."); // libgit2 clone sets readonly attributes - status = RemoveReadonlyFileAttributeFromDir(this->downloadPath); + status = removeReadonlyFn(downloadPath); if (!status.ok()) { return status; } + + libgit2::removeLfsWipMarker(downloadPath); return StatusCode::OK; } +Status handleFreshClone(const std::string& downloadPath, + const std::string& repoUrl, + const std::string& passRepoUrl, + bool useProxy, + const std::string& proxyUrl, + const std::function& checkRepositoryStatusFn, + const std::function& removeReadonlyFn) { + SPDLOG_DEBUG("Downloading to path: {}", downloadPath); + + git_clone_options cloneOptions = GIT_CLONE_OPTIONS_INIT; + configureCloneOptions(cloneOptions, useProxy, proxyUrl); + + SPDLOG_DEBUG("Downloading from url: {}", repoUrl.c_str()); + + auto status = executeClone(downloadPath, passRepoUrl, cloneOptions); + if (!status.ok()) { + return status; + } + + return finalizeAfterClone(downloadPath, checkRepositoryStatusFn, removeReadonlyFn); +} + +} // namespace + +/** + * Main method to download a model from Hugging Face via git clone. + * Handles initial full clone, resumption of partial downloads, and validation of repository state. + * Applies appropriate authentication, proxy settings, and LFS configuration during clone. + * + * @return StatusCode::OK on successful download or if model already exists locally, + * StatusCode::HF_GIT_CLONE_CANCELLED if shutdown requested during operation, + * StatusCode::HF_GIT_CLONE_FAILED or StatusCode::HF_GIT_LIBGIT2_LFS_DOWNLOAD_FAILED on errors. + * @note Connects to internet: performs git clone from Hugging Face endpoint (requires network). + * @note Works on local filesystem and downloads git repository to downloadPath. + * @note Implements the full git domain: clone, LFS, checkout, authentication, and repository validation. + */ +Status HfDownloader::downloadModel() { + if (FileSystem::isPathEscaped(this->downloadPath)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", this->downloadPath); + return StatusCode::PATH_INVALID; + } + + auto checkRepositoryStatusFn = [this](bool checkUntracked) { + return this->CheckRepositoryStatus(checkUntracked); + }; + auto removeReadonlyFn = [this](const std::string& path) { + return this->RemoveReadonlyFileAttributeFromDir(path); + }; + + // Repository exists and we do not want to overwrite + if (std::filesystem::is_directory(this->downloadPath) && !this->overwriteModels) { + return handleExistingRepositoryWithoutOverwrite(this->downloadPath, checkRepositoryStatusFn); + } + + auto status = IModelDownloader::checkIfOverwriteAndRemove(); + if (!status.ok()) { + return status; + } + + const bool useProxy = CheckIfProxySet(); + std::string repoUrl = GetRepoUrl(); + std::string passRepoUrl = GetRepositoryUrlWithPassword(); + + return handleFreshClone(this->downloadPath, repoUrl, passRepoUrl, useProxy, this->httpProxy, checkRepositoryStatusFn, removeReadonlyFn); +} + } // namespace ovms + +/** + * C-style function exported for libgit2 LFS extension to check if LFS downloads should be aborted. + * Called by patched libgit2 to determine if server shutdown was requested. + * + * @return 1 if LFS shutdown/cancellation is requested, 0 otherwise. + * @note Reads in-process shutdown state only; does not perform filesystem or network I/O. + * @note Extern C interface: for compatibility with libgit2 C library patches. + */ +extern "C" int git_lfs_shutdown_requested(void) { + return ovms::libgit2::isCloneCancellationRequestedFromServer() ? 1 : 0; +} diff --git a/src/pull_module/libgit2.hpp b/src/pull_module/libgit2.hpp index 224995c0d3..d977d70b0d 100644 --- a/src/pull_module/libgit2.hpp +++ b/src/pull_module/libgit2.hpp @@ -72,6 +72,12 @@ class HfDownloader : public IModelDownloader { namespace libgit2 { inline constexpr size_t READ_FIRST_THREE_LINES_DEFAULT_MAX_LINE_BYTES = 8U * 1024U * 1024U; +fs::path getLfsWipMarkerPath(const std::string& repositoryPath); +bool createLfsWipMarker(const std::string& repositoryPath); +bool hasLfsWipMarker(const std::string& repositoryPath); +void removeLfsWipMarker(const std::string& repositoryPath); + +bool isCloneCancellationRequestedFromServer(); void rtrimCrLfWhitespace(std::string& s); bool containsCaseInsensitive(const std::string& hay, const std::string& needle); bool readFirstThreeLines(const fs::path& p, std::vector& outLines, size_t maxLineBytes = READ_FIRST_THREE_LINES_DEFAULT_MAX_LINE_BYTES); diff --git a/src/server.cpp b/src/server.cpp index bdf572f76c..6614bf72f9 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -67,6 +67,7 @@ #include "profilermodule.hpp" #include "pull_module/hf_pull_model_module.hpp" #include "servablemanagermodule.hpp" +#include "shutdown_state.hpp" #include "servables_config_manager_module/servablesconfigmanagermodule.hpp" #include "stringutils.hpp" #include "version.hpp" @@ -79,6 +80,32 @@ using grpc::ServerBuilder; namespace ovms { +extern "C" { +void git_lfs_cancel_set(int value); +int git_lfs_cancel_get(void); +} + +static void setLfsCancelRequestedFromSignal(int value) { + git_lfs_cancel_set(value != 0 ? 1 : 0); +} + +static void requestShutdownFromSignal(int value) { + // Only set the async-signal-safe flag in the signal handler. + // Actual shutdown and LFS cancel operations are deferred to a safe polling context. + setSignalShutdownRequested(value); +} + +static void processSignalShutdownAndSetLfsCancelFlag() { + // This function is called from the polling loop to safely process signal shutdown requests. + // It performs the actual shutdown and LFS cancel operations in a safe context. + if (isSignalShutdownRequested()) { + processSignalShutdownRequest(); + // Also set LFS cancel flag upon shutdown + int shutdownValue = getShutdownRequestValue(); + setLfsCancelRequestedFromSignal(shutdownValue); + } +} + Server& Server::instance() { static Server global; return global; @@ -134,14 +161,15 @@ static void logConfig(const Config& config) { } static void onInterrupt(int status) { - Server::instance().setShutdownRequest(1); + requestShutdownFromSignal(1); } static void onTerminate(int status) { - Server::instance().setShutdownRequest(1); + requestShutdownFromSignal(1); } static void onIllegal(int status) { + requestShutdownFromSignal(2); (void)status; const char msg[] = "SIGILL received: illegal instruction. This may indicate an unsupported CPU or device or an internal error. Terminating.\n"; #ifdef __linux__ @@ -255,8 +283,9 @@ void Server::setShutdownRequest(int i) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); } if (counter) { - shutdown_request = i; - SPDLOG_TRACE("Ovms shutdown request set to: {}", shutdown_request); + setShutdownRequestValue(i); + setLfsCancelRequestedFromSignal(i); + SPDLOG_TRACE("Ovms shutdown request set to: {}", i); } else { SPDLOG_ERROR("Server shutdown mutex lock failed."); } @@ -270,7 +299,7 @@ int Server::getShutdownStatus() { return 0; } - return shutdown_request; + return getShutdownRequestValue(); } int Server::getExitStatus() { @@ -281,7 +310,7 @@ int Server::getExitStatus() { return 0; } - return ovms_exited; + return getExitStatusValue(); } void Server::setExitStatus(int i) { @@ -292,8 +321,8 @@ void Server::setExitStatus(int i) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); } if (counter) { - ovms_exited = i; - SPDLOG_TRACE("Ovms exit status set to: {}", ovms_exited); + setExitStatusValue(i); + SPDLOG_TRACE("Ovms exit status set to: {}", getExitStatusValue()); } else { SPDLOG_ERROR("Server shutdown mutex lock failed."); } @@ -534,6 +563,8 @@ int Server::startServerFromSettings(ServerSettingsImpl& serverSettings, ModelsSe } while (!getShutdownStatus() && (serverSettings.serverMode == HF_PULL_AND_START_MODE || serverSettings.serverMode == SERVING_MODELS_MODE)) { + // Process any signal shutdown requests in a safe context + processSignalShutdownAndSetLfsCancelFlag(); std::this_thread::sleep_for(std::chrono::milliseconds(200)); } } catch (const std::exception& e) { diff --git a/src/server.hpp b/src/server.hpp index 23306636d3..eb8eaf8aa2 100644 --- a/src/server.hpp +++ b/src/server.hpp @@ -24,10 +24,6 @@ #include "capi_frontend/server_settings.hpp" #include "module.hpp" #include "module_names.hpp" -namespace { -volatile sig_atomic_t shutdown_request = 0; -volatile sig_atomic_t ovms_exited = 0; -} // namespace namespace ovms { class Config; class Status; diff --git a/src/shutdown_state.cpp b/src/shutdown_state.cpp new file mode 100644 index 0000000000..3825693302 --- /dev/null +++ b/src/shutdown_state.cpp @@ -0,0 +1,61 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "shutdown_state.hpp" + +#include +#include + +namespace { +std::atomic shutdown_request{0}; +std::atomic ovms_exited{0}; +// Volatile sig_atomic_t flags: async-signal-safe for use in signal handlers +volatile sig_atomic_t signal_shutdown_requested{0}; +volatile sig_atomic_t signal_shutdown_value{0}; +} // namespace + +namespace ovms { +int getShutdownRequestValue() { + return shutdown_request.load(std::memory_order_relaxed); +} + +void setShutdownRequestValue(int value) { + shutdown_request.store(value, std::memory_order_relaxed); +} + +int getExitStatusValue() { + return ovms_exited.load(std::memory_order_relaxed); +} + +void setExitStatusValue(int value) { + ovms_exited.store(value, std::memory_order_relaxed); +} + +bool isSignalShutdownRequested() { + return signal_shutdown_requested != 0; +} + +void processSignalShutdownRequest() { + if (isSignalShutdownRequested()) { + signal_shutdown_requested = 0; + setShutdownRequestValue(signal_shutdown_value); + } +} + +void setSignalShutdownRequested(int value) { + signal_shutdown_value = value; + signal_shutdown_requested = 1; +} +} // namespace ovms diff --git a/src/shutdown_state.hpp b/src/shutdown_state.hpp new file mode 100644 index 0000000000..abc3477f21 --- /dev/null +++ b/src/shutdown_state.hpp @@ -0,0 +1,33 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +namespace ovms { +int getShutdownRequestValue(); +void setShutdownRequestValue(int value); +int getExitStatusValue(); +void setExitStatusValue(int value); + +// Signal-safe API: for use in signal handlers and polling contexts +// Signal handlers should call setSignalShutdownRequested() which is async-signal-safe. +// Main loop/polling thread should periodically call processSignalShutdownRequest() to +// perform actual shutdown operations from a safe context. +bool isSignalShutdownRequested(); +void processSignalShutdownRequest(); +void setSignalShutdownRequested(int value); +} // namespace ovms diff --git a/src/status.cpp b/src/status.cpp index 6129b0671d..0fecf71c08 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -347,10 +347,12 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::HF_FAILED_TO_INIT_OPTIMUM_CLI, "Failed to run optimum-cli executable"}, {StatusCode::HF_RUN_OPTIMUM_CLI_EXPORT_FAILED, "Failed to run optimum-cli export command"}, {StatusCode::HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED, "Failed to run convert-tokenizer export command"}, + {StatusCode::HF_GIT_CLONE_CANCELLED, "Libgit2 clone cancelled due to shutdown request"}, {StatusCode::HF_GIT_CLONE_FAILED, "Failed in libgit2 execution of clone method"}, {StatusCode::HF_GIT_STATUS_FAILED, "Failed in libgit2 execution of status method"}, {StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH, "Failed in libgit2 to check repository status for a given path"}, {StatusCode::HF_GIT_LIBGIT2_NOT_INITIALIZED, "Libgit2 was not initialized"}, + {StatusCode::HF_GIT_LIBGIT2_LFS_DOWNLOAD_FAILED, "Libgit2 LFS download failed"}, {StatusCode::HF_GIT_STATUS_UNCLEAN, "Unclean status detected in libgit2 repository path"}, {StatusCode::PARTIAL_END, "Request has finished and no further communication is needed"}, diff --git a/src/status.hpp b/src/status.hpp index 04c6e251c2..d5ab5762e7 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -359,10 +359,12 @@ enum class StatusCode { HF_FAILED_TO_INIT_OPTIMUM_CLI, HF_RUN_OPTIMUM_CLI_EXPORT_FAILED, HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED, + HF_GIT_CLONE_CANCELLED, HF_GIT_CLONE_FAILED, HF_GIT_STATUS_FAILED, HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH, HF_GIT_LIBGIT2_NOT_INITIALIZED, + HF_GIT_LIBGIT2_LFS_DOWNLOAD_FAILED, HF_GIT_STATUS_UNCLEAN, PARTIAL_END, diff --git a/src/test/libgit2_test.cpp b/src/test/libgit2_test.cpp index 09e873ae01..d44f8c2fbe 100644 --- a/src/test/libgit2_test.cpp +++ b/src/test/libgit2_test.cpp @@ -724,3 +724,128 @@ TEST_F(LibGit2FindLfsLikeFilesTest, NonRecursiveDoesNotDescendButStillUsesRelati sortPaths(expected); EXPECT_EQ(matches_rec, expected); } + +// --------------------------------------------------------------------------- +// .lfswip marker helpers +// +// The marker is a SIBLING of the repository directory, named +// ".lfswip" placed in the repository's parent directory. +// Example: repositoryPath "/opt/models/OV/model1" produces the marker path +// "/opt/models/OV/model1.lfswip". +// The repository directory itself does NOT need to exist for the marker to +// be written — only its parent must exist (createLfsWipMarker creates it). +// --------------------------------------------------------------------------- + +TEST(LibGit2LfsWipMarker, PathIsSiblingOfRepository) { + const std::string repositoryPath = "/opt/models/OV/model1"; + const fs::path marker = ovms::libgit2::getLfsWipMarkerPath(repositoryPath); + EXPECT_EQ(marker, fs::path("/opt/models/OV/model1.lfswip")); + EXPECT_EQ(marker.parent_path(), fs::path("/opt/models/OV")); + EXPECT_EQ(marker.filename(), fs::path("model1.lfswip")); +} + +TEST(LibGit2LfsWipMarker, PathWithRelativeRepository) { + const std::string repositoryPath = "repo/sub"; + const fs::path marker = ovms::libgit2::getLfsWipMarkerPath(repositoryPath); + EXPECT_EQ(marker, fs::path("repo") / "sub.lfswip"); +} + +TEST(LibGit2LfsWipMarker, CreateAndDetectAndRemove) { + TempDir td; + const fs::path repository = td.dir / "model1"; + const std::string repositoryPath = repository.string(); + const fs::path expectedMarker = td.dir / "model1.lfswip"; + + EXPECT_FALSE(ovms::libgit2::hasLfsWipMarker(repositoryPath)); + EXPECT_FALSE(fs::exists(expectedMarker)); + + ASSERT_TRUE(ovms::libgit2::createLfsWipMarker(repositoryPath)); + + EXPECT_TRUE(fs::exists(expectedMarker)); + EXPECT_TRUE(fs::is_regular_file(expectedMarker)); + EXPECT_TRUE(ovms::libgit2::hasLfsWipMarker(repositoryPath)); + // Repository directory itself is NOT created by the marker helpers. + EXPECT_FALSE(fs::exists(repository)); + + ovms::libgit2::removeLfsWipMarker(repositoryPath); + + EXPECT_FALSE(fs::exists(expectedMarker)); + EXPECT_FALSE(ovms::libgit2::hasLfsWipMarker(repositoryPath)); +} + +TEST(LibGit2LfsWipMarker, CreateOnExistingMarkerTruncates) { + TempDir td; + const fs::path repository = td.dir / "model1"; + const std::string repositoryPath = repository.string(); + const fs::path expectedMarker = td.dir / "model1.lfswip"; + + // Pre-populate marker with content; createLfsWipMarker should truncate it. + { + std::ofstream pre(expectedMarker, std::ios::binary); + pre << "stale-content"; + } + ASSERT_TRUE(fs::exists(expectedMarker)); + ASSERT_GT(fs::file_size(expectedMarker), 0u); + + ASSERT_TRUE(ovms::libgit2::createLfsWipMarker(repositoryPath)); + + EXPECT_TRUE(fs::exists(expectedMarker)); + EXPECT_EQ(fs::file_size(expectedMarker), 0u); +} + +TEST(LibGit2LfsWipMarker, CreateAutoCreatesParentDirectory) { + TempDir td; + const fs::path repository = td.dir / "nested" / "parent" / "model1"; + const std::string repositoryPath = repository.string(); + const fs::path expectedMarker = td.dir / "nested" / "parent" / "model1.lfswip"; + + ASSERT_FALSE(fs::exists(expectedMarker.parent_path())); + + ASSERT_TRUE(ovms::libgit2::createLfsWipMarker(repositoryPath)); + + EXPECT_TRUE(fs::is_directory(expectedMarker.parent_path())); + EXPECT_TRUE(fs::is_regular_file(expectedMarker)); + EXPECT_TRUE(ovms::libgit2::hasLfsWipMarker(repositoryPath)); +} + +TEST(LibGit2LfsWipMarker, RemoveWhenMarkerAbsentIsNoOp) { + TempDir td; + const fs::path repository = td.dir / "model1"; + const std::string repositoryPath = repository.string(); + + ASSERT_FALSE(ovms::libgit2::hasLfsWipMarker(repositoryPath)); + // Must not throw or fail when the marker does not exist. + ovms::libgit2::removeLfsWipMarker(repositoryPath); + EXPECT_FALSE(ovms::libgit2::hasLfsWipMarker(repositoryPath)); +} + +TEST(LibGit2LfsWipMarker, HasMarkerReturnsFalseWhenMarkerIsADirectory) { + TempDir td; + const fs::path repository = td.dir / "model1"; + const std::string repositoryPath = repository.string(); + const fs::path expectedMarker = td.dir / "model1.lfswip"; + + // Create a directory at the marker location; helper requires regular file. + ASSERT_TRUE(fs::create_directories(expectedMarker)); + + EXPECT_FALSE(ovms::libgit2::hasLfsWipMarker(repositoryPath)); +} + +TEST(LibGit2LfsWipMarker, MarkersForDifferentRepositoriesAreIndependent) { + TempDir td; + const fs::path repoA = td.dir / "modelA"; + const fs::path repoB = td.dir / "modelB"; + const std::string repoAPath = repoA.string(); + const std::string repoBPath = repoB.string(); + + ASSERT_TRUE(ovms::libgit2::createLfsWipMarker(repoAPath)); + EXPECT_TRUE(ovms::libgit2::hasLfsWipMarker(repoAPath)); + EXPECT_FALSE(ovms::libgit2::hasLfsWipMarker(repoBPath)); + + ASSERT_TRUE(ovms::libgit2::createLfsWipMarker(repoBPath)); + EXPECT_TRUE(ovms::libgit2::hasLfsWipMarker(repoBPath)); + + ovms::libgit2::removeLfsWipMarker(repoAPath); + EXPECT_FALSE(ovms::libgit2::hasLfsWipMarker(repoAPath)); + EXPECT_TRUE(ovms::libgit2::hasLfsWipMarker(repoBPath)); +} diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index e9044ad2cd..d5e6779dc5 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -14,15 +14,26 @@ // limitations under the License. //***************************************************************************** #include +#include +#include #include #include #include +#include #include #include #include #include #include +#ifndef _WIN32 +#include +#include +#include +#else +#include +#endif + #include #include @@ -46,10 +57,20 @@ #include "environment.hpp" -class HfDownloaderPullHfModel : public TestWithTempDir { +class HfPull : public TestWithTempDir { protected: ovms::Server& server = ovms::Server::instance(); std::unique_ptr t; + std::string modelName; + std::string downloadPath; + std::string task; + + void SetUp() override { + TestWithTempDir::SetUp(); + modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; + downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); + task = "text_generation"; + } void ServerPullHfModel(std::string& sourceModel, std::string& downloadPath, std::string& task, int expected_code = 0, int timeoutSeconds = 60) { ::SetUpServerForDownload(this->t, this->server, sourceModel, downloadPath, task, expected_code, timeoutSeconds); @@ -80,6 +101,58 @@ class HfDownloaderPullHfModel : public TestWithTempDir { } }; +class HfPullCache : public HfPull { +protected: + static std::once_flag cacheInitFlag; + static std::unique_ptr cacheDir; + static std::string cachedRepositoryPath; + + static constexpr const char* MODEL_NAME = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; + static constexpr const char* TASK_NAME = "text_generation"; + + void SetUp() override { + HfPull::SetUp(); + initializeSharedCache(); + seedCurrentTestRepository(); + } + + void initializeSharedCache() { + std::call_once(cacheInitFlag, [this]() { + cacheDir = std::make_unique(); + std::string modelName = MODEL_NAME; + std::string downloadPath = ovms::FileSystem::joinPath({cacheDir->dir.string(), "repository"}); + std::string task = TASK_NAME; + + this->ServerPullHfModel(modelName, downloadPath, task); + server.setShutdownRequest(1); + if (t) + t->join(); + server.setShutdownRequest(0); + + cachedRepositoryPath = downloadPath; + ASSERT_TRUE(std::filesystem::exists(cachedRepositoryPath)); + }); + } + + void seedCurrentTestRepository() { + const std::string testRepositoryPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); + std::error_code ec; + std::filesystem::copy(cachedRepositoryPath, + testRepositoryPath, + std::filesystem::copy_options::recursive, + ec); + ASSERT_EQ(ec, std::errc()) << "Failed to copy cached model repository to test directory"; +#ifdef _WIN32 + std::string mutableRepositoryPath = testRepositoryPath; + RemoveReadonlyFileAttributeFromDir(mutableRepositoryPath); +#endif + } +}; + +std::once_flag HfPullCache::cacheInitFlag; +std::unique_ptr HfPullCache::cacheDir = nullptr; +std::string HfPullCache::cachedRepositoryPath; + const std::string expectedGraphContents = R"( input_stream: "HTTP_REQUEST_PAYLOAD:input" output_stream: "HTTP_RESPONSE_PAYLOAD:output" @@ -156,11 +229,8 @@ const std::string expectedGraphContentsDraft = R"( } )"; -TEST_F(HfDownloaderPullHfModel, PositiveDownload) { +TEST_F(HfPull, Download) { GTEST_SKIP() << "Skipping test in CI - PositiveDownloadAndStart has full scope testing."; - std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; - std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); - std::string task = "text_generation"; this->ServerPullHfModel(modelName, downloadPath, task); std::string basePath = ovms::FileSystem::joinPath({this->directoryPath, "repository", "OpenVINO", "Phi-3-mini-FastDraft-50M-int8-ov"}); @@ -271,16 +341,28 @@ class TestHfDownloader : public ovms::HfDownloader { ovms::Status CheckRepositoryStatus(bool checkUntracked) { return HfDownloader::CheckRepositoryStatus(checkUntracked); } }; -TEST_F(HfDownloaderPullHfModel, Resume) { - SKIP_AND_EXIT_IF_NOT_RUNNING_UNSTABLE(); // SSL proxy blocked workaround +TEST_F(HfPullCache, RePull) { std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); std::string task = "text_generation"; + + testing::internal::CaptureStdout(); this->ServerPullHfModel(modelName, downloadPath, task); - server.setShutdownRequest(1); - if (t) - t->join(); - server.setShutdownRequest(0); + std::string out = testing::internal::GetCapturedStdout(); + + EXPECT_NE(out.find("Path already exists on local filesystem. Skipping download to path: "), std::string::npos); + EXPECT_EQ(out.find("LFS file(s) to resume"), std::string::npos); + EXPECT_EQ(out.find(" Resuming "), std::string::npos); + // The LFS work-in-progress marker is a SIBLING of the repository directory + // (e.g. for "/repository" the marker is "/repository.lfswip"), not a child of it. + std::string lfsWipPath = ovms::libgit2::getLfsWipMarkerPath(downloadPath).string(); + EXPECT_EQ(std::filesystem::exists(lfsWipPath), false); +} + +TEST_F(HfPullCache, Resume) { + std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; + std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); + std::string task = "text_generation"; std::string ovModelName = "openvino_model.bin"; std::string basePath = ovms::FileSystem::joinPath({this->directoryPath, "repository", "OpenVINO", "Phi-3-mini-FastDraft-50M-int8-ov"}); @@ -337,16 +419,360 @@ TEST_F(HfDownloaderPullHfModel, Resume) { ASSERT_EQ(expectedDigest, resumedDigest); } -TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStart) { +// ResumeAfterShutdownRequestAndRerun +TEST_F(HfPull, ResumeShutdown) { + std::string basePath = ovms::FileSystem::joinPath({this->directoryPath, "repository", "OpenVINO", "Phi-3-mini-FastDraft-50M-int8-ov"}); + std::string modelPath = ovms::FileSystem::appendSlash(basePath) + "openvino_model.bin"; + std::string model2Path = ovms::FileSystem::appendSlash(basePath) + "openvino_detokenizer.bin"; + std::string model3Path = ovms::FileSystem::appendSlash(basePath) + "openvino_tokenizer.bin"; + std::string model4Path = ovms::FileSystem::appendSlash(basePath) + "tokenizer.model"; + std::string graphPath = ovms::FileSystem::appendSlash(basePath) + "graph.pbtxt"; + + server.setShutdownRequest(0); + int firstRunCode = EXIT_SUCCESS; + char* argv[] = {(char*)"ovms", + (char*)"--pull", + (char*)"--source_model", + (char*)modelName.c_str(), + (char*)"--model_repository_path", + (char*)downloadPath.c_str(), + (char*)"--task", + (char*)task.c_str()}; + int argc = 8; + t.reset(new std::thread([&argc, &argv, &firstRunCode, this]() { + firstRunCode = this->server.start(argc, argv); + })); + + // Wait until the large LFS file (openvino_model.bin) starts downloading before + // sending the shutdown request. A fixed sleep is unreliable: on a fast CPU/network + // machine the download may finish before the sleep expires, leaving no partial files + // and causing the EXPECT_FALSE(remainingPointers.empty()) assertion to fail. + // We poll until the model file exists as an LFS pointer or a partial download + // is in progress (lfs_part file present), then interrupt immediately. + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(60); + while (std::chrono::steady_clock::now() < deadline) { + auto lfsCandidates = ovms::libgit2::findLfsLikeFiles(downloadPath, true); + const bool hasModelPointer = std::find_if(lfsCandidates.begin(), lfsCandidates.end(), + [](const std::filesystem::path& p) { return p.filename() == "openvino_model.bin"; }) != lfsCandidates.end(); + const std::string partPath = ovms::FileSystem::appendSlash(basePath) + "openvino_model.binlfs_part"; + const bool hasPartFile = std::filesystem::exists(partPath); + if (hasModelPointer || hasPartFile) { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } + server.setShutdownRequest(1); + EnsureServerModelDownloadFinishedWithTimeout(server, 120); + if (t) + t->join(); + server.setShutdownRequest(0); + + EXPECT_NE(firstRunCode, EXIT_SUCCESS); + auto remainingPointers = ovms::libgit2::findLfsLikeFiles(downloadPath, true); + EXPECT_FALSE(remainingPointers.empty()); + + this->ServerPullHfModel(modelName, downloadPath, task); + + ASSERT_EQ(std::filesystem::exists(modelPath), true) << modelPath; + ASSERT_EQ(std::filesystem::exists(graphPath), true) << graphPath; + ASSERT_EQ(std::filesystem::file_size(modelPath), 52417240); + ASSERT_EQ(std::filesystem::file_size(model2Path), 339125); + ASSERT_EQ(std::filesystem::file_size(model3Path), 500292); + ASSERT_EQ(std::filesystem::file_size(model4Path), 499723); +} + +// PullAfterUserRemovedTrackedFileDoesNotRestoreIt +TEST_F(HfPullCache, UserRemoved) { + std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; + std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); + std::string task = "text_generation"; + + std::string basePath = ovms::FileSystem::joinPath({this->directoryPath, "repository", "OpenVINO", "Phi-3-mini-FastDraft-50M-int8-ov"}); + std::string preservedFilePath = ovms::FileSystem::appendSlash(basePath) + "openvino_model.bin"; + std::string removedFilePath = ovms::FileSystem::appendSlash(basePath) + "openvino_tokenizer.bin"; + std::string removedFilePath2 = ovms::FileSystem::appendSlash(basePath) + "tokenizer.json"; + + ASSERT_TRUE(std::filesystem::exists(preservedFilePath)); + ASSERT_TRUE(std::filesystem::exists(removedFilePath)); + ASSERT_TRUE(std::filesystem::exists(removedFilePath2)); + + std::error_code ec; + std::string preservedDigestBefore = sha256File(preservedFilePath, ec); + ASSERT_EQ(ec, std::errc()); + + ec.clear(); + ASSERT_TRUE(std::filesystem::remove(removedFilePath, ec)); + ASSERT_EQ(ec, std::errc()); + ASSERT_FALSE(std::filesystem::exists(removedFilePath)); + ec.clear(); + ASSERT_TRUE(std::filesystem::remove(removedFilePath2, ec)); + ASSERT_EQ(ec, std::errc()); + ASSERT_FALSE(std::filesystem::exists(removedFilePath2)); + + int secondRunCode = EXIT_SUCCESS; + server.setShutdownRequest(0); + char* argv[] = {(char*)"ovms", + (char*)"--pull", + (char*)"--source_model", + (char*)modelName.c_str(), + (char*)"--model_repository_path", + (char*)downloadPath.c_str(), + (char*)"--task", + (char*)task.c_str()}; + int argc = 8; + t.reset(new std::thread([&argc, &argv, &secondRunCode, this]() { + secondRunCode = this->server.start(argc, argv); + })); + + EnsureServerModelDownloadFinishedWithTimeout(server, 120); + + EXPECT_EQ(secondRunCode, EXIT_SUCCESS); + EXPECT_FALSE(std::filesystem::exists(removedFilePath)); + EXPECT_FALSE(std::filesystem::exists(removedFilePath2)); + + std::string preservedDigestAfter = sha256File(preservedFilePath, ec); + ASSERT_EQ(ec, std::errc()); + EXPECT_EQ(preservedDigestBefore, preservedDigestAfter); +} + +// PullAfterUserEditedTrackedFileDoesNotOverwriteIt +TEST_F(HfPullCache, UserEdited) { + std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; + std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); + std::string task = "text_generation"; + + std::string basePath = ovms::FileSystem::joinPath({this->directoryPath, "repository", "OpenVINO", "Phi-3-mini-FastDraft-50M-int8-ov"}); + std::string editedFilePath = ovms::FileSystem::appendSlash(basePath) + "openvino_tokenizer.bin"; + std::string editedFilePath2 = ovms::FileSystem::appendSlash(basePath) + "tokenizer.json"; + + ASSERT_TRUE(std::filesystem::exists(editedFilePath)); + ASSERT_TRUE(std::filesystem::exists(editedFilePath2)); + const std::uintmax_t originalSize = std::filesystem::file_size(editedFilePath); + const std::uintmax_t originalSize2 = std::filesystem::file_size(editedFilePath2); + + std::error_code ec; + std::string originalDigest = sha256File(editedFilePath, ec); + ASSERT_EQ(ec, std::errc()); + std::string originalDigest2 = sha256File(editedFilePath2, ec); + ASSERT_EQ(ec, std::errc()); + + ASSERT_TRUE(removeSecondHalf(editedFilePath)); + ASSERT_TRUE(removeSecondHalf(editedFilePath2)); + const std::uintmax_t editedSize = std::filesystem::file_size(editedFilePath); + const std::uintmax_t editedSize2 = std::filesystem::file_size(editedFilePath2); + ASSERT_LT(editedSize, originalSize); + ASSERT_LT(editedSize2, originalSize2); + + std::string editedDigestBeforeRerun = sha256File(editedFilePath, ec); + ASSERT_EQ(ec, std::errc()); + ASSERT_NE(originalDigest, editedDigestBeforeRerun); + std::string editedDigestBeforeRerun2 = sha256File(editedFilePath2, ec); + ASSERT_EQ(ec, std::errc()); + ASSERT_NE(originalDigest2, editedDigestBeforeRerun2); + + int secondRunCode = EXIT_SUCCESS; + server.setShutdownRequest(0); + char* argv[] = {(char*)"ovms", + (char*)"--pull", + (char*)"--source_model", + (char*)modelName.c_str(), + (char*)"--model_repository_path", + (char*)downloadPath.c_str(), + (char*)"--task", + (char*)task.c_str()}; + int argc = 8; + t.reset(new std::thread([&argc, &argv, &secondRunCode, this]() { + secondRunCode = this->server.start(argc, argv); + })); + + EnsureServerModelDownloadFinishedWithTimeout(server, 120); + + EXPECT_EQ(secondRunCode, EXIT_SUCCESS); + EXPECT_EQ(std::filesystem::file_size(editedFilePath), editedSize); + EXPECT_EQ(std::filesystem::file_size(editedFilePath2), editedSize2); + + std::string editedDigestAfterRerun = sha256File(editedFilePath, ec); + ASSERT_EQ(ec, std::errc()); + EXPECT_EQ(editedDigestBeforeRerun, editedDigestAfterRerun); + EXPECT_NE(originalDigest, editedDigestAfterRerun); + + std::string editedDigestAfterRerun2 = sha256File(editedFilePath2, ec); + ASSERT_EQ(ec, std::errc()); + EXPECT_EQ(editedDigestBeforeRerun2, editedDigestAfterRerun2); + EXPECT_NE(originalDigest2, editedDigestAfterRerun2); +} + +#ifdef _WIN32 +// Helper test used only as a child process launched by HfPull.ResumeTerminate. +TEST(HfPullWindowsWorker, ResumeTerminateChildProcess) { + const char* runWorker = std::getenv("OVMS_RESUME_TERMINATE_WORKER"); + if ((runWorker == nullptr) || (std::string(runWorker) != "1")) { + GTEST_SKIP() << "Helper test - runs only when launched by HfPull.ResumeTerminate."; + } + + const char* modelNameEnv = std::getenv("OVMS_RESUME_TERMINATE_MODEL"); + const char* downloadPathEnv = std::getenv("OVMS_RESUME_TERMINATE_DOWNLOAD_PATH"); + const char* taskEnv = std::getenv("OVMS_RESUME_TERMINATE_TASK"); + ASSERT_NE(modelNameEnv, nullptr); + ASSERT_NE(downloadPathEnv, nullptr); + ASSERT_NE(taskEnv, nullptr); + + ovms::Server& childServer = ovms::Server::instance(); + childServer.setShutdownRequest(0); + + std::string modelName = modelNameEnv; + std::string downloadPath = downloadPathEnv; + std::string task = taskEnv; + char* argv[] = {(char*)"ovms", + (char*)"--pull", + (char*)"--source_model", + (char*)modelName.c_str(), + (char*)"--model_repository_path", + (char*)downloadPath.c_str(), + (char*)"--task", + (char*)task.c_str()}; + int argc = 8; + + (void)childServer.start(argc, argv); +} +#endif + +// ResumeAfterForcedTerminationAndRerun +TEST_F(HfPull, ResumeTerminate) { + std::string basePath = ovms::FileSystem::joinPath({this->directoryPath, "repository", "OpenVINO", "Phi-3-mini-FastDraft-50M-int8-ov"}); + std::string modelPath = ovms::FileSystem::appendSlash(basePath) + "openvino_model.bin"; + std::string model2Path = ovms::FileSystem::appendSlash(basePath) + "openvino_detokenizer.bin"; + std::string model3Path = ovms::FileSystem::appendSlash(basePath) + "openvino_tokenizer.bin"; + std::string model4Path = ovms::FileSystem::appendSlash(basePath) + "tokenizer.model"; + std::string graphPath = ovms::FileSystem::appendSlash(basePath) + "graph.pbtxt"; + +#ifdef _WIN32 + char testExePath[MAX_PATH] = {0}; + DWORD exePathLen = GetModuleFileNameA(nullptr, testExePath, MAX_PATH); + ASSERT_GT(exePathLen, 0u); + ASSERT_LT(exePathLen, static_cast(MAX_PATH)); + + ASSERT_TRUE(SetEnvironmentVariableA("OVMS_RESUME_TERMINATE_WORKER", "1")); + ASSERT_TRUE(SetEnvironmentVariableA("OVMS_RESUME_TERMINATE_MODEL", modelName.c_str())); + ASSERT_TRUE(SetEnvironmentVariableA("OVMS_RESUME_TERMINATE_DOWNLOAD_PATH", downloadPath.c_str())); + ASSERT_TRUE(SetEnvironmentVariableA("OVMS_RESUME_TERMINATE_TASK", task.c_str())); + + std::string commandLine = std::string("\"") + testExePath + + "\" --gtest_filter=HfPullWindowsWorker.ResumeTerminateChildProcess"; + STARTUPINFOA si; + PROCESS_INFORMATION pi; + ZeroMemory(&si, sizeof(si)); + si.cb = sizeof(si); + ZeroMemory(&pi, sizeof(pi)); + + ASSERT_TRUE(CreateProcessA( + nullptr, + commandLine.data(), + nullptr, + nullptr, + TRUE, + 0, + nullptr, + nullptr, + &si, + &pi)); +#else + pid_t childPid = fork(); + ASSERT_NE(childPid, -1); + + if (childPid == 0) { + ovms::Server& childServer = ovms::Server::instance(); + childServer.setShutdownRequest(0); + char* argv[] = {(char*)"ovms", + (char*)"--pull", + (char*)"--source_model", + (char*)modelName.c_str(), + (char*)"--model_repository_path", + (char*)downloadPath.c_str(), + (char*)"--task", + (char*)task.c_str()}; + int argc = 8; + + std::thread childThread([&argc, &argv, &childServer]() { + (void)childServer.start(argc, argv); + }); + childThread.detach(); + + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (std::chrono::steady_clock::now() < deadline) { + auto lfsCandidates = ovms::libgit2::findLfsLikeFiles(downloadPath, true); + auto hasOpenvinoModelPointer = std::find_if(lfsCandidates.begin(), lfsCandidates.end(), + [](const std::filesystem::path& p) { return p.filename() == "openvino_model.bin"; }) != lfsCandidates.end(); + if (std::filesystem::exists(modelPath) || hasOpenvinoModelPointer) { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + kill(getpid(), SIGKILL); + _exit(1); + } + +#endif + + bool observedPartialDownload = false; + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(60); + while (std::chrono::steady_clock::now() < deadline) { + auto lfsCandidates = ovms::libgit2::findLfsLikeFiles(downloadPath, true); + auto hasOpenvinoModelPointer = std::find_if(lfsCandidates.begin(), lfsCandidates.end(), + [](const std::filesystem::path& p) { return p.filename() == "openvino_model.bin"; }) != lfsCandidates.end(); + const std::string partPath = ovms::FileSystem::appendSlash(basePath) + "openvino_model.binlfs_part"; + const bool hasPartFile = std::filesystem::exists(partPath); + if (hasOpenvinoModelPointer || hasPartFile) { + observedPartialDownload = true; + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + +#ifdef _WIN32 + ASSERT_TRUE(TerminateProcess(pi.hProcess, 1)); + ASSERT_EQ(WaitForSingleObject(pi.hProcess, 10000), WAIT_OBJECT_0); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + + ASSERT_TRUE(SetEnvironmentVariableA("OVMS_RESUME_TERMINATE_WORKER", nullptr)); + ASSERT_TRUE(SetEnvironmentVariableA("OVMS_RESUME_TERMINATE_MODEL", nullptr)); + ASSERT_TRUE(SetEnvironmentVariableA("OVMS_RESUME_TERMINATE_DOWNLOAD_PATH", nullptr)); + ASSERT_TRUE(SetEnvironmentVariableA("OVMS_RESUME_TERMINATE_TASK", nullptr)); +#else + int childStatus = 0; + ASSERT_EQ(waitpid(childPid, &childStatus, 0), childPid); + ASSERT_TRUE(WIFSIGNALED(childStatus)); + ASSERT_EQ(WTERMSIG(childStatus), SIGKILL); +#endif + + EXPECT_TRUE(observedPartialDownload); + auto remainingPointers = ovms::libgit2::findLfsLikeFiles(downloadPath, true); + EXPECT_FALSE(remainingPointers.empty()); + + this->ServerPullHfModel(modelName, downloadPath, task); + + ASSERT_EQ(std::filesystem::exists(modelPath), true) << modelPath; + ASSERT_EQ(std::filesystem::exists(graphPath), true) << graphPath; + ASSERT_EQ(std::filesystem::file_size(modelPath), 52417240); + ASSERT_EQ(std::filesystem::file_size(model2Path), 339125); + ASSERT_EQ(std::filesystem::file_size(model3Path), 500292); + ASSERT_EQ(std::filesystem::file_size(model4Path), 499723); +} + +TEST_F(HfPull, Start) { SKIP_AND_EXIT_IF_NOT_RUNNING_UNSTABLE(); // CVS-180127 // EnvGuard guard; // guard.set("HF_ENDPOINT", "https://modelscope.cn"); // guard.set("HF_ENDPOINT", "https://hf-mirror.com"); this->filesToPrintInCaseOfFailure.emplace_back("graph.pbtxt"); this->filesToPrintInCaseOfFailure.emplace_back("config.json"); - std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; - std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); - std::string task = "text_generation"; this->SetUpServerForDownloadAndStart(modelName, downloadPath, task); std::string basePath = ovms::FileSystem::joinPath({this->directoryPath, "repository", "OpenVINO", "Phi-3-mini-FastDraft-50M-int8-ov"}); @@ -361,16 +787,14 @@ TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStart) { ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; } -TEST_F(HfDownloaderPullHfModel, ModelOutOfOvOrg) { +TEST_F(HfPull, OutOfOvOrg) { SKIP_AND_EXIT_IF_NOT_RUNNING_UNSTABLE(); // CVS-180127 // EnvGuard guard; // guard.set("HF_ENDPOINT", "https://modelscope.cn"); // guard.set("HF_ENDPOINT", "https://hf-mirror.com"); - std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; - std::string downloadPath = this->directoryPath; - std::string task = "text_generation"; - this->ServerPullHfModel(modelName, downloadPath, task); + std::string downloadPathRoot = this->directoryPath; + this->ServerPullHfModel(modelName, downloadPathRoot, task); // Shutdown server.setShutdownRequest(1); @@ -400,12 +824,12 @@ TEST_F(HfDownloaderPullHfModel, ModelOutOfOvOrg) { std::string modelName2 = "META/Phi-3-mini-FastDraft-50M-int8-ov"; std::filesystem::file_time_type ftime1 = std::filesystem::last_write_time(newPath); - this->SetUpServerForDownloadAndStart(modelName2, downloadPath, task); + this->SetUpServerForDownloadAndStart(modelName2, downloadPathRoot, task); std::filesystem::file_time_type ftime2 = std::filesystem::last_write_time(newPath); ASSERT_EQ(ftime1, ftime2); } -TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStartModelOutsideOvOrg) { +TEST_F(HfPull, StartOutsideOvOrg) { SKIP_AND_EXIT_IF_NOT_RUNNING_UNSTABLE(); // CVS-180127 this->filesToPrintInCaseOfFailure.emplace_back("graph.pbtxt"); this->filesToPrintInCaseOfFailure.emplace_back("config.json"); @@ -425,15 +849,13 @@ TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStartModelOutsideOvOrg) { ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; } -TEST_F(HfDownloaderPullHfModel, DownloadDraftModel) { +TEST_F(HfPull, DraftModel) { SKIP_AND_EXIT_IF_NOT_RUNNING_UNSTABLE(); // CVS-180127 // EnvGuard guard; // guard.set("HF_ENDPOINT", "https://modelscope.cn"); // guard.set("HF_ENDPOINT", "https://hf-mirror.com"); this->filesToPrintInCaseOfFailure.emplace_back("graph.pbtxt"); - std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string draftModel = "OpenVINO/distil-small.en-int4-ov"; - std::string task = "text_generation"; this->ServerPullHfModelWithDraft(draftModel, modelName, this->directoryPath, task); std::string basePath = ovms::FileSystem::joinPath({this->directoryPath, "OpenVINO", "Phi-3-mini-FastDraft-50M-int8-ov"}); @@ -529,6 +951,17 @@ TEST(HfDownloaderClassTest, RepositoryStatusCheckErrors) { ::testing::ExitedWithCode(0), ""); } +TEST(HfDownloaderClassTest, CloneCancellationFollowsServerShutdownRequest) { + ovms::Server& server = ovms::Server::instance(); + server.setShutdownRequest(0); + EXPECT_FALSE(ovms::libgit2::isCloneCancellationRequestedFromServer()); + + server.setShutdownRequest(1); + EXPECT_TRUE(ovms::libgit2::isCloneCancellationRequestedFromServer()); + + server.setShutdownRequest(0); +} + class TestOptimumDownloaderSetup : public ::testing::Test { public: ovms::HFSettingsImpl inHfSettings; @@ -772,12 +1205,27 @@ TEST(HfDownloaderClassTest, ProtocollsWithPassword) { EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "what_ever_is_here://123!$token:123!$token@www.new_hf.com/model/name"); } -TEST_F(HfDownloaderPullHfModel, MethodsNegative) { +TEST_F(HfPull, MethodsNegative) { EXPECT_EQ(TestHfDownloader("name/test", "../some/path", "", "", "", false).downloadModel(), ovms::StatusCode::PATH_INVALID); // Library not initialized EXPECT_EQ(TestHfDownloader("name/test", ovms::IModelDownloader::getGraphDirectory(this->directoryPath, "name2/test"), "", "", "", false).downloadModel(), ovms::StatusCode::HF_GIT_CLONE_FAILED); } +TEST_F(HfPull, CloneCancelledByShutdownRequest) { + std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository_cancel"}); + std::unique_ptr hfDownloader = std::make_unique( + modelName, + ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), + "https://huggingface.co/", + "", + "", + false); + + server.setShutdownRequest(1); + EXPECT_EQ(hfDownloader->downloadModel(), ovms::StatusCode::HF_GIT_CLONE_CANCELLED); + server.setShutdownRequest(0); +} + class TestHfPullModelModule : public ovms::HfPullModelModule { public: const std::string GetHfToken() const { return HfPullModelModule::GetHfToken(); } diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 7569d90478..0afadec3b3 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -9,10 +9,10 @@ index 1b482f038..d0fe70060 100644 +.vscode .idea diff --git a/CMakeLists.txt b/CMakeLists.txt -index 31da49a88..d61c9735e 100644 +index 3e9747575..96035cdc6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -121,11 +121,6 @@ include(ExperimentalFeatures) +@@ -108,11 +108,6 @@ include(ExperimentalFeatures) add_subdirectory(src) @@ -21,8 +21,8 @@ index 31da49a88..d61c9735e 100644 - add_subdirectory(tests) -endif() - - if(BUILD_EXAMPLES) - add_subdirectory(examples) + if(BUILD_BENCHMARKS) + add_subdirectory(benchmarks) endif() diff --git a/cmake/ExperimentalFeatures.cmake b/cmake/ExperimentalFeatures.cmake index 7eff40bdb..5562acc77 100644 @@ -36,10 +36,10 @@ index 7eff40bdb..5562acc77 100644 - set(LIBGIT2_FILENAME "${LIBGIT2_FILENAME}-experimental") -endif() diff --git a/include/git2/oid.h b/include/git2/oid.h -index 0af9737a0..6d9a8b08a 100644 +index f1920648b..726b46614 100644 --- a/include/git2/oid.h +++ b/include/git2/oid.h -@@ -22,14 +22,8 @@ GIT_BEGIN_DECL +@@ -21,14 +21,8 @@ GIT_BEGIN_DECL /** The type of object id. */ typedef enum { @@ -54,26 +54,6 @@ index 0af9737a0..6d9a8b08a 100644 } git_oid_t; /* -diff --git a/include/git2/repository.h b/include/git2/repository.h -index b203576af..26309dd3f 100644 ---- a/include/git2/repository.h -+++ b/include/git2/repository.h -@@ -184,6 +184,15 @@ GIT_EXTERN(int) git_repository_open_ext( - unsigned int flags, - const char *ceiling_dirs); - -+/** -+ * Set repository url member -+ * -+ * -+ * @param repo repository handle to update. If NULL nothing occurs. -+ * @param url the remote repository to clone or run checkout against. -+ */ -+GIT_EXTERN(int) git_repository_set_url(git_repository *repo, const char *url); -+ - /** - * Open a bare repository on the serverside. - * diff --git a/include/git2/sys/filter.h b/include/git2/sys/filter.h index 60466d173..a35ad5f98 100644 --- a/include/git2/sys/filter.h @@ -111,7 +91,7 @@ index d121c588a..b54a01a4b 100644 # diff --git a/src/cli/cmd_clone.c b/src/cli/cmd_clone.c -index c18cb28d4..9f89cd1b3 100644 +index c18cb28d4..20b83e410 100644 --- a/src/cli/cmd_clone.c +++ b/src/cli/cmd_clone.c @@ -4,9 +4,9 @@ @@ -134,73 +114,74 @@ index c18cb28d4..9f89cd1b3 100644 if (!checkout) clone_opts.checkout_opts.checkout_strategy = GIT_CHECKOUT_NONE; -@@ -182,6 +183,69 @@ int cmd_clone(int argc, char **argv) +@@ -182,6 +183,70 @@ int cmd_clone(int argc, char **argv) cli_progress_finish(&progress); -+ // Code below for testing resume in native libgit2 -+ git_repository *repo2 = NULL; -+ int error = git_repository_open_ext(&repo2, local_path, 0, NULL); -+ // HEAD state info -+ bool is_detached = git_repository_head_detached(repo2) == 1; -+ bool is_unborn = git_repository_head_unborn(repo2) == 1; -+ -+ // Collect status (staged/unstaged/untracked) -+ git_status_options opts = GIT_STATUS_OPTIONS_INIT; -+ -+ opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; -+ opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED // include untracked files -+ // // | -+ // GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX -+ // // detect renames -+ // HEAD->index - not -+ // required currently and -+ // impacts performance -+ | GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; -+ -+ git_status_list *status_list = NULL; -+ ret = git_status_list_new(&status_list, repo2, &opts); -+ -+ size_t staged = 0, unstaged = 0, untracked = 0, conflicted = 0; -+ const size_t n = git_status_list_entrycount(status_list); -+ -+ for (size_t i = 0; i < n; ++i) { -+ const git_status_entry *e = git_status_byindex(status_list, i); -+ if (!e) -+ continue; -+ unsigned s = e->status; -+ -+ // Staged (index) changes -+ if (s & (GIT_STATUS_INDEX_NEW | GIT_STATUS_INDEX_MODIFIED | -+ GIT_STATUS_INDEX_DELETED | GIT_STATUS_INDEX_RENAMED | -+ GIT_STATUS_INDEX_TYPECHANGE)) -+ ++staged; -+ -+ // Unstaged (workdir) changes -+ if (s & (GIT_STATUS_WT_MODIFIED | GIT_STATUS_WT_DELETED | -+ GIT_STATUS_WT_RENAMED | GIT_STATUS_WT_TYPECHANGE)) -+ ++unstaged; -+ -+ // Untracked -+ if (s & GIT_STATUS_WT_NEW) -+ ++untracked; -+ -+ // Conflicted -+ if (s & GIT_STATUS_CONFLICTED) -+ ++conflicted; -+ } -+ -+ // Print summary (mirrors your original stream output) -+ printf("HEAD state : %s\n", -+ is_unborn ? "unborn (no commits)" : -+ (is_detached ? "detached" : "attached")); -+ printf("Staged changes : %zu\n", staged); -+ printf("Unstaged changes: %zu\n", unstaged); -+ printf("Untracked files : %zu", untracked); -+ if (conflicted) { -+ printf(" (%zu paths flagged)", conflicted); -+ } -+ printf("\n"); ++ /* Code below for testing resume in native libgit2 */ ++ { ++ git_repository *repo2 = NULL; ++ git_status_options opts = GIT_STATUS_OPTIONS_INIT; ++ git_status_list *status_list = NULL; ++ size_t staged = 0, unstaged = 0, untracked = 0, conflicted = 0; ++ size_t n, i; ++ int is_detached, is_unborn; ++ ++ git_repository_open_ext(&repo2, local_path, 0, NULL); ++ ++ /* HEAD state info */ ++ is_detached = git_repository_head_detached(repo2) == 1; ++ is_unborn = git_repository_head_unborn(repo2) == 1; ++ ++ /* Collect status (staged/unstaged/untracked) */ ++ opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; ++ opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED ++ | GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; ++ ++ ret = git_status_list_new(&status_list, repo2, &opts); ++ ++ n = git_status_list_entrycount(status_list); ++ for (i = 0; i < n; ++i) { ++ const git_status_entry *e = git_status_byindex(status_list, i); ++ unsigned int s; ++ if (!e) ++ continue; ++ s = e->status; ++ ++ /* Staged (index) changes */ ++ if (s & (GIT_STATUS_INDEX_NEW | GIT_STATUS_INDEX_MODIFIED | ++ GIT_STATUS_INDEX_DELETED | GIT_STATUS_INDEX_RENAMED | ++ GIT_STATUS_INDEX_TYPECHANGE)) ++ ++staged; ++ ++ /* Unstaged (workdir) changes */ ++ if (s & (GIT_STATUS_WT_MODIFIED | GIT_STATUS_WT_DELETED | ++ GIT_STATUS_WT_RENAMED | GIT_STATUS_WT_TYPECHANGE)) ++ ++unstaged; ++ ++ /* Untracked */ ++ if (s & GIT_STATUS_WT_NEW) ++ ++untracked; ++ ++ /* Conflicted */ ++ if (s & GIT_STATUS_CONFLICTED) ++ ++conflicted; ++ } ++ ++ /* Print summary */ ++ printf("HEAD state : %s\n", ++ is_unborn ? "unborn (no commits)" : ++ (is_detached ? "detached" : "attached")); ++ printf("Staged changes : %zu\n", staged); ++ printf("Unstaged changes: %zu\n", unstaged); ++ printf("Untracked files : %zu", untracked); ++ if (conflicted) ++ printf(" (%zu paths flagged)", conflicted); ++ printf("\n"); ++ ++ git_status_list_free(status_list); ++ git_repository_free(repo2); ++ } done: cli_progress_dispose(&progress); git__free(computed_path); @@ -261,7 +242,7 @@ index 05a67fb14..da56c42f6 100644 BOOL result; diff --git a/src/libgit2/CMakeLists.txt b/src/libgit2/CMakeLists.txt -index a7d3c7ca4..13e9dfc37 100644 +index 195be8002..d3bfe0377 100644 --- a/src/libgit2/CMakeLists.txt +++ b/src/libgit2/CMakeLists.txt @@ -5,25 +5,29 @@ add_library(libgit2 OBJECT) @@ -330,7 +311,7 @@ index a7d3c7ca4..13e9dfc37 100644 target_compile_definitions(libgit2package PRIVATE LIBGIT2_FILENAME=\"${LIBGIT2_FILENAME}\") set_target_properties(libgit2package PROPERTIES OUTPUT_NAME ${LIBGIT2_FILENAME}) diff --git a/src/libgit2/clone.c b/src/libgit2/clone.c -index 237efc0ba..d439d01cc 100644 +index e4d9c134f..2d5165c5a 100644 --- a/src/libgit2/clone.c +++ b/src/libgit2/clone.c @@ -24,6 +24,7 @@ @@ -363,7 +344,7 @@ index 237efc0ba..d439d01cc 100644 /* enforce some behavior on fetch */ options.fetch_opts.update_fetchhead = 0; -@@ -638,14 +654,22 @@ static int clone_repo( +@@ -638,14 +654,21 @@ static int clone_repo( else repository_cb = default_repository_create; @@ -376,7 +357,6 @@ index 237efc0ba..d439d01cc 100644 return error; + } -+ repo->url = git__strdup(url); if (!(error = create_and_configure_origin(&origin, repo, url, &options))) { bool clone_local; @@ -387,7 +367,7 @@ index 237efc0ba..d439d01cc 100644 return error; } -@@ -669,6 +693,8 @@ static int clone_repo( +@@ -669,6 +692,8 @@ static int clone_repo( git_error_restore(last_error); } @@ -444,10 +424,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..630573c3d +index 000000000..cde3a1f23 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,1611 @@ +@@ -0,0 +1,1935 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -471,6 +451,7 @@ index 000000000..630573c3d +#include +#include +#include ++#include +#include + +#include "array.h" @@ -479,17 +460,172 @@ index 000000000..630573c3d +#include "hash.h" +#include "oid.h" +#include "filter.h" ++#include "net.h" +#include "str.h" +#include "repository.h" ++#include "remote.h" +#include "regexp.h" ++#include "thread.h" +#include "time.h" + ++#if defined(__GNUC__) || defined(__clang__) ++/* Optional host-provided cancellation probe. ++ * If not defined by the embedding process, this stays NULL (weak symbol). ++ */ ++extern int git_lfs_shutdown_requested(void) __attribute__((weak)); ++#endif ++ ++/* ++ * Exported cancel accessors. ++ * The embedding process may set the cancellation state to non-zero at any ++ * time to abort all ongoing or future LFS downloads in this process. ++ * Resetting it to 0 before a new clone allows reuse. ++ */ ++#if defined(_WIN32) ++#define GIT_LFS_CANCEL_EXPORT __declspec(dllexport) ++#elif defined(__GNUC__) || defined(__clang__) ++#define GIT_LFS_CANCEL_EXPORT __attribute__((visibility("default"))) ++#else ++#define GIT_LFS_CANCEL_EXPORT ++#endif ++ ++GIT_LFS_CANCEL_EXPORT void git_lfs_cancel_set(int value); ++GIT_LFS_CANCEL_EXPORT int git_lfs_cancel_get(void); ++ ++static git_atomic32 g_lfs_cancel_requested; ++ ++GIT_LFS_CANCEL_EXPORT void git_lfs_cancel_set(int value) ++{ ++ git_atomic32_set(&g_lfs_cancel_requested, value); ++} ++ ++GIT_LFS_CANCEL_EXPORT int git_lfs_cancel_get(void) ++{ ++ return git_atomic32_get(&g_lfs_cancel_requested); ++} ++ +#define LFS_RESUME_ATTEMPTS_DEFAULT 5 +#define LFS_RESUME_INTERVAL_DEFAULT_SECONDS 10 + +/* Configure how many resume attempts and how long to wait between them */ -+static int g_lfs_resume_attempts = 5; /* <-- make configurable */ -+static unsigned int g_lfs_resume_interval_secs = 10; /* <-- make configurable */ ++static int g_lfs_resume_attempts = 5; ++static unsigned int g_lfs_resume_interval_secs = 10; ++ ++struct lfs_log_state { ++ char path[4096]; ++ bool truncated; ++}; ++ ++static int lfs_shutdown_requested(void) ++{ ++ /* Check the exported global flag first – reliably set from the host ++ * without requiring symbol interposition (-rdynamic). */ ++ if (git_lfs_cancel_get()) ++ return 1; ++#if defined(__GNUC__) || defined(__clang__) ++ /* Fallback: weak symbol provided by the host executable. */ ++ if (git_lfs_shutdown_requested) ++ return git_lfs_shutdown_requested() != 0; ++#endif ++ return 0; ++} ++ ++/* ++ * lfs_log_set_path ++ * ----------------- ++ * Sets the path for lfs_error.txt using the repository workdir. ++ * workdir must end with a path separator (as returned by ++ * git_repository_workdir). ++ * ++ * The log file is NOT created here. It will be created on the first ++ * error message for the current workdir. ++ */ ++static void lfs_log_set_path(struct lfs_log_state *log_state, const char *workdir) ++{ ++ char new_path[4096]; ++ ++ if (!log_state) ++ return; ++ ++ if (!workdir || !*workdir) { ++ log_state->path[0] = '\0'; ++ log_state->truncated = false; ++ return; ++ } ++ ++ snprintf(new_path, sizeof(new_path), "%slfs_error.txt", workdir); ++ ++ /* If the path changed (new clone/operation), mark for truncation on ++ * next error so errors from previous runs don't appear. */ ++ if (strcmp(new_path, log_state->path) != 0) { ++ memcpy(log_state->path, new_path, strlen(new_path) + 1); ++ log_state->truncated = false; ++ } ++} ++ ++/* ++ * lfs_log_error ++ * -------------- ++ * Writes a formatted error message to stderr and, if a log path has been ++ * set via lfs_log_set_path(), also appends the message to lfs_error.txt ++ * in the repository root directory. ++ * ++ * The log file is created only when this function is first called for a ++ * given workdir. If no errors occur, the file is never created. ++ */ ++static void lfs_log_vwrite( ++ struct lfs_log_state *log_state, ++ const char *fmt, ++ va_list ap) ++{ ++ va_list ap_copy; ++ FILE *f; ++ ++ va_copy(ap_copy, ap); ++ vfprintf(stderr, fmt, ap_copy); ++ va_end(ap_copy); ++ ++ if (log_state && log_state->path[0] != '\0') { ++ /* On first error for this workdir, truncate the file to clear ++ * any errors from previous operations */ ++ if (!log_state->truncated) { ++ f = fopen(log_state->path, "w"); ++ if (f) ++ fclose(f); ++ log_state->truncated = true; ++ } ++ ++ /* Append this error message */ ++ f = fopen(log_state->path, "a"); ++ if (f) { ++ va_copy(ap_copy, ap); ++ vfprintf(f, fmt, ap_copy); ++ va_end(ap_copy); ++ fclose(f); ++ } ++ } ++} ++ ++static void lfs_log_error(const char *fmt, ...) ++{ ++ va_list ap; ++ ++ va_start(ap, fmt); ++ lfs_log_vwrite(NULL, fmt, ap); ++ va_end(ap); ++} ++ ++static void lfs_log_error_with_state( ++ struct lfs_log_state *log_state, ++ const char *fmt, ++ ...) ++{ ++ va_list ap; ++ ++ va_start(ap, fmt); ++ lfs_log_vwrite(log_state, fmt, ap); ++ va_end(ap); ++} + +/* + * parse_env_nonneg_int @@ -530,7 +666,7 @@ index 000000000..630573c3d + val = strtoull(s, &end, 10); + + if (errno == ERANGE || end == s) { -+ fprintf(stderr, "[WARN] %s: invalid number, using default=%d\n", ++ lfs_log_error("[WARN] %s: invalid number, using default=%d\n", + env_name, default_value); + *out_value = default_value; + return -1; @@ -539,7 +675,7 @@ index 000000000..630573c3d + while (isspace((unsigned char)*end)) + end++; + if (*end != '\0') { -+ fprintf(stderr, ++ lfs_log_error( + "[WARN] %s: trailing characters ignored, using default=%d\n", + env_name, default_value); + *out_value = default_value; @@ -547,7 +683,7 @@ index 000000000..630573c3d + } + + if (val > (unsigned long long)INT_MAX) { -+ fprintf(stderr, "[WARN] %s: value too large, capping to %d\n", ++ lfs_log_error("[WARN] %s: value too large, capping to %d\n", + env_name, INT_MAX); + val = (unsigned long long)INT_MAX; + } @@ -598,7 +734,7 @@ index 000000000..630573c3d + val = strtoull(s, &end, 10); + + if (errno == ERANGE || end == s) { -+ fprintf(stderr, "[WARN] %s: invalid number, using default=%u\n", ++ lfs_log_error("[WARN] %s: invalid number, using default=%u\n", + env_name, default_value); + *out_value = default_value; + return -1; @@ -606,7 +742,7 @@ index 000000000..630573c3d + while (isspace((unsigned char)*end)) + end++; + if (*end != '\0') { -+ fprintf(stderr, ++ lfs_log_error( + "[WARN] %s: trailing characters ignored, using default=%u\n", + env_name, default_value); + *out_value = default_value; @@ -711,6 +847,7 @@ index 000000000..630573c3d + char *lfs_size; + char *url; + bool is_download; ++ struct lfs_log_state log_state; +} lfs_attrs; + +/* @@ -835,19 +972,19 @@ index 000000000..630573c3d + unsigned long long number; + errno = 0; + if (buffer == NULL) { -+ fprintf(stderr, "\n[ERROR] get_digit on NULL\n"); ++ lfs_log_error("\n[ERROR] get_digit on NULL\n"); + return 0; + } + + number = strtoull(buffer, &endptr, 10); + + if (errno == ERANGE) { -+ fprintf(stderr, "\n[ERROR] Conversion error\n"); ++ lfs_log_error("\n[ERROR] Conversion error\n"); + } + if (endptr == buffer) { -+ fprintf(stderr, "\n[ERROR] No digits were found\n"); ++ lfs_log_error("\n[ERROR] No digits were found\n"); + } else if (*endptr != '\0') { -+ fprintf(stderr, ++ lfs_log_error( + "\n[ERROR] Additional characters after number: %s\n", + endptr); + } @@ -977,7 +1114,7 @@ index 000000000..630573c3d + + /* 1) Init SHA-256 hashing context (internal API) */ + if (git_hash_ctx_init(&ctx, GIT_HASH_ALGORITHM_SHA256) < 0) { -+ fprintf(stderr, "\n[ERROR] git_hash_ctx_init failed\n"); ++ lfs_log_error("\n[ERROR] git_hash_ctx_init failed\n"); + goto error; + } + @@ -989,7 +1126,7 @@ index 000000000..630573c3d + while (remaining > 0) { + size_t n = remaining > CHUNK ? CHUNK : remaining; + if (git_hash_update(&ctx, p, n) < 0) { -+ fprintf(stderr, "\n[ERROR] git_hash_update failed\n"); ++ lfs_log_error("\n[ERROR] git_hash_update failed\n"); + goto error; + } + p += n; @@ -998,7 +1135,7 @@ index 000000000..630573c3d + + /* 3) Finalize into git_oid (32-byte raw digest for SHA-256). */ + if (git_hash_final(out->id, &ctx) < 0) { -+ fprintf(stderr, "\n[ERROR] git_hash_final failed\n"); ++ lfs_log_error("\n[ERROR] git_hash_final failed\n"); + goto error; + } + @@ -1008,7 +1145,7 @@ index 000000000..630573c3d + char hex[64 + 1]; + /* Formats full hex; no NUL added. */ + if (git_oid_fmt(hex, out) < 0) { -+ fprintf(stderr, ++ lfs_log_error( + "\n[ERROR] failure, git_oid_fmt failed\n"); + goto error; + } @@ -1068,7 +1205,7 @@ index 000000000..630573c3d + lfs_oid.type = GIT_OID_SHA256; + if (git_oid_sha256_from_git_str_blob( + &lfs_oid, from, line, sizeof(line)) < 0) { -+ fprintf(stderr, "\n[ERROR] failure, cannot calculate sha256\n"); ++ lfs_log_error("\n[ERROR] failure, cannot calculate sha256\n"); + return -1; + } + @@ -1077,26 +1214,26 @@ index 000000000..630573c3d + /* 1) version line (LFS spec requires this literal string) */ + if ((error = git_str_puts( + to, "version https://git-lfs.github.com/spec/v1\n")) < 0) { -+ fprintf(stderr, "\n[ERROR] git_str_puts failed\n"); ++ lfs_log_error("\n[ERROR] git_str_puts failed\n"); + return error; + } + + /* 2) the oid line passed by caller (must end with '\n') */ + if ((error = git_str_puts(to, line)) < 0) { -+ fprintf(stderr, "\n[ERROR] git_str_puts failed\n"); ++ lfs_log_error("\n[ERROR] git_str_puts failed\n"); + return error; + } + + if (line[strlen(line) - 1] != '\n') { + if ((error = git_str_putc(to, '\n')) < 0) { -+ fprintf(stderr, "\n[ERROR] git_str_putc failed\n"); ++ lfs_log_error("\n[ERROR] git_str_putc failed\n"); + return error; + } + } + + /* 3) size line from the original file size */ + if ((error = git_str_printf(to, "size %zu\n", from->size)) < 0) { -+ fprintf(stderr, "\n[ERROR] git_str_printf failed\n"); ++ lfs_log_error("\n[ERROR] git_str_printf failed\n"); + return error; + } + @@ -1134,6 +1271,8 @@ index 000000000..630573c3d + const char *size_regexp = "\nsize (.*)\n"; + + git_repository *repo = git_filter_source_repo(src); ++ git_remote *remote = NULL; ++ const char *remote_url = NULL; + const char *path = git_filter_source_path(src); + const char *workdir = git_repository_workdir(repo); + @@ -1143,7 +1282,11 @@ index 000000000..630573c3d + + lfs_attrs_set_path(la, path); + lfs_attrs_set_workdir(la, workdir); -+ lfs_attrs_set_url(la, repo->url); ++ if ((error = git_remote_lookup(&remote, repo, "origin")) < 0) ++ goto on_error; ++ remote_url = git_remote_url(remote); ++ lfs_attrs_set_url(la, remote_url); ++ git_remote_free(remote); + la->is_download = true; + + /* Duplicate incoming string so regex functions can modify safely */ @@ -1160,7 +1303,7 @@ index 000000000..630573c3d + goto on_error; + + if (get_lfs_info_match(&lfs_oid, obj_regexp) < 0) { -+ fprintf(stderr, ++ lfs_log_error( + "\n[ERROR] failure, cannot find lfs oid in: %s\n", + lfs_oid.ptr); + goto on_error; @@ -1169,7 +1312,7 @@ index 000000000..630573c3d + lfs_attrs_set_oid(la, lfs_oid.ptr); + + if (get_lfs_info_match(&lfs_size, size_regexp) < 0) { -+ fprintf(stderr, ++ lfs_log_error( + "\n[ERROR] failure, cannot find lfs size in: %s\n", + lfs_size.ptr); + goto on_error; @@ -1178,7 +1321,7 @@ index 000000000..630573c3d + lfs_attrs_set_size(la, lfs_size.ptr); + + if (git_repository_workdir_path(&full_path, repo, path) < 0) { -+ fprintf(stderr, ++ lfs_log_error( + "\n[ERROR] failure, cannot get repository path: %s\n", + path); + goto on_error; @@ -1306,6 +1449,7 @@ index 000000000..630573c3d + FILE *stream; + uint64_t expected_size; + uint64_t written_size; ++ struct lfs_log_state *log_state; +}; + +static const char *sizeUnits[] = { "B", "KB", "MB", "GB", "TB", NULL }; @@ -1397,6 +1541,9 @@ index 000000000..630573c3d + struct progress_data *pcs = (struct progress_data *)clientp; + time_t currentTime = time(NULL); + bool shouldPrintDueToTime = false; ++ if (lfs_shutdown_requested()) ++ return 1; /* CURLE_ABORTED_BY_CALLBACK */ ++ + GIT_UNUSED(ulnow); + GIT_UNUSED(ultotal); + if (dlnow == 0) { @@ -1450,7 +1597,8 @@ index 000000000..630573c3d + /* open file for writing */ + out->stream = fopen(out->filename, "wb"); + if (!out->stream) { -+ fprintf(stderr, ++ lfs_log_error_with_state( ++ out->log_state, + "\n[ERROR] failure, cannot open file to write: %s\n", + out->filename); + return 0; /* failure, cannot open file to write */ @@ -1460,7 +1608,8 @@ index 000000000..630573c3d + + if (out->expected_size > 0) { + if (out->written_size >= out->expected_size) { -+ fprintf(stderr, ++ lfs_log_error_with_state( ++ out->log_state, + "\n[ERROR] refusing extra download bytes for %s (expected=%" PRIu64 + ")\n", + out->filename, out->expected_size); @@ -1469,7 +1618,8 @@ index 000000000..630573c3d + + if ((uint64_t)realsize > + (out->expected_size - out->written_size)) { -+ fprintf(stderr, ++ lfs_log_error_with_state( ++ out->log_state, + "\n[ERROR] server sent more data than expected for %s (expected=%" PRIu64 + ")\n", + out->filename, out->expected_size); @@ -1484,7 +1634,8 @@ index 000000000..630573c3d + return 0; + + if (to_write != realsize) { -+ fprintf(stderr, ++ lfs_log_error_with_state( ++ out->log_state, + "\n[WARN] server sent more data than expected for %s (expected=%" PRIu64 + ")\n", + out->filename, out->expected_size); @@ -1500,7 +1651,8 @@ index 000000000..630573c3d + return CURLE_OK; + + if (ftpfile->written_size != ftpfile->expected_size) { -+ fprintf(stderr, ++ lfs_log_error_with_state( ++ ftpfile->log_state, + "\n[ERROR] downloaded size mismatch for %s (got=%" PRIu64 + ", expected=%" PRIu64 ")\n", + ftpfile->filename, ftpfile->written_size, @@ -1557,6 +1709,90 @@ index 000000000..630573c3d + } + +/* ++ * print_curl_error_details ++ * ------------------------- ++ * Prints transport/TLS context for easier debugging of cURL failures. ++ */ ++static void print_curl_error_details( ++ struct lfs_log_state *log_state, ++ CURL *curl, ++ CURLcode res, ++ const char *phase, ++ const char *error_buffer) ++{ ++ git_net_url url_info = GIT_NET_URL_INIT; ++ git_str sanitized_url = GIT_STR_INIT; ++ const char *url = NULL; ++ const char *primary_ip = NULL; ++ const char *local_ip = NULL; ++ long response_code = 0; ++ long primary_port = 0; ++ long local_port = 0; ++ long os_errno = 0; ++ long ssl_verify_result = 0; ++ ++ if (!curl) ++ return; ++ ++ (void)curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url); ++ (void)curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); ++ (void)curl_easy_getinfo(curl, CURLINFO_PRIMARY_IP, &primary_ip); ++ (void)curl_easy_getinfo(curl, CURLINFO_PRIMARY_PORT, &primary_port); ++ (void)curl_easy_getinfo(curl, CURLINFO_LOCAL_IP, &local_ip); ++ (void)curl_easy_getinfo(curl, CURLINFO_LOCAL_PORT, &local_port); ++ (void)curl_easy_getinfo(curl, CURLINFO_OS_ERRNO, &os_errno); ++ (void)curl_easy_getinfo( ++ curl, CURLINFO_SSL_VERIFYRESULT, &ssl_verify_result); ++ ++ lfs_log_error_with_state(log_state, "[ERROR] LFS %s failed: %s\n", phase, ++ curl_easy_strerror(res)); ++ ++ if (error_buffer && *error_buffer) ++ lfs_log_error_with_state(log_state, "[ERROR] cURL detail: %s\n", error_buffer); ++ ++ if (url && *url) { ++ if (git_net_url_parse(&url_info, url) == 0) { ++ git__free(url_info.username); ++ url_info.username = NULL; ++ git__free(url_info.password); ++ url_info.password = NULL; ++ ++ if (git_net_url_fmt(&sanitized_url, &url_info) == 0) ++ lfs_log_error_with_state(log_state, "[ERROR] URL: %s\n", sanitized_url.ptr); ++ else ++ lfs_log_error_with_state(log_state, "[ERROR] URL: [redacted]\n"); ++ } else { ++ lfs_log_error_with_state(log_state, "[ERROR] URL: [redacted]\n"); ++ } ++ } ++ ++ if (response_code > 0) ++ lfs_log_error_with_state(log_state, "[ERROR] HTTP status: %ld\n", response_code); ++ ++ if (primary_ip && *primary_ip) ++ lfs_log_error_with_state( ++ log_state, ++ "[ERROR] Remote endpoint: %s:%ld (local: %s:%ld)\n", ++ primary_ip, primary_port, local_ip ? local_ip : "?", ++ local_port); ++ ++ if (os_errno != 0) ++ lfs_log_error_with_state(log_state, "[ERROR] OS errno: %ld (%s)\n", os_errno, ++ strerror((int)os_errno)); ++ ++ if (res == CURLE_PEER_FAILED_VERIFICATION || ++ res == CURLE_SSL_CACERT || ++ res == CURLE_SSL_CONNECT_ERROR) ++ lfs_log_error_with_state( ++ log_state, ++ "[ERROR] TLS verify result: %ld (0 means verify passed)\n", ++ ssl_verify_result); ++ ++ git_str_dispose(&sanitized_url); ++ git_net_url_dispose(&url_info); ++} ++ ++/* + * curl_resume_url_execute + * ------------------------ + * Attempts to resume an interrupted download using HTTP Range. @@ -1587,7 +1823,8 @@ index 000000000..630573c3d + + if (ftpfile->expected_size > 0 && + (uint64_t)offset > ftpfile->expected_size) { -+ fprintf(stderr, ++ lfs_log_error_with_state( ++ ftpfile->log_state, + "\n[WARN] local partial file is larger than expected (%" PRIu64 + "), restarting download\n", + ftpfile->expected_size); @@ -1595,7 +1832,8 @@ index 000000000..630573c3d + fclose(ftpfile->stream); + ftpfile->stream = fopen(ftpfile->filename, "wb"); + if (!ftpfile->stream) { -+ fprintf(stderr, ++ lfs_log_error_with_state( ++ ftpfile->log_state, + "\n[ERROR] Cannot truncate file %s\n", + ftpfile->filename); + return -1; @@ -1609,7 +1847,7 @@ index 000000000..630573c3d + existing file + fclose(ftpfile->stream);*/ + } else { -+ fprintf(stderr, "\n[ERROR] Cannot open file %s\n", ++ lfs_log_error_with_state(ftpfile->log_state, "\n[ERROR] Cannot open file %s\n", + ftpfile->filename); + return -1; + } @@ -1617,9 +1855,13 @@ index 000000000..630573c3d + /* Tell libcurl to resume */ + curl_easy_setopt(dl_curl, CURLOPT_RESUME_FROM_LARGE, offset); + /* Perform the request, res gets the return code */ ++ if (lfs_shutdown_requested()) ++ return CURLE_ABORTED_BY_CALLBACK; + + /* Perform the request, res gets the return code */ + res = curl_easy_perform(dl_curl); ++ if (res == CURLE_OK && lfs_shutdown_requested()) ++ return CURLE_ABORTED_BY_CALLBACK; + + /* Validate that server honored Range (206) when offset > 0 */ + if (res == CURLE_OK && offset > 0) { @@ -1627,7 +1869,8 @@ index 000000000..630573c3d + curl_easy_getinfo(dl_curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code != 206) { + /* Strict resume policy: restart from zero on non-206 */ -+ fprintf(stderr, ++ lfs_log_error_with_state( ++ ftpfile->log_state, + "\n[ERROR] Server did not return 206 for resumed request (HTTP %ld), restarting from zero\n", + http_code); + @@ -1638,7 +1881,8 @@ index 000000000..630573c3d + + ftpfile->stream = fopen(ftpfile->filename, "wb"); + if (!ftpfile->stream) { -+ fprintf(stderr, ++ lfs_log_error_with_state( ++ ftpfile->log_state, + "\n[ERROR] Cannot truncate file %s\n", + ftpfile->filename); + return -1; @@ -1650,7 +1894,11 @@ index 000000000..630573c3d + dl_curl, CURLOPT_RESUME_FROM_LARGE, offset); + + /* Retry exactly once as a full download. */ ++ if (lfs_shutdown_requested()) ++ return CURLE_ABORTED_BY_CALLBACK; + res = curl_easy_perform(dl_curl); ++ if (res == CURLE_OK && lfs_shutdown_requested()) ++ return CURLE_ABORTED_BY_CALLBACK; + } + } + @@ -1682,7 +1930,11 @@ index 000000000..630573c3d +{ + CURLcode res = CURLE_OK; + int attempt; ++ unsigned int slept; + for (attempt = 1; attempt <= max_retries; ++attempt) { ++ if (lfs_shutdown_requested()) ++ return CURLE_ABORTED_BY_CALLBACK; ++ + res = curl_resume_url_execute(dl_curl, ftpfile); + + if (res == CURLE_OK) { @@ -1693,14 +1945,23 @@ index 000000000..630573c3d + return CURLE_OK; + } + -+ fprintf(stderr, "[WARN] Resume attempt %d/%d failed: %s\n", ++ lfs_log_error_with_state(ftpfile->log_state, "[WARN] Resume attempt %d/%d failed: %s\n", + attempt, max_retries, curl_easy_strerror(res)); + + if (attempt < max_retries) { ++ if (lfs_shutdown_requested()) ++ return CURLE_ABORTED_BY_CALLBACK; ++ + printf("[INFO] Waiting %u seconds before next resume attempt...\n", + interval_seconds); + fflush(stdout); -+ sleep_seconds(interval_seconds); ++ /* Sleep in short chunks so cancellation is responsive during ++ * backoff waits. */ ++ for (slept = 0; slept < interval_seconds; ++slept) { ++ if (lfs_shutdown_requested()) ++ return CURLE_ABORTED_BY_CALLBACK; ++ sleep_seconds(1); ++ } + } + } + @@ -1738,12 +1999,15 @@ index 000000000..630573c3d + struct memory response = { 0 }; + struct curl_slist *chunk = NULL; + struct FtpFile ftpfile = { 0 }; ++ char info_error_buffer[CURL_ERROR_SIZE] = { 0 }; ++ char download_error_buffer[CURL_ERROR_SIZE] = { 0 }; + const char *href_regexp = + "\"download\"\\s*:\\s*\\{\\s*\"href\":\"([^\"]+)\""; + GIT_UNUSED(self); + if (!la) { + goto cleanup; + } ++ lfs_log_set_path(&la->log_state, la->workdir); + + /* Currently only download is supoprted, no lfs file upload */ + if (!la->is_download) { @@ -1752,7 +2016,7 @@ index 000000000..630573c3d + + tmp_out_file = append_cstr_to_buffer(la->full_path, "lfs_part"); + if (tmp_out_file == NULL) { -+ fprintf(stderr, "\n[ERROR] lfs create temp filename failed\n"); ++ lfs_log_error("\n[ERROR] lfs create temp filename failed\n"); + goto cleanup; + } + @@ -1760,18 +2024,19 @@ index 000000000..630573c3d + ftpfile.filename = tmp_out_file; + ftpfile.expected_size = (uint64_t)lfs_expected_size; + ftpfile.written_size = 0; ++ ftpfile.log_state = &la->log_state; + + /* get a curl handle */ + info_curl = curl_easy_init(); + if (!info_curl) { -+ fprintf(stderr, "[ERROR] curl_easy_init(info_curl) failed\n"); ++ lfs_log_error("[ERROR] curl_easy_init(info_curl) failed\n"); + goto cleanup; + } + + if (git_str_join( + &lfs_info_url, '.', la->url, "git/info/lfs/objects/batch") < + 0) { -+ fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", ++ lfs_log_error("\n[ERROR] failed to create url '%s'\n", + la->full_path); + goto cleanup; + } @@ -1787,6 +2052,8 @@ index 000000000..630573c3d + /* First set the URL that is about to receive our POST. This URL + can just as well be an https:// URL if that is what should + receive the data. */ ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_ERRORBUFFER, info_error_buffer)); + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_URL, lfs_info_url.ptr)); + /* Add cURL resiliency */ + /* unlimited data */ @@ -1800,7 +2067,7 @@ index 000000000..630573c3d + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_LOW_SPEED_TIME, 30L)); + + if (status != CURLE_OK) { -+ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", ++ lfs_log_error("\n[ERROR] curl_easy_setopt() failed: %s\n", + curl_easy_strerror(status)); + goto cleanup; + } @@ -1811,7 +2078,7 @@ index 000000000..630573c3d + &lfs_info_data, '"', 5, + "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":", + la->lfs_oid, ",\"size\":", la->lfs_size, "}]}") < 0) { -+ fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", ++ lfs_log_error("\n[ERROR] failed to create url '%s'\n", + la->full_path); + goto cleanup; + } @@ -1829,18 +2096,35 @@ index 000000000..630573c3d + info_curl, CURLOPT_USERAGENT, "git-lfs/3.5.0")); + CURL_SETOPT(curl_easy_setopt( + info_curl, CURLOPT_WRITEDATA, (void *)&response)); ++ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_CERTINFO, 1L)); ++ /* Enable xferinfo callback on info_curl so batch request can be ++ * cancelled mid-transfer by progress_callback returning non-zero. */ ++ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_NOPROGRESS, 0L)); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_XFERINFOFUNCTION, progress_callback)); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_XFERINFODATA, &progress_d)); + + if (status != CURLE_OK) { -+ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", ++ lfs_log_error("\n[ERROR] curl_easy_setopt() failed: %s\n", + curl_easy_strerror(status)); + goto cleanup; + } + /* Perform the request, res gets the return code */ ++ if (lfs_shutdown_requested()) { ++ res = CURLE_ABORTED_BY_CALLBACK; ++ goto cleanup; ++ } + res = curl_easy_perform(info_curl); ++ if (res == CURLE_OK && lfs_shutdown_requested()) { ++ res = CURLE_ABORTED_BY_CALLBACK; ++ goto cleanup; ++ } + /* Check for errors */ + if (res != CURLE_OK) { -+ fprintf(stderr, "\n[ERROR] curl_easy_perform() failed: %s\n", -+ curl_easy_strerror(res)); ++ print_curl_error_details( ++ &la->log_state, info_curl, res, "batch request", ++ info_error_buffer); + goto cleanup; + } + @@ -1852,12 +2136,12 @@ index 000000000..630573c3d + /* get a curl handle */ + dl_curl = curl_easy_init(); + if (!dl_curl) { -+ fprintf(stderr, "[ERROR] curl_easy_init(dl_curl) failed\n"); ++ lfs_log_error("[ERROR] curl_easy_init(dl_curl) failed\n"); + goto cleanup; + } + + if (get_lfs_info_match(&res_str, href_regexp) < 0) { -+ fprintf(stderr, "[ERROR] Cannot extract LFS download URL\n"); ++ lfs_log_error("[ERROR] Cannot extract LFS download URL\n"); + goto cleanup; + } + /* Progress info */ @@ -1866,10 +2150,13 @@ index 000000000..630573c3d + /* First set the URL that is about to receive our POST. This URL + can just as well be an https:// URL if that is what should + receive the data. */ ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_ERRORBUFFER, download_error_buffer)); + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_URL, res_str.ptr)); + CURL_SETOPT(curl_easy_setopt( + dl_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA)); + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_FOLLOWLOCATION, 1L)); ++ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_CERTINFO, 1L)); + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_USE_SSL, CURLUSESSL_ALL)); + CURL_SETOPT( + curl_easy_setopt(dl_curl, CURLOPT_USERAGENT, "git-lfs/3.5.0")); @@ -1895,7 +2182,7 @@ index 000000000..630573c3d + /* for 30s */ + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_LOW_SPEED_TIME, 30L)); + if (status != CURLE_OK) { -+ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", ++ lfs_log_error("\n[ERROR] curl_easy_setopt() failed: %s\n", + curl_easy_strerror(status)); + goto cleanup; + } @@ -1915,14 +2202,23 @@ index 000000000..630573c3d + } else { + print_download_info(la->full_path, lfs_expected_size); + /* Perform the request, res gets the return code */ ++ if (lfs_shutdown_requested()) { ++ res = CURLE_ABORTED_BY_CALLBACK; ++ goto cleanup; ++ } + res = curl_easy_perform(dl_curl); ++ if (res == CURLE_OK && lfs_shutdown_requested()) { ++ res = CURLE_ABORTED_BY_CALLBACK; ++ goto cleanup; ++ } + if (res == CURLE_OK) + res = ftpfile_validate_final_size(&ftpfile); + } + + /* Check for resume of partial download error */ + if (res == CURLE_PARTIAL_FILE) { -+ fprintf(stderr, ++ lfs_log_error_with_state( ++ &la->log_state, + "[WARN] Got CURLE_PARTIAL_FILE, attempting resume sequence\n"); + res = download_with_resume( + dl_curl, &ftpfile, g_lfs_resume_attempts, @@ -1931,8 +2227,14 @@ index 000000000..630573c3d + + /* Check for errors */ + if (res != CURLE_OK) { -+ fprintf(stderr, "\n[ERROR] curl_easy_perform() failed: %s\n", -+ curl_easy_strerror(res)); ++ lfs_log_error_with_state( ++ &la->log_state, ++ "[ERROR] Error downloading object: %s (%s)\n", ++ la->path ? la->path : "(unknown)", ++ la->lfs_oid ? la->lfs_oid : "(unknown)"); ++ print_curl_error_details( ++ &la->log_state, dl_curl, res, "object download", ++ download_error_buffer); + /* Very important to close the file to write any bytes + * downloaded */ + if (ftpfile.stream) { @@ -1953,7 +2255,7 @@ index 000000000..630573c3d + if (!resumingFileByBlobFilter) { + /* File does not exist when using blob filters */ + if (p_unlink(la->full_path) < 0) { -+ fprintf(stderr, ++ lfs_log_error( + "\n[ERROR] failed to delete file '%s'\n", + la->full_path); + /* Ignore error here, react on next error */ @@ -1961,7 +2263,7 @@ index 000000000..630573c3d + } + + if (p_rename(tmp_out_file, la->full_path) < 0) { -+ fprintf(stderr, "\n[ERROR] failed to rename file to '%s'\n", ++ lfs_log_error("\n[ERROR] failed to rename file to '%s'\n", + la->full_path); + goto cleanup; + } @@ -1976,7 +2278,9 @@ index 000000000..630573c3d + * ---------------------------------------------------- + */ +cleanup: -+ fprintf(stderr, "[ERROR] LFS download failed for %s\n", ++ lfs_log_error_with_state( ++ la ? &la->log_state : NULL, ++ "[ERROR] LFS download failed for %s\n", + la ? la->full_path : "(null)"); +done: + /* Close stream if open */ @@ -2059,48 +2363,3 @@ index 000000000..630573c3d + + return f; +} -diff --git a/src/libgit2/repository.c b/src/libgit2/repository.c -index 73876424a..b5c7b5c7a 100644 ---- a/src/libgit2/repository.c -+++ b/src/libgit2/repository.c -@@ -190,6 +190,7 @@ void git_repository_free(git_repository *repo) - git__free(repo->namespace); - git__free(repo->ident_name); - git__free(repo->ident_email); -+ git__free(repo->url); - - git__memzero(repo, sizeof(*repo)); - git__free(repo); -@@ -1104,6 +1105,20 @@ static int repo_is_worktree(unsigned *out, const git_repository *repo) - return error; - } - -+int git_repository_set_url( -+ git_repository *repo, -+ const char *url) -+{ -+ char *new_url; -+ GIT_ASSERT_ARG(repo); -+ GIT_ASSERT_ARG(url); -+ new_url = git__strdup(url); -+ GIT_ERROR_CHECK_ALLOC(new_url); -+ if (repo->url) git__free(repo->url); -+ repo->url = new_url; -+ return 0; -+} -+ - int git_repository_open_ext( - git_repository **repo_ptr, - const char *start_path, -diff --git a/src/libgit2/repository.h b/src/libgit2/repository.h -index fbf143894..1890c61c1 100644 ---- a/src/libgit2/repository.h -+++ b/src/libgit2/repository.h -@@ -168,6 +168,7 @@ struct git_repository { - - intptr_t configmap_cache[GIT_CONFIGMAP_CACHE_MAX]; - git_submodule_cache *submodule_cache; -+ char *url; - }; - - GIT_INLINE(git_attr_cache *) git_repository_attr_cache(git_repository *repo) diff --git a/third_party/libgit2/libgit2_engine.bzl b/third_party/libgit2/libgit2_engine.bzl index 809fb31f1e..68fb9bdc71 100644 --- a/third_party/libgit2/libgit2_engine.bzl +++ b/third_party/libgit2/libgit2_engine.bzl @@ -24,7 +24,7 @@ def libgit2_engine(): new_git_repository( name = "libgit2_engine", remote = "https://github.com/libgit2/libgit2.git", - commit = "338e6fb681369ff0537719095e22ce9dc602dbf0", # Dec 28, 2024 - v1.9.0 + commit = "1f34e2a57a3d03f174771203b64aed2b17e8522c", # Tue Mar 31 20:34:06 2026 main build_file = "@_libgit2_engine//:BUILD", patch_args = ["-p1"], # Patch implements git-lfs filter, required for HF models download diff --git a/windows_create_package.bat b/windows_create_package.bat index 13402df4ce..533a2f82ee 100644 --- a/windows_create_package.bat +++ b/windows_create_package.bat @@ -88,10 +88,10 @@ copy /Y %cd%\bazel-out\x64_windows-opt\bin\src\openvino_genai.dll dist\windows\o if !errorlevel! neq 0 exit /b !errorlevel! copy /Y %cd%\bazel-out\x64_windows-opt\bin\src\openvino_tokenizers.dll dist\windows\ovms if !errorlevel! neq 0 exit /b !errorlevel! -copy /Y %cd%\bazel-out\x64_windows-opt\bin\src\git2.dll dist\windows\ovms -if !errorlevel! neq 0 exit /b !errorlevel! copy /Y %cd%\bazel-out\x64_windows-opt\bin\src\libcurl-x64.dll dist\windows\ovms if !errorlevel! neq 0 exit /b !errorlevel! +copy /Y %cd%\bazel-out\x64_windows-opt\bin\src\git2.dll dist\windows\ovms +if !errorlevel! neq 0 exit /b !errorlevel! :: Old package had core_tokenizers if exist %cd%\bazel-out\x64_windows-opt\bin\src\core_tokenizers.dll ( copy /Y %cd%\bazel-out\x64_windows-opt\bin\src\core_tokenizers.dll dist\windows\ovms