diff --git a/.gitignore b/.gitignore index c1f045853..4bafee42c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Generated by Cargo # will have compiled files and executables /target +vendor/** # These are backup files generated by rustfmt **/*.rs.bk @@ -16,7 +17,6 @@ .DS_Store **/*.dylib -.cursor .md # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. diff --git a/Cargo.lock b/Cargo.lock index 0585627df..033cee773 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,9 +68,9 @@ dependencies = [ [[package]] name = "aes-gcm" -version = "0.11.0-rc.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da8c919c118108f144adecad74b425b804ad075580d605d9b33c2d6d1c62a2f8" +checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ "aead 0.6.1", "aes 0.9.1", @@ -253,9 +253,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -763,20 +763,41 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bitcoin-consensus-encoding" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2d6094e2a1ba3c93b5a596fe5a10d1a10c3c6e06785cde89f693a044c01aa40" +dependencies = [ + "bitcoin-internals", +] + +[[package]] +name = "bitcoin-internals" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a30a22d1f112dde8e16be7b45c63645dc165cef254f835b3e1e9553e485cfa64" +dependencies = [ + "hex-conservative 0.3.2", +] + [[package]] name = "bitcoin-io" -version = "0.1.4" +version = "0.1.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953" +checksum = "bb5de036369d1ac59d3c1819ebc4d850f89466f5401c571a285b6ed564a4cb78" +dependencies = [ + "bitcoin-consensus-encoding", +] [[package]] name = "bitcoin_hashes" -version = "0.14.2" +version = "0.14.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ed83caece3afc59919481b33b472e1432d1abc4641ed9100be142ef5110b406" +checksum = "bca4c7abb40c8817d77403c880988cfd484f23ab2365726afb2f798363e2c4a2" dependencies = [ "bitcoin-io", - "hex-conservative", + "hex-conservative 0.2.2", ] [[package]] @@ -992,13 +1013,13 @@ dependencies = [ [[package]] name = "bstr" -version = "1.12.1" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "5cee35f73844aa3014bb606320a6c1f010249dbdf43342fe54b5a4f6a8ed4b79" dependencies = [ "memchr", "regex-automata", - "serde", + "serde_core", ] [[package]] @@ -1316,9 +1337,9 @@ dependencies = [ [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", "cipher 0.5.2", @@ -2641,18 +2662,18 @@ dependencies = [ [[package]] name = "enum-ordinalize" -version = "4.3.2" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0" +checksum = "07f808d588c10e464ea6f7d3eaed500049eff30aaac103460f61828c2d65b3eb" dependencies = [ "enum-ordinalize-derive", ] [[package]] name = "enum-ordinalize-derive" -version = "4.3.2" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" +checksum = "42e528e2d34ba8a67a1a650b86beae8ef69fc5fdb638016f386b973226590432" dependencies = [ "proc-macro2", "quote", @@ -2673,9 +2694,9 @@ dependencies = [ [[package]] name = "env_filter" -version = "1.0.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", "regex", @@ -2683,9 +2704,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ "anstream", "anstyle", @@ -3487,6 +3508,15 @@ dependencies = [ "arrayvec 0.7.7", ] +[[package]] +name = "hex-conservative" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830e599c2904b08f0834ee6337d8fe8f0ed4a63b5d9e7a7f49c0ffa06d08d360" +dependencies = [ + "arrayvec 0.7.7", +] + [[package]] name = "hex-literal" version = "0.4.1" @@ -3597,9 +3627,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ "ctutils", "subtle", @@ -3878,9 +3908,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.18.4" +version = "0.18.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" +checksum = "993f007684f2e9727160da8b960ec161264703bfd1af084fd2e34d040c9a0dd4" dependencies = [ "console", "portable-atomic", @@ -4073,9 +4103,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.29" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46" +checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" dependencies = [ "defmt", "jiff-static", @@ -4087,9 +4117,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.29" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f" +checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" dependencies = [ "proc-macro2", "quote", @@ -4157,9 +4187,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.102" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", @@ -6101,11 +6131,12 @@ dependencies = [ [[package]] name = "pkcs5" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279a91971a1d8eb1260a30938eae3be9cb67b472dffecb222fbbbe2fd2dc1453" +checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca" dependencies = [ "aes 0.9.1", + "aes-gcm 0.11.0", "cbc 0.2.1", "der 0.8.0", "pbkdf2 0.13.0", @@ -6134,7 +6165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ "der 0.8.0", - "pkcs5 0.8.0", + "pkcs5 0.8.1", "rand_core 0.10.1", "spki 0.8.0", ] @@ -6264,9 +6295,9 @@ dependencies = [ [[package]] name = "primefield" -version = "0.14.0-rc.13" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2db02b39ea98560a1fec81df6266f3c1ef7fdde06ac5ef17f69aee6101602630" +checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" dependencies = [ "crypto-bigint 0.7.5", "crypto-common 0.2.2", @@ -6708,7 +6739,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ - "chacha20 0.10.0", + "chacha20 0.10.1", "getrandom 0.4.3", "rand_core 0.10.1", ] @@ -6779,9 +6810,9 @@ dependencies = [ [[package]] name = "redis" -version = "1.2.4" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae41a63fd0b8a5372f82b21e810e09a316f5dd7efd96bf08e678fb240fc1918" +checksum = "2fa6f8e4b491d7a8ef3a9550a4d71969bd0064f46e32b8dbbcc7fc60dad94fed" dependencies = [ "arc-swap", "arcstr", @@ -7316,7 +7347,7 @@ dependencies = [ "pageant", "pbkdf2 0.13.0", "pkcs1 0.8.0-rc.4", - "pkcs5 0.8.0", + "pkcs5 0.8.1", "pkcs8 0.11.0", "polyval 0.7.1", "rand 0.10.1", @@ -8747,9 +8778,9 @@ checksum = "10db6f219196a8528f9ec904d9d45cdad692d65b0e57e72be4dedd1c5fddce36" dependencies = [ "aead 0.6.1", "aes 0.9.1", - "aes-gcm 0.11.0-rc.4", + "aes-gcm 0.11.0", "cbc 0.2.1", - "chacha20 0.10.0", + "chacha20 0.10.1", "cipher 0.5.2", "ctr 0.10.1", "ctutils", @@ -9128,7 +9159,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -9818,9 +9849,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd69c5aa8f924c7519d6372789a74eac5b94fb0f8fcf0d4a97eb0bfc3e785f39" +checksum = "b40688ea6389c8171614b25491f71d4a27946e0c7ce2da1c6de27e25abf1a0ae" dependencies = [ "serde", "stable_deref_trait", @@ -10274,9 +10305,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -10288,9 +10319,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.75" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ "js-sys", "wasm-bindgen", @@ -10298,9 +10329,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -10308,9 +10339,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", @@ -10321,9 +10352,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] @@ -10343,9 +10374,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.102" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 3393944ba..91ae43b01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ git-internal = "0.7.6" libvault-core = "0.1.0" #==== -anyhow = "1.0.102" +anyhow = "1.0.103" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" serde_urlencoded = "0.7" @@ -83,7 +83,7 @@ sea-orm-migration = "1.1.20" #==== rand = "0.10.1" smallvec = "1.15.2" -bytes = "1.11.1" +bytes = "1.12.0" chrono = { version = "0.4.45", features = ["serde"] } hex = "0.4.3" sha1 = "0.11" @@ -92,11 +92,11 @@ rsa = "0.9.10" hmac = "0.13" idgenerator = "2.0.0" -config = "0.15.23" +config = "0.15.25" reqwest = "0.13.4" lazy_static = "1.5.0" -uuid = "1.23.3" -regex = "1.12.3" +uuid = "1.23.4" +regex = "1.12.4" ed25519-dalek = "2.2.0" ctrlc = "3.5.2" cedar-policy = "4.11.2" @@ -112,7 +112,7 @@ dashmap = "6.2.1" once_cell = "1.21.4" serial_test = "3.5.0" sysinfo = "0.39.5" -http = "1.4.1" +http = "1.4.2" url = "2.5.8" jemallocator = "0.5.4" mimalloc = "0.1.52" @@ -124,7 +124,7 @@ bs58 = "0.5.1" indexmap = "2.14" envsubst = "0.2.1" directories = "6.0.0" -redis = "1.2.3" +redis = "1.3.0" redis-test = "1.0.4" rustls = "0.23.41" object_store = "0.14.0" diff --git a/ceres/README.md b/ceres/README.md index 901c7b075..a0def4500 100644 --- a/ceres/README.md +++ b/ceres/README.md @@ -1 +1,61 @@ -## Ceres Module \ No newline at end of file +# Ceres + +Monorepo domain library for Mega: Git transport, REST application logic, and shared models. + +## Module layout + +``` +ceres/src/ +├── lib.rs +├── bus/ # Transport ↔ application event bus +├── infra/ # Shared infrastructure (GitObjectCache) +├── transport/ +│ ├── protocol/ # Smart HTTP/SSH Git protocol +│ └── pack/ # receive-pack / upload-pack handlers +├── application/ +│ ├── api_service/ # MonoApiService, REST-facing ops +│ ├── code_edit/ # CL create/update pipelines + post-receive handlers +│ └── build_trigger/ # Orion build dispatch +├── model/ # HTTP/API DTOs +├── diff/, merge_checker/, lfs/ +``` + +Legacy paths (`ceres::protocol`, `ceres::pack`, `ceres::api_service`, etc.) remain as re-exports in `lib.rs` for compatibility with `mono`. + +## Dependency rules + +| Module | May depend on | Must not depend on | +|--------|---------------|-------------------| +| `transport` | `bus`, `infra`, `model`, `jupiter`, `git-internal` | `application::*` | +| `application` | `bus`, `infra`, `model`, `jupiter`, `git-internal` | `transport::*` (except bus event DTOs) | +| `bus` | Minimal shared types for events | `transport` / `application` implementations | +| `mono` (binary) | Assembles `TransportRuntime` + injects handlers | — | + +## Git push event flow + +```mermaid +sequenceDiagram + participant Client + participant Protocol as transport/protocol + participant Pack as transport/pack/MonoRepo + participant Bus as bus + participant App as application/post_receive + + Client->>Protocol: git-receive-pack + Protocol->>Pack: unpack + save_entry + Pack->>Pack: persist_mono_refs + filepath update + Pack->>Bus: MonoReceivePackFinalized + Bus->>App: handle(event) + App->>App: OnpushCodeEdit / bootstrap / build / reanchor +``` + +Import-repo pushes follow the same pattern via `ImportReceivePackFinalized` → `application/code_edit/post_receive/import.rs`. + +## Assembly (`mono`) + +`mono` constructs a `TransportRuntime` (alias: `ProtocolApiState`) with storage, `GitObjectCache`, and `RuntimeApplicationHandler`, then passes it to HTTP/SSH Git routers and REST handlers. + +```rust +let runtime = TransportRuntime::new(storage, git_object_cache); +// runtime.application handles MonoReceivePackFinalized / ImportReceivePackFinalized +``` diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs deleted file mode 100644 index a2cac1a44..000000000 --- a/ceres/src/api_service/mono_api_service.rs +++ /dev/null @@ -1,4824 +0,0 @@ -//! # Mono API Service -//! -//! This module provides the API service implementation for monorepo operations in the Mega system. -//! The `MonoApiService` struct implements the `ApiHandler` trait to provide comprehensive -//! monorepo management capabilities including file operations, merge request handling, -//! and Git-like version control functionality. -//! -//! ## Key Features -//! -//! - **File Management**: Create files and directories within the monorepo structure -//! - **Tree Operations**: Handle Git tree objects for version control -//! - **Merge Requests**: Process and merge pull/merge requests with conflict resolution -//! - **Diff Operations**: Generate content differences between commits using libra -//! - **Commit Management**: Retrieve and manage commit objects and their relationships -//! - **Storage Integration**: Seamless integration with the underlying storage layer -//! -//! ## Core Components -//! -//! - `MonoApiService`: Main service struct that wraps storage functionality -//! - `ApiHandler` implementation: Provides standardized API operations -//! - Merge request processing with automated conflict detection -//! - Tree traversal and blob extraction utilities -//! -//! ## Dependencies -//! -//! This module relies on several core components: -//! - `git_internal`: Git object handling and version control primitives -//! - `jupiter`: Storage layer abstraction and data persistence -//! - `callisto`: Database models and ORM functionality -//! - `libra`: External Git-compatible command-line tool for diff operations -//! -//! ## Usage -//! -//! The service is typically instantiated with a storage backend and used to handle -//! API requests for monorepo operations. All operations are asynchronous and return -//! appropriate error types for robust error handling. - -use std::{ - collections::{HashMap, HashSet}, - path::{Path, PathBuf}, - str::FromStr, - sync::{Arc, LazyLock}, - time::Duration, -}; - -use api_model::common::Pagination; -use async_trait::async_trait; -use bytes::Bytes; -use callisto::{ - mega_cl, mega_refs, mega_tag, mega_tree, - sea_orm_active_enums::{ConvTypeEnum, MergeStatusEnum, QueueFailureTypeEnum, QueueStatusEnum}, -}; -use common::{ - errors::{BuckError, MegaError}, - utils::{MEGA_BRANCH_NAME, ZERO_ID}, -}; -use futures::{StreamExt, stream}; -use git_internal::{ - DiffItem, - diff::Diff as GitDiff, - errors::GitError, - hash::ObjectHash, - internal::{ - metadata::EntryMeta, - object::{ - blob::Blob, - commit::Commit, - tree::{Tree, TreeItem, TreeItemMode}, - }, - }, -}; -use io_orbit::object_storage::{ObjectKey, ObjectMeta, ObjectNamespace}; -use jupiter::{ - service::buck_service::{ - CommitArtifacts, CompletePayload as SvcCompletePayload, - CompleteResponse as SvcCompleteResponse, - }, - storage::{ - Storage, - base_storage::StorageConnector, - buck_storage::{session_status, upload_status}, - mono_storage::RefUpdateData, - }, - utils::converter::{FromMegaModel, IntoMegaModel, generate_git_keep_with_timestamp}, -}; -use orion_client::OrionBuildClient; -use regex::Regex; -use tracing::debug; - -use crate::{ - api_service::{ - ApiHandler, buck_tree_builder::BuckCommitBuilder, cache::GitObjectCache, - state::ProtocolApiState, tree_ops, - }, - build_trigger::{BuildTriggerService, TriggerContext}, - code_edit::{on_edit::OneditCodeEdit, utils as edit_utils}, - diff::tree_diff, - merge_checker::CheckerRegistry, - model::{ - buck::{ - CompletePayload, CompleteResponse, DEFAULT_MODE, FileChange, - FileToUpload as ApiFileToUpload, ManifestPayload, ManifestResponse, - }, - change_list::{ClDiffFile, ClFilesChangedItemSchema, UpdateBranchStatusRes}, - git::{CreateEntryInfo, CreateEntryResult, EditFilePayload, EditFileResult}, - tag::TagInfo, - third_party::{ThirdPartyClient, ThirdPartyRepoTrait}, - }, - pack::{import_repo::ImportRepo, monorepo::MonoRepo}, - protocol::{ServiceType, SmartSession, TransportProtocol}, -}; - -#[derive(Clone)] -pub struct MonoApiService { - pub storage: Storage, - pub git_object_cache: Arc, -} - -const LARGE_CL_RENAME_DETECTION_THRESHOLD: usize = 1000; - -impl From<&MonoRepo> for MonoApiService { - fn from(mono_repo: &MonoRepo) -> Self { - MonoApiService { - storage: mono_repo.storage.clone(), - git_object_cache: mono_repo.git_object_cache.clone(), - } - } -} - -impl From<&ImportRepo> for MonoApiService { - fn from(import_repo: &ImportRepo) -> Self { - MonoApiService { - storage: import_repo.storage.clone(), - git_object_cache: import_repo.git_object_cache.clone(), - } - } -} -// Key for storing the current CLA content in the object storage -const CLA_CONTENT_OBJECT_KEY: &str = "cla/content/current.txt"; - -pub struct TreeUpdateResult { - pub updated_trees: Vec, - pub ref_updates: Vec, -} - -pub struct RefUpdate { - path: String, - tree_id: ObjectHash, -} - -struct PatchSections<'a> { - header_lines: Vec<&'a str>, - divider_line: Option<&'a str>, - payload_lines: Vec<&'a str>, - has_trailing_newline: bool, -} - -struct PagedClDiffItem { - item: DiffItem, - old_path: Option, -} - -struct CreateEntryUpdate { - update_result: TreeUpdateResult, - blob: Blob, - entry_oid: ObjectHash, - repo_path: PathBuf, - save_trees: Vec, -} - -struct ApplyChangeContext<'a> { - components: &'a [String], - chain_paths: &'a [PathBuf], - chain_trees: &'a [Tree], - tree_cache: &'a mut HashMap, - new_trees: &'a mut HashMap, -} - -/// `MonoServiceLogic` is a helper struct for `MonoApiService` containing stateless logic. -/// -/// It encapsulates the pure logic methods of `MonoApiService` that do not depend on -/// databases, caches, or other external state, making them easy to unit test and reuse. -/// -/// Usage: -/// - Methods in `MonoApiService` can delegate their core logic to these static methods. -/// - In tests, you can call `MonoServiceLogic` methods directly without initializing -/// `MonoApiService` or a database. -pub struct MonoServiceLogic; - -static PATH_NOT_EXIST_RE: LazyLock = LazyLock::new(|| { - Regex::new(r"Path '([^']+)' not exist").expect("PATH_NOT_EXIST_RE must be valid") -}); - -impl MonoServiceLogic { - pub fn clean_path_str(path: &str) -> String { - let s = path.trim_end_matches('/'); - if s.is_empty() { - "/".to_string() - } else { - s.to_string() - } - } - - /// Normalize and validate repository path. - /// - /// Rules: trim; reject empty or whitespace-only (validation error). Reject `..`, backslash, - /// Windows drive letters (e.g. `C:`), and paths starting with `:`. Strip trailing `/`; - /// input consisting only of slashes becomes `"/"`. Collapse middle repeated slashes and - /// remove `.` segments (e.g. `//project//foo` -> `/project/foo`, `project/./foo` -> `/project/foo`). - /// Paths that consist only of `.` and slashes (e.g. `"."`, `"./"`) are rejected so they do not - /// silently resolve to root. Non-empty result gets a leading `"/"` if missing. Result matches - /// mega_refs.path format. - pub fn normalize_repo_path(path: &str) -> Result { - let s = path.trim(); - if s.is_empty() { - return Err(MegaError::Buck(BuckError::ValidationError( - "Path cannot be empty".to_string(), - ))); - } - if s.contains("..") { - return Err(MegaError::Buck(BuckError::ValidationError(format!( - "Path traversal not allowed: {}", - s - )))); - } - if s.contains('\\') { - return Err(MegaError::Buck(BuckError::ValidationError(format!( - "Path must use '/' separator: {}", - s - )))); - } - if s.len() >= 2 { - let mut chars = s.chars(); - if let (Some(c1), Some(':')) = (chars.next(), chars.next()) - && c1.is_ascii_alphabetic() - { - return Err(MegaError::Buck(BuckError::ValidationError(format!( - "Absolute path not allowed (Windows drive letter detected): {}", - s - )))); - } - } - if s.starts_with(':') { - return Err(MegaError::Buck(BuckError::ValidationError( - "Path must not start with ':'".to_string(), - ))); - } - let s = s.trim_end_matches('/'); - if s.is_empty() { - return Ok("/".to_string()); - } - let parts: Vec<&str> = s - .split('/') - .filter(|p| !p.is_empty() && *p != ".") - .collect(); - let s = parts.join("/"); - if s.is_empty() { - return Err(MegaError::Buck(BuckError::ValidationError( - "Path cannot be empty or consist only of '.' segments".to_string(), - ))); - } - Ok(format!("/{}", s)) - } - - /// Enumerate candidate repo roots from the deepest directory back to `/`. - pub fn repo_root_candidates(path: &Path) -> Vec { - let mut current = PathBuf::from("/").join(path); - let mut candidates = Vec::new(); - - loop { - candidates.push(Self::clean_path_str(¤t.to_string_lossy())); - if !current.pop() { - break; - } - } - - candidates - } - - pub fn subtree_ref_path(path: &Path) -> Result { - Self::normalize_repo_path(&path.display().to_string()) - } - - pub fn update_tree_hash( - tree: Arc, - name: &str, - target_hash: ObjectHash, - ) -> Result { - let index = tree - .tree_items - .iter() - .position(|item| item.name == name) - .ok_or_else(|| GitError::CustomError(format!("Tree item '{}' not found", name)))?; - let mut items = tree.tree_items.clone(); - items[index].id = target_hash; - Tree::from_tree_items(items).map_err(|_| GitError::CustomError("Invalid tree".to_string())) - } - - /// Update parent trees along the given update chain with the new child tree hash. - /// This function prepares all updated trees and their associated ref updates. - /// Trees that do not depend on each other (e.g., sibling directories) can be updated in parallel. - /// No new commits are created; only tree objects and ref updates are produced. - pub fn build_result_by_chain( - mut path: PathBuf, - mut update_chain: Vec>, - mut updated_tree_hash: ObjectHash, - ) -> Result { - let mut updated_trees = Vec::new(); - let mut ref_updates = Vec::new(); - let mut path_str = path.to_string_lossy().to_string(); - - loop { - let clean_path = MonoServiceLogic::clean_path_str(&path_str); - let ref_path = if clean_path == "/" || clean_path.starts_with('/') { - clean_path - } else { - format!("/{clean_path}") - }; - - ref_updates.push(RefUpdate { - path: ref_path, - tree_id: updated_tree_hash, - }); - - if update_chain.is_empty() { - break; - } - - let cloned_path = path.clone(); - let name = cloned_path - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| GitError::CustomError("Invalid path".into()))?; - path.pop(); - path_str = path.to_string_lossy().to_string(); - - let tree = update_chain - .pop() - .ok_or_else(|| GitError::CustomError("Empty update chain".into()))?; - - let new_tree = MonoServiceLogic::update_tree_hash(tree, name, updated_tree_hash)?; - updated_tree_hash = new_tree.id; - updated_trees.push(new_tree); - } - - Ok(TreeUpdateResult { - updated_trees, - ref_updates, - }) - } - - /// Processes all ref updates by creating new commits and updating refs accordingly. - /// - /// This method abstracts the entire loop logic for processing ref updates, - /// creating commits for each update and managing the refs that need to be updated. - pub fn process_ref_updates( - result: &TreeUpdateResult, - refs: &[mega_refs::Model], - commit_msg: &str, - commits: &mut Vec, - updates: &mut Vec, - new_commit_id: &mut String, - ) -> Result<(), GitError> { - for update in &result.ref_updates { - if let Some(p_ref) = refs.iter().find(|r| r.path == update.path) { - let commit = Commit::from_tree_id( - update.tree_id, - vec![ObjectHash::from_str(&p_ref.ref_commit_hash).unwrap()], - commit_msg, - ); - let commit_id = commit.id.to_string(); - *new_commit_id = commit_id.clone(); - - commits.push(commit); - - let mut push_update = |ref_name: &str| { - updates.push(RefUpdateData { - path: p_ref.path.clone(), - ref_name: ref_name.to_string(), - commit_id: commit_id.to_string(), - tree_hash: update.tree_id.to_string(), - }); - }; - - push_update(&p_ref.ref_name); - if p_ref.ref_name.starts_with("refs/cl/") { - push_update(MEGA_BRANCH_NAME); - } - } - } - - Ok(()) - } - - /// Processes ref updates but only for CL refs; never touches main and supports chaining parents. - pub fn process_ref_updates_cl_only( - result: &TreeUpdateResult, - cl_ref: &mega_refs::Model, - commit_msg: &str, - parent_override: Option, - commits: &mut Vec, - updates: &mut Vec, - new_commit_id: &mut String, - ) -> Result<(), GitError> { - let mut prev_parent: Option = None; - - for update in &result.ref_updates { - let parent_ids = if let Some(prev) = prev_parent { - vec![prev] - } else if let Some(po) = parent_override { - vec![po] - } else { - vec![ObjectHash::from_str(&cl_ref.ref_commit_hash).map_err(|_| { - GitError::CustomError(format!( - "Invalid CL ref hash: {}", - cl_ref.ref_commit_hash - )) - })?] - }; - - let commit = Commit::from_tree_id(update.tree_id, parent_ids, commit_msg); - let commit_id = commit.id; - *new_commit_id = commit_id.to_string(); - - commits.push(commit.clone()); - prev_parent = Some(commit_id); - - updates.push(RefUpdateData { - path: cl_ref.path.clone(), - ref_name: cl_ref.ref_name.clone(), - commit_id: commit_id.to_string(), - tree_hash: update.tree_id.to_string(), - }); - } - - Ok(()) - } - - /// Maps each TreeItem in a Tree to its corresponding Commit, if available. - /// - /// # Arguments - /// - /// * `tree` - The tree containing the TreeItems to map. - /// * `item_to_commit_id` - Mapping from TreeItem id (as string) to commit id. - /// * `commit_map` - Mapping from commit id to Commit object. - /// - /// # Returns - /// - /// A HashMap where each TreeItem maps to an Option. If a commit cannot - /// be found, the value is None. - pub fn map_tree_items_to_commits( - tree: Tree, - item_to_commit_id: &HashMap, - commit_map: &HashMap, - ) -> HashMap> { - let mut result: HashMap> = HashMap::new(); - - for item in tree.tree_items { - if let Some(commit_id) = item_to_commit_id.get(&item.id.to_string()) { - let commit = commit_map.get(commit_id).cloned(); - if commit.is_none() { - tracing::warn!( - item_name = %item.name, - item_mode = ?item.mode, - commit_id = %commit_id, - "failed fetch from commit map" - ); - } - result.insert(item, commit); - } else { - result.insert(item, None); - } - } - result - } -} - -#[async_trait] -impl ApiHandler for MonoApiService { - fn get_context(&self) -> Storage { - self.storage.clone() - } - - fn object_cache(&self) -> &GitObjectCache { - &self.git_object_cache - } - - async fn get_root_commit(&self) -> Result { - let storage = self.storage.mono_storage(); - let refs = storage.get_main_ref("/").await.unwrap().unwrap(); - self.get_commit_by_hash(&refs.ref_commit_hash).await - } - - /// Save file edit in monorepo with optimistic concurrency check - async fn save_file_edit(&self, payload: EditFilePayload) -> Result { - let file_path = PathBuf::from("/").join(PathBuf::from(&payload.path)); - let parent_path = file_path - .parent() - .ok_or_else(|| GitError::CustomError("Invalid file path".to_string()))?; - let cl_root_path = MonoServiceLogic::subtree_ref_path(parent_path) - .map_err(|e| GitError::CustomError(e.to_string()))?; - let build_repo_path = match edit_utils::resolve_build_repo_root( - &self.storage, - &cl_root_path, - ) - .await - { - Ok(path) => path, - Err(e) => { - tracing::warn!( - repo_path = %cl_root_path, - "Failed to resolve build repo root for edit, fallback to CL subtree root: {}", - e - ); - cl_root_path.clone() - } - }; - - let parent_tree = tree_ops::search_tree_by_path(self, parent_path, None) - .await? - .ok_or(GitError::CustomError(format!( - "invalid repo_path {}, Parent tree not found", - cl_root_path - )))?; - - let file_name = file_path - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| GitError::CustomError("Invalid file name".to_string()))?; - - let _current_item = parent_tree - .tree_items - .iter() - .find(|x| x.name == file_name && x.mode == TreeItemMode::Blob) - .ok_or_else(|| GitError::CustomError("[code:404] File not found".to_string()))?; - - // Create new blob and build update result up to root - let new_blob = Blob::from_content(&payload.content); - let new_tree = MonoServiceLogic::update_tree_hash( - parent_tree.into(), - file_path - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| GitError::CustomError("Invalid path".into()))?, - new_blob.id, - )?; - - let mut update_chain = self.search_tree_for_update(parent_path).await?; - let _target_tree = update_chain - .pop() - .ok_or_else(|| GitError::CustomError("Empty update chain".to_string()))?; - let update_result = MonoServiceLogic::build_result_by_chain( - parent_path.to_path_buf(), - update_chain, - new_tree.id, - )?; - let target_tree_id = Self::ref_update_tree_id_for_path(&update_result, &build_repo_path) - .ok_or_else(|| { - GitError::CustomError(format!( - "Missing updated tree for build repo root {build_repo_path}" - )) - })?; - - let src_commit = - edit_utils::get_repo_main_latest_commit(&self.storage, &build_repo_path).await?; - let dst_commit = Commit::from_tree_id( - target_tree_id, - vec![ - ObjectHash::from_str(&src_commit.id.to_string()).map_err(|e| { - GitError::CustomError(format!("Invalid commit hash {}: {e}", src_commit.id)) - })?, - ], - &payload.commit_message, - ); - let new_commit_id = dst_commit.id.to_string(); - - let username = payload - .author_username - .clone() - .unwrap_or("Anonymous".to_string()); - - self.storage - .mono_service - .mono_storage - .save_mega_commits(vec![dst_commit], None) - .await?; - - let mut all_trees = vec![new_tree]; - all_trees.extend(update_result.updated_trees); - let save_trees: Vec = all_trees - .into_iter() - .map(|save_t| { - let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new()); - tree_model.commit_id.clone_from(&new_commit_id); - tree_model.into() - }) - .collect(); - - self.storage - .mono_service - .mono_storage - .batch_save_model(save_trees) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - let editor = OneditCodeEdit::from( - &build_repo_path, - MEGA_BRANCH_NAME - .strip_prefix("refs/heads/") - .unwrap_or(MEGA_BRANCH_NAME), - &src_commit.id.to_string(), - self, - self.storage.mono_storage(), - ); - let cl = editor - .find_or_create_cl_for_edit( - &self.storage, - &editor, - payload.mode, - &new_commit_id, - &username, - ) - .await?; - - self.storage - .mono_service - .save_blobs(&new_commit_id, vec![new_blob.clone()]) - .await?; - - if !payload.skip_build { - self.trigger_build_for_cl(&editor, &cl, &username).await?; - } - - Ok(EditFileResult { - commit_id: new_commit_id, - new_oid: new_blob.id.to_string(), - path: build_repo_path, - cl_link: Some(cl.link), - }) - } - - /// Creates a new file or directory in the monorepo based on the provided file information. - /// - /// # Arguments - /// - /// * `entry_info` - Information about the file or directory to create. - /// - /// # Returns - /// - /// Returns commit metadata on success, or a `GitError` on failure. - async fn create_monorepo_entry( - &self, - entry_info: CreateEntryInfo, - ) -> Result { - let storage = self.storage.mono_storage(); - let CreateEntryUpdate { - update_result, - blob, - entry_oid, - repo_path, - mut save_trees, - } = self.prepare_create_entry_update(&entry_info).await?; - - let repo_path_str = MonoServiceLogic::subtree_ref_path(&repo_path) - .map_err(|e| GitError::CustomError(e.to_string()))?; - let build_repo_path = match edit_utils::resolve_build_repo_root( - &self.storage, - &repo_path_str, - ) - .await - { - Ok(path) => path, - Err(e) => { - tracing::warn!( - repo_path = %repo_path_str, - "Failed to resolve build repo root for create entry, fallback to CL subtree root: {}", - e - ); - repo_path_str.clone() - } - }; - - let src_commit = - edit_utils::get_repo_main_latest_commit(&self.storage, &build_repo_path).await?; - let base_commit = ObjectHash::from_str(&src_commit.id.to_string()).map_err(|e| { - GitError::CustomError(format!("Invalid commit hash {}: {e}", src_commit.id)) - })?; - let target_tree_id = Self::ref_update_tree_id_for_path(&update_result, &build_repo_path) - .ok_or_else(|| { - GitError::CustomError(format!( - "Missing updated tree for build repo root {build_repo_path}" - )) - })?; - let dst_commit = - Commit::from_tree_id(target_tree_id, vec![base_commit], &entry_info.commit_msg()); - let new_commit_id = dst_commit.id.to_string(); - - let username = entry_info - .author_username - .clone() - .unwrap_or("Anonymous".to_string()); - - let new_oid = entry_oid.to_string(); - - let mut all_trees = update_result.updated_trees; - all_trees.append(&mut save_trees); - let save_trees: Vec = all_trees - .into_iter() - .map(|save_t| { - let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new()); - tree_model.commit_id.clone_from(&new_commit_id); - tree_model.into() - }) - .collect(); - self.storage - .mono_service - .save_blobs(&new_commit_id, vec![blob]) - .await?; - - storage - .batch_save_model(save_trees) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - self.storage - .mono_service - .mono_storage - .save_mega_commits(vec![dst_commit], None) - .await?; - - let editor = OneditCodeEdit::from( - &build_repo_path, - MEGA_BRANCH_NAME - .strip_prefix("refs/heads/") - .unwrap_or(MEGA_BRANCH_NAME), - &src_commit.id.to_string(), - self, - self.storage.mono_storage(), - ); - let cl = editor - .find_or_create_cl_for_edit( - &self.storage, - &editor, - entry_info.mode.clone(), - &new_commit_id, - &username, - ) - .await?; - - if !entry_info.skip_build { - self.trigger_build_for_cl(&editor, &cl, &username).await?; - } - - let entry_path = Self::build_entry_path(&entry_info.path, &entry_info.name); - - Ok(CreateEntryResult { - commit_id: new_commit_id, - new_oid, - path: entry_path, - cl_link: Some(cl.link), - }) - } - - fn strip_relative(&self, path: &Path) -> Result { - Ok(path.to_path_buf()) - } - - async fn get_root_tree(&self, refs: Option<&str>) -> Result { - let refs = refs.unwrap_or("").trim(); - - // condition 1: empty refs, return default root tree - if refs.is_empty() { - let storage = self.storage.mono_storage(); - let refs = storage.get_main_ref("/").await.unwrap().unwrap(); - return self.get_tree_by_hash(&refs.ref_tree_hash).await; - } - - // condition 2: commit hash - if refs.len() == 40 && refs.chars().all(|c| c.is_ascii_hexdigit()) { - let commit = self.get_commit_by_hash(refs).await?; - return self.get_tree_by_hash(&commit.tree_id.to_string()).await; - } - - // condition 3: tag name - if let Ok(Some(tag)) = self.get_tag(None, refs.to_string()).await { - let commit = self.get_commit_by_hash(&tag.object_id).await?; - return self.get_tree_by_hash(&commit.tree_id.to_string()).await; - } - - // condition 4: invalid refs - Err(MegaError::Other(format!( - "Invalid refs: '{}' is not a valid commit hash or tag", - refs - ))) - } - - async fn get_tree_by_hash(&self, hash: &str) -> Result { - let model = self - .storage - .mono_storage() - .get_tree_by_hash(hash) - .await? - .ok_or_else(|| MegaError::NotFound(format!("tree not found: {}", hash)))?; - Ok(Tree::from_mega_model(model)) - } - - async fn get_commit_by_hash(&self, hash: &str) -> Result { - let model = self - .storage - .mono_storage() - .get_commit_by_hash(hash) - .await? - .ok_or_else(|| MegaError::NotFound(format!("commit not found: {}", hash)))?; - Ok(Commit::from_mega_model(model)) - } - - async fn item_to_commit_map( - &self, - path: PathBuf, - reference: Option<&str>, - ) -> Result>, GitError> { - match tree_ops::search_tree_by_path(self, &path, reference).await? { - Some(tree) => { - let mut item_to_commit = HashMap::new(); - - let storage = self.storage.mono_storage(); - let tree_hashes = tree - .tree_items - .iter() - .filter(|x| x.mode == TreeItemMode::Tree) - .map(|x| x.id.to_string()) - .collect(); - let trees = storage.get_trees_by_hashes(tree_hashes).await.unwrap(); - for tree in trees { - // Skip invalid/empty commit ids to avoid noise and incorrect mapping - if !tree.commit_id.is_empty() { - item_to_commit.insert(tree.tree_id, tree.commit_id); - } - } - - let blob_hashes = tree - .tree_items - .iter() - .filter(|x| x.mode == TreeItemMode::Blob) - .map(|x| x.id.to_string()) - .collect(); - let blobs = storage.get_mega_blobs_by_hashes(blob_hashes).await.unwrap(); - for blob in blobs { - if !blob.commit_id.is_empty() { - item_to_commit.insert(blob.blob_id, blob.commit_id); - } - } - - let commit_ids: HashSet = item_to_commit.values().cloned().collect(); - let commits = self - .get_commits_by_hashes(commit_ids.into_iter().collect()) - .await - .unwrap(); - - let commit_map: HashMap = - commits.into_iter().map(|x| (x.id.to_string(), x)).collect(); - - Ok(MonoServiceLogic::map_tree_items_to_commits( - tree, - &item_to_commit, - &commit_map, - )) - } - None => Ok(HashMap::new()), - } - } - - async fn get_commits_by_hashes(&self, c_hashes: Vec) -> Result, GitError> { - let commits = self - .storage - .mono_storage() - .get_commits_by_hashes(&c_hashes) - .await - .unwrap(); - Ok(commits.into_iter().map(Commit::from_mega_model).collect()) - } - - // helper to convert mega_tag model into TagInfo (defined on MonoApiService below) - async fn create_tag( - &self, - repo_path: Option, - name: String, - target: Option, - tagger_name: Option, - tagger_email: Option, - message: Option, - ) -> Result { - let mono_storage = self.storage.mono_storage(); - - let is_annotated = message.as_ref().map(|s| !s.is_empty()).unwrap_or(false); - let tagger_info = match (tagger_name, tagger_email) { - (Some(n), Some(e)) => format!("{} <{}>", n, e), - (Some(n), None) => n, - (None, Some(e)) => e, - (None, None) => "unknown".to_string(), - }; - - // validate target commit presence - self.validate_target_commit_mono(target.as_ref()).await?; - - let full_ref = format!("refs/tags/{}", name.clone()); - - // Prevent duplicate tag/ref creation - match mono_storage.get_tag_by_name(&name).await { - Ok(Some(_)) => { - return Err(GitError::CustomError(format!( - "[code:400] Tag '{}' already exists", - name - ))); - } - Ok(None) => {} - Err(e) => { - tracing::error!("DB error while checking tag existence: {}", e); - return Err(GitError::CustomError("[code:500] DB error".to_string())); - } - } - - if let Ok(Some(_)) = mono_storage.get_ref_by_name(&full_ref).await { - return Err(GitError::CustomError(format!( - "[code:400] Tag '{}' already exists", - name - ))); - } - - if is_annotated { - return self - .create_annotated_tag_mono( - repo_path.clone(), - name.clone(), - target.clone(), - tagger_info.clone(), - message.clone(), - full_ref.clone(), - ) - .await; - } - - // lightweight - self.create_lightweight_tag_mono( - repo_path.clone(), - name.clone(), - target.clone(), - tagger_info.clone(), - full_ref.clone(), - ) - .await - } - - async fn list_tags( - &self, - repo_path: Option, - pagination: Pagination, - ) -> Result<(Vec, u64), GitError> { - let mono_storage = self.storage.mono_storage(); - // annotated tags from DB (paged) - let (annotated_page, annotated_total) = - match mono_storage.get_tags_by_page(pagination.clone()).await { - Ok(v) => v, - Err(e) => { - tracing::error!("DB error while listing tags: {}", e); - return Err(GitError::CustomError("[code:500] DB error".to_string())); - } - }; - - let mut result: Vec = annotated_page - .into_iter() - .map(|t| self.tag_model_to_info(t)) - .collect(); - - // lightweight refs from refs table under path - let repo_path = repo_path.as_deref().unwrap_or("/"); - let mut lightweight_refs: Vec = vec![]; - if let Ok(refs) = mono_storage.get_all_refs(repo_path, false).await { - for r in refs { - if r.ref_name.starts_with("refs/tags/") { - let tag_name = r.ref_name.trim_start_matches("refs/tags/").to_string(); - if result.iter().any(|t| t.name == tag_name) { - continue; - } - lightweight_refs.push(TagInfo { - name: tag_name.clone(), - tag_id: r.ref_commit_hash.clone(), - object_id: r.ref_commit_hash.clone(), - object_type: "commit".to_string(), - tagger: "".to_string(), - message: "".to_string(), - created_at: r.created_at.and_utc().to_rfc3339(), - }); - } - } - } - - let total = annotated_total + lightweight_refs.len() as u64; - let per_page = if pagination.per_page == 0 { - 20 - } else { - pagination.per_page - } as usize; - if result.len() < per_page { - let need = per_page - result.len(); - for r in lightweight_refs.into_iter().take(need) { - result.push(r); - } - } - - Ok((result, total)) - } - - async fn get_tag( - &self, - repo_path: Option, - name: String, - ) -> Result, GitError> { - let mono_storage = self.storage.mono_storage(); - // check annotated DB first - match mono_storage.get_tag_by_name(&name).await { - Ok(Some(tag)) => return Ok(Some(self.tag_model_to_info(tag))), - Ok(None) => {} - Err(e) => { - tracing::error!("DB error while getting tag: {}", e); - return Err(GitError::CustomError("[code:500] DB error".to_string())); - } - } - // check refs for lightweight tag - let _repo_path = repo_path.unwrap_or_else(|| "/".to_string()); - let full_ref = format!("refs/tags/{}", name.clone()); - if let Ok(Some(r)) = mono_storage.get_ref_by_name(&full_ref).await { - return Ok(Some(TagInfo { - name: name.clone(), - tag_id: r.ref_commit_hash.clone(), - object_id: r.ref_commit_hash.clone(), - object_type: "commit".to_string(), - tagger: "".to_string(), - message: "".to_string(), - created_at: r.created_at.and_utc().to_rfc3339(), - })); - } - Ok(None) - } - - async fn delete_tag(&self, repo_path: Option, name: String) -> Result<(), GitError> { - let mono_storage = self.storage.mono_storage(); - // check annotated in DB first - match mono_storage.get_tag_by_name(&name).await { - Ok(Some(_tag)) => { - // remove ref if exists - let full_ref = format!("refs/tags/{}", name.clone()); - if let Ok(Some(r)) = mono_storage.get_ref_by_name(&full_ref).await { - mono_storage.remove_ref(r).await.map_err(|e| { - tracing::error!("Failed to remove ref while deleting annotated tag: {}", e); - GitError::CustomError("[code:500] Failed to remove ref".to_string()) - })?; - } - mono_storage.delete_tag_by_name(&name).await.map_err(|e| { - tracing::error!("DB delete error when deleting annotated tag: {}", e); - GitError::CustomError("[code:500] DB delete error".to_string()) - })?; - Ok(()) - } - Ok(None) => { - // try delete lightweight ref - let _repo_path = repo_path.unwrap_or_else(|| "/".to_string()); - let full_ref = format!("refs/tags/{}", name.clone()); - // find ref by name and remove - if let Ok(Some(r)) = mono_storage.get_ref_by_name(&full_ref).await { - mono_storage.remove_ref(r).await.map_err(|e| { - tracing::error!( - "Failed to remove ref while deleting lightweight tag: {}", - e - ); - GitError::CustomError("[code:500] Failed to remove ref".to_string()) - })?; - Ok(()) - } else { - Err(GitError::CustomError( - "[code:404] Tag not found".to_string(), - )) - } - } - Err(e) => { - tracing::error!("DB error while deleting tag: {}", e); - Err(GitError::CustomError("[code:500] DB error".to_string())) - } - } - } -} - -impl MonoApiService { - async fn prepare_create_entry_update( - &self, - entry_info: &CreateEntryInfo, - ) -> Result { - let path = PathBuf::from(&entry_info.path); - let mut save_trees = vec![]; - let file_content = if entry_info.is_directory { - None - } else { - Some(entry_info.content.as_deref().ok_or_else(|| { - GitError::CustomError("content is required for file creation".to_string()) - })?) - }; - - // Try to get the update chain for the given path. - // If the path exists, return an empty missing_parts and prefix. - // If part of the path does not exist, extract the missing segments (missing_parts), - // determine the valid existing prefix, and rebuild the update_chain from that prefix. - let (missing_parts, prefix, mut update_chain) = - match self.search_tree_for_update(&path).await { - Ok(chain) => (Vec::new(), "", chain), - Err(err) => { - // If search_tree_for_update failed, try to extract the - // portion of the path that does not exist from the - // error message. The error message is expected to - // contain a substring like: Path '.../missing' not exist - // We capture that substring to determine which segments - // need to be created. - let err_str = err.to_string(); - let extracted = PATH_NOT_EXIST_RE - .captures(&err_str) - .map(|caps| caps[1].to_string()) - .ok_or_else(|| { - GitError::CustomError(format!("Path resolution failed: {err_str}")) - })?; - - // missing_parts: the trailing path segments after the - // first occurrence of the extracted non-existent path. - // Example: entry_info.path = "a/b/c/d" and extracted = "c/d" - // Then missing_parts = ["c", "d"] - let missing_parts = entry_info - .path - .find(&extracted) - .map(|pos| &entry_info.path[pos..]) - .map(|sub| sub.split('/').collect::>()) - .unwrap_or_default(); - - if missing_parts.is_empty() { - return Err(GitError::CustomError(format!( - "Missing path segments for '{}': {err_str}", - entry_info.path - ))); - } - - // prefix: the valid existing path before the missing parts. - // Using the same example above, prefix = "a/b/" - let prefix = entry_info - .path - .find(&extracted) - .map(|pos| &entry_info.path[..pos]) - .unwrap_or(""); - - // Rebuild the update chain starting from the valid prefix - // so subsequent operations only update from that known - // existing tree downward. - let chain = self.search_tree_for_update(Path::new(prefix)).await?; - (missing_parts, prefix, chain) - } - }; - - let target_items = update_chain - .pop() - .ok_or_else(|| GitError::CustomError("Empty update chain".to_string()))? - .tree_items - .clone(); - - // If there are no missing parts, we are inserting directly into an - // existing tree. This branch handles both creating a new file or - // creating a new directory in the target tree. - let (update_result, blob, entry_oid, repo_path) = if missing_parts.is_empty() { - let mut target_items = target_items; - - // Check for duplicate - let is_tree_mode = if entry_info.is_directory { - TreeItemMode::Tree - } else { - TreeItemMode::Blob - }; - if target_items - .iter() - .any(|x| x.mode == is_tree_mode && x.name == entry_info.name) - { - return Err(GitError::CustomError("Duplicate name".to_string())); - } - - // Create a new tree item based on whether it's a directory or file - let (new_item, blob, entry_oid) = if entry_info.is_directory { - // For a new directory, create a .gitkeep blob so the - // directory can be represented as a tree with at least - // one blob entry. The blob contains a timestamp so it's - // unique. - let blob = generate_git_keep_with_timestamp(); - let tree_item = TreeItem { - mode: TreeItemMode::Blob, - id: blob.id, - name: String::from(".gitkeep"), - }; - let new_dir_tree = Tree::from_tree_items(vec![tree_item]).unwrap(); - save_trees.push(new_dir_tree.clone()); - let entry_oid = new_dir_tree.id; - ( - TreeItem { - mode: TreeItemMode::Tree, - id: new_dir_tree.id, - name: entry_info.name.clone(), - }, - blob, - entry_oid, - ) - } else { - let content = file_content - .ok_or_else(|| GitError::CustomError("Missing file content".to_string()))?; - let blob = Blob::from_content(content); - let entry_oid = blob.id; - ( - TreeItem { - mode: TreeItemMode::Blob, - id: blob.id, - name: entry_info.name.clone(), - }, - blob, - entry_oid, - ) - }; - - target_items.push(new_item); - target_items.sort_by(|a, b| a.name.cmp(&b.name)); - let target_tree = Tree::from_tree_items(target_items).unwrap(); - save_trees.push(target_tree.clone()); - - // Build update instructions for parent trees and refs. - // build_result_by_chain walks the update_chain (parent trees) - // and prepares the list of updated trees and ref updates - // that must be applied to persist the change. - let update_result = MonoServiceLogic::build_result_by_chain( - if prefix.is_empty() { - path.clone() - } else { - PathBuf::from(prefix) - }, - update_chain, - target_tree.id, - )?; - let repo_path = if prefix.is_empty() { - path.clone() - } else { - PathBuf::from(prefix) - }; - (update_result, blob, entry_oid, repo_path) - } else { - // If missing_parts is not empty, we must create intermediate - // directories (trees) for each missing segment. This branch - // constructs the leaf tree first and then wraps it with - // additional trees for each missing path component up to the - // existing prefix. - // Create a new tree item based on whether it's a directory or file - let (leaf_item, blob, entry_oid) = if entry_info.is_directory { - // Create .gitkeep blob and an initial tree for the new - // directory leaf. This represents the directory's own - // tree object which will be nested under new parent trees. - let blob = generate_git_keep_with_timestamp(); - let tree_item = TreeItem { - mode: TreeItemMode::Blob, - id: blob.id, - name: String::from(".gitkeep"), - }; - let new_dir_tree = Tree::from_tree_items(vec![tree_item]).unwrap(); - save_trees.push(new_dir_tree.clone()); - let entry_oid = new_dir_tree.id; - ( - TreeItem { - mode: TreeItemMode::Tree, - id: new_dir_tree.id, - name: entry_info.name.clone(), - }, - blob, - entry_oid, - ) - } else { - let content = file_content - .ok_or_else(|| GitError::CustomError("Missing file content".to_string()))?; - let blob = Blob::from_content(content); - let entry_oid = blob.id; - ( - TreeItem { - mode: TreeItemMode::Blob, - id: blob.id, - name: entry_info.name.clone(), - }, - blob, - entry_oid, - ) - }; - - let mut current_tree = Tree::from_tree_items(vec![leaf_item]).unwrap(); - save_trees.push(current_tree.clone()); - - // Wrap the leaf tree with trees for each missing parent segment. - // We iterate the missing parts in reverse (from leaf's parent up - // to the topmost missing segment) and create a tree object for - // each level that points to the previously built child tree. - let missing_len = missing_parts.len(); - for part in missing_parts.iter().rev().take(missing_len - 1) { - let sub_item = TreeItem { - mode: TreeItemMode::Tree, - id: current_tree.id, - name: part.to_string(), - }; - - current_tree = Tree::from_tree_items(vec![sub_item]).unwrap(); - save_trees.push(current_tree.clone()); - } - - // top_part is the highest-level missing segment (closest to the - // existing prefix). We'll insert this as a child into the - // existing target_items collected from the update chain. - let top_part = missing_parts - .first() - .expect("missing_parts is non-empty by branch condition") - .to_string(); - let top_item = TreeItem { - mode: TreeItemMode::Tree, - id: current_tree.id, - name: top_part.clone(), - }; - - let mut target_items = target_items; - - // Check for duplicate - if target_items - .iter() - .any(|x| x.mode == TreeItemMode::Tree && x.name == top_part) - { - return Err(GitError::CustomError("Duplicate name".to_string())); - } - - target_items.push(top_item); - target_items.sort_by(|a, b| a.name.cmp(&b.name)); - let target_tree = Tree::from_tree_items(target_items).unwrap(); - save_trees.push(target_tree.clone()); - - // After constructing the nested trees, build update instructions - // and apply them to update the parent trees and refs so the - // new nested directory/file is persisted in the repository. - let update_result = MonoServiceLogic::build_result_by_chain( - PathBuf::from(prefix), - update_chain, - target_tree.id, - )?; - let repo_path = PathBuf::from(prefix); - (update_result, blob, entry_oid, repo_path) - }; - - Ok(CreateEntryUpdate { - update_result, - blob, - entry_oid, - repo_path, - save_trees, - }) - } - - fn build_entry_path(path: &str, name: &str) -> String { - let trimmed = path.trim_end_matches('/'); - if trimmed.is_empty() || trimmed == "/" { - format!("/{name}") - } else { - format!("{trimmed}/{name}") - } - } - - fn ref_update_tree_id_for_path( - result: &TreeUpdateResult, - repo_path: &str, - ) -> Option { - let normalized = MonoServiceLogic::clean_path_str(repo_path); - result - .ref_updates - .iter() - .find(|update| update.path == normalized) - .map(|update| update.tree_id) - } - - // helper to convert mega_tag model into TagInfo - fn tag_model_to_info(&self, tag: mega_tag::Model) -> TagInfo { - TagInfo { - name: tag.tag_name, - tag_id: tag.tag_id, - object_id: tag.object_id, - object_type: tag.object_type, - tagger: tag.tagger, - message: tag.message, - created_at: tag.created_at.and_utc().to_rfc3339(), - } - } - - pub async fn get_or_init_cla_sign_status( - &self, - username: &str, - ) -> Result<(bool, Option), MegaError> { - let model = self - .storage - .cla_storage() - .get_or_create_status(username) - .await?; - Ok((model.cla_signed, model.cla_signed_at)) - } - - pub async fn get_cla_content(&self) -> Result { - let key = ObjectKey { - namespace: ObjectNamespace::Log, - key: CLA_CONTENT_OBJECT_KEY.to_string(), - }; - - let stream = self - .storage - .git_service - .obj_storage - .inner - .get_stream(&key) - .await; - let (mut stream, _meta) = match stream { - Ok(result) => result, - Err(MegaError::ObjStorageNotFound(_)) => return Ok(String::new()), - Err(e) => return Err(e), - }; - - let mut data = Vec::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - data.extend_from_slice(&chunk); - } - - String::from_utf8(data).map_err(|e| { - MegaError::Other(format!( - "Invalid UTF-8 in CLA content from object storage: {e}" - )) - }) - } - - pub async fn update_cla_content(&self, content: &str) -> Result<(), MegaError> { - let key = ObjectKey { - namespace: ObjectNamespace::Log, - key: CLA_CONTENT_OBJECT_KEY.to_string(), - }; - - let bytes = Bytes::from(content.as_bytes().to_vec()); - let stream = stream::once(async move { Ok::(bytes) }); - let meta = ObjectMeta { - size: content.len() as i64, - content_type: Some("text/plain; charset=utf-8".to_string()), - ..Default::default() - }; - - self.storage - .git_service - .obj_storage - .inner - .put_stream(&key, Box::pin(stream), meta) - .await - } - - pub async fn change_cla_sign_status( - &self, - username: &str, - ) -> Result<(bool, Option), MegaError> { - let model = self.storage.cla_storage().sign(username).await?; - self.refresh_checks_for_open_cls_by_author(username).await?; - Ok((model.cla_signed, model.cla_signed_at)) - } - - async fn refresh_checks_for_open_cls_by_author(&self, username: &str) -> Result<(), MegaError> { - let open_cls = self - .storage - .cl_storage() - .get_open_cls() - .await? - .into_iter() - .filter(|cl| cl.username == username) - .collect::>(); - if open_cls.is_empty() { - return Ok(()); - } - - let check_reg = CheckerRegistry::new(self.storage.clone().into(), username.to_string()); - for cl in open_cls { - check_reg.run_checks(cl.into()).await?; - } - - Ok(()) - } - // This function is intended to be called before merging a CL to ensure it meets all required checks. - // It gathers the required checks for the CL's path, retrieves the CL's check results, and returns an error if any required checks have failed. - // Temporarily do not block the merge process when check fails. - - // async fn ensure_cl_mergeable(&self, cl: &mega_cl::Model) -> Result<(), MegaError> { - // let check_reg = CheckerRegistry::new(self.storage.clone().into(), cl.username.clone()); - // check_reg.run_checks(cl.clone().into()).await?; - - // let required_check_types = self - // .storage - // .cl_storage() - // .get_checks_config_by_path(&cl.path) - // .await? - // .into_iter() - // .filter(|cfg| cfg.required) - // .map(|cfg| cfg.check_type_code) - // .collect::>(); - - // let failed_checks = self - // .storage - // .cl_storage() - // .get_check_result(&cl.link) - // .await? - // .into_iter() - // .filter(|result| { - // result.status == "FAILED" - // && required_check_types - // .iter() - // .any(|required_type| required_type == &result.check_type_code) - // }) - // .map(|result| format!("{:?}", result.check_type_code)) - // .collect::>(); - - // if failed_checks.is_empty() { - // Ok(()) - // } else { - // Err(MegaError::Other(format!( - // "CL is unmergeable, failed checks: {}", - // failed_checks.join(", ") - // ))) - // } - // } - - async fn trigger_build_for_cl( - &self, - editor: &OneditCodeEdit, - cl: &mega_cl::Model, - username: &str, - ) -> Result<(), GitError> { - let config = self.storage.config(); - let orion_client = OrionBuildClient::new(config.build.clone()); - let git_cache = self.git_object_cache.clone(); - editor - .trigger_build_and_check( - self.storage.clone(), - git_cache, - Arc::new(orion_client), - cl, - username, - ) - .await?; - - Ok(()) - } - - /// Triggers a build for Buck upload completion - fn trigger_build_for_buck_upload(&self, response: &CompleteResponse, username: &str) { - let config = self.storage.config(); - let orion_client = Arc::new(OrionBuildClient::new(config.build.clone())); - if !orion_client.enable_build() { - return; - } - let storage = self.storage.clone(); - let git_cache = self.git_object_cache.clone(); - let mut context = TriggerContext::from_buck_upload( - response.repo_path.clone(), - response.from_hash.clone(), - response.commit_id.clone(), - response.cl_link.clone(), - Some(response.cl_id), - Some(username.to_string()), - ); - context.ref_name = Some("main".to_string()); - context.ref_type = Some("branch".to_string()); - tokio::spawn(async move { - if let Err(e) = - BuildTriggerService::build_by_context(storage, git_cache, orion_client, context) - .await - { - tracing::error!("Failed to create build trigger for buck upload: {}", e); - } - }); - } - - async fn create_annotated_tag_mono( - &self, - repo_path: Option, - name: String, - target: Option, - tagger_info: String, - message: Option, - full_ref: String, - ) -> Result { - let mono_storage = self.storage.mono_storage(); - - // build git_internal/mega tag models - let (tag_id_hex, object_id) = self.build_git_internal_tag_mono( - name.clone(), - target.clone(), - tagger_info.clone(), - message.clone(), - )?; - let tag_model = self.build_mega_tag_model( - tag_id_hex.clone(), - object_id.clone(), - name.clone(), - tagger_info.clone(), - message.clone(), - ); - - match mono_storage.insert_tag(tag_model).await { - Ok(saved_tag) => { - // try to write ref; if ref write fails, rollback DB insert - let path_str = repo_path.unwrap_or_else(|| "/".to_string()); - // Resolve tree hash from target commit so ref metadata is complete - let tree_hash = self.resolve_tree_hash_for_commit(&object_id).await?; - let refs = - mega_refs::Model::new(&path_str, full_ref.clone(), object_id, tree_hash, false); - - if let Err(e) = mono_storage.save_refs(refs, None).await { - // attempt to remove DB record - if let Err(del_e) = mono_storage.delete_tag_by_name(&name).await { - tracing::error!( - "Failed to rollback tag DB record after ref write failure: {}", - del_e - ); - } - tracing::error!("Failed to write ref after DB insert: {}", e); - return Err(GitError::CustomError( - "[code:500] Failed to write ref".to_string(), - )); - } - Ok(self.tag_model_to_info(saved_tag)) - } - Err(e) => { - tracing::error!("DB insert error when creating annotated tag: {}", e); - Err(GitError::CustomError( - "[code:500] DB insert error".to_string(), - )) - } - } - } - - async fn create_lightweight_tag_mono( - &self, - repo_path: Option, - name: String, - target: Option, - tagger_info: String, - full_ref: String, - ) -> Result { - let mono_storage = self.storage.mono_storage(); - - let path_str = repo_path.unwrap_or_else(|| "/".to_string()); - let object_id = target.clone().unwrap_or_default(); - if object_id.is_empty() { - return Err(GitError::CustomError( - "[code:400] Missing target commit for lightweight tag".to_string(), - )); - } - // Resolve tree hash from target commit - let tree_hash = self.resolve_tree_hash_for_commit(&object_id).await?; - - let refs = mega_refs::Model::new( - &path_str, - full_ref.clone(), - object_id.clone(), - tree_hash, - false, - ); - mono_storage.save_refs(refs, None).await.map_err(|e| { - tracing::error!("Failed to write lightweight tag ref: {}", e); - GitError::CustomError("[code:500] Failed to write lightweight tag ref".to_string()) - })?; - // Fetch saved ref to use its creation time - let saved_ref = mono_storage - .get_ref_by_name(&full_ref) - .await - .map_err(|e| GitError::CustomError(e.to_string()))? - .ok_or_else(|| GitError::CustomError("Ref not found after creation".to_string()))?; - - Ok(TagInfo { - name: name.clone(), - tag_id: object_id.clone(), - object_id: object_id.clone(), - object_type: "commit".to_string(), - tagger: tagger_info.clone(), - message: String::new(), - created_at: saved_ref.created_at.and_utc().to_rfc3339(), - }) - } - - /// Resolve the tree hash for a given commit id with proper error mapping/logging - async fn resolve_tree_hash_for_commit(&self, commit_id: &str) -> Result { - let mono_storage = self.storage.mono_storage(); - match mono_storage.get_commit_by_hash(commit_id).await { - Ok(Some(commit_model)) => Ok(commit_model.tree.clone()), - Ok(None) => { - tracing::error!( - "Target commit '{}' not found while resolving tree hash", - commit_id - ); - Err(GitError::CustomError(format!( - "[code:404] Target commit '{}' not found", - commit_id - ))) - } - Err(e) => { - tracing::error!( - "DB error fetching commit '{}' for tree hash resolution: {}", - commit_id, - e - ); - Err(GitError::CustomError("[code:500] DB error".to_string())) - } - } - } - async fn validate_target_commit_mono(&self, target: Option<&String>) -> Result<(), GitError> { - let mono_storage = self.storage.mono_storage(); - if let Some(ref t) = target { - match mono_storage.get_commit_by_hash(t).await { - Ok(commit_opt) => { - if commit_opt.is_none() { - return Err(GitError::CustomError(format!( - "[code:404] Target commit '{}' not found", - t - ))); - } - } - Err(e) => { - tracing::error!("DB error while fetching commit by hash: {}", e); - return Err(GitError::CustomError("[code:500] DB error".to_string())); - } - } - } - Ok(()) - } - - fn build_git_internal_tag_mono( - &self, - name: String, - target: Option, - tagger_info: String, - message: Option, - ) -> Result<(String, String), GitError> { - let tag_target = target - .as_ref() - .ok_or(GitError::InvalidCommitObject) - .and_then(|t| ObjectHash::from_str(t).map_err(|_| GitError::InvalidCommitObject))?; - let tagger_sig = git_internal::internal::object::signature::Signature::new( - git_internal::internal::object::signature::SignatureType::Tagger, - tagger_info.clone(), - String::new(), - ); - let git_internal_tag = git_internal::internal::object::tag::Tag::new( - tag_target, - git_internal::internal::object::types::ObjectType::Commit, - name.clone(), - tagger_sig, - message.clone().unwrap_or_default(), - ); - Ok(( - git_internal_tag.id.to_string(), - target.unwrap_or_else(|| "HEAD".to_string()), - )) - } - - fn build_mega_tag_model( - &self, - tag_id_hex: String, - object_id: String, - name: String, - tagger_info: String, - message: Option, - ) -> mega_tag::Model { - mega_tag::Model { - id: common::utils::generate_id(), - tag_id: tag_id_hex, - object_id, - object_type: "commit".to_string(), - tag_name: name, - tagger: tagger_info, - message: message.unwrap_or_default(), - pack_id: String::new(), - pack_offset: 0, - created_at: chrono::Utc::now().naive_utc(), - } - } - /// Merges a CL after checking for conflicts. - /// This is the public API that includes conflict checking. - pub async fn merge_cl(&self, username: &str, cl: mega_cl::Model) -> Result<(), GitError> { - let storage = self.storage.mono_storage(); - let refs = storage - .get_main_ref(&cl.path) - .await - .map_err(|e| GitError::CustomError(format!("Failed to get main ref: {}", e)))? - .ok_or_else(|| GitError::CustomError("Main ref not found".to_string()))?; - - if cl.from_hash != refs.ref_commit_hash { - return Err(GitError::CustomError("ref hash conflict".to_owned())); - } - - self.merge_cl_unchecked(username, cl).await - } - - /// Apply all CL changes onto the target_head in-memory and emit a single commit on the CL ref. - async fn apply_changes_as_single_commit( - &self, - cl: &mega_cl::Model, - changes: &[ClDiffFile], - target_head: &str, - ) -> Result { - let mono_storage = self.storage.mono_storage(); - - // Load base commit and its root tree - let base_commit = mono_storage - .get_commit_by_hash(target_head) - .await? - .ok_or_else(|| GitError::CustomError(format!("Commit not found: {target_head}")))?; - - let base_tree_model = mono_storage - .get_tree_by_hash(&base_commit.tree) - .await? - .ok_or_else(|| GitError::CustomError("Root tree not found".to_string()))?; - let mut root_tree = Tree::from_mega_model(base_tree_model); - - // Cache trees by path to reuse updated versions - let mut tree_cache: HashMap = HashMap::new(); - tree_cache.insert(PathBuf::from("/"), root_tree.clone()); - - // Collect all new trees we generate (dedup by hash) - let mut new_trees: HashMap = HashMap::new(); - - for diff in changes { - let operations: Vec<(PathBuf, Option)> = match diff { - ClDiffFile::New(path, new_hash) => vec![(path.clone(), Some(*new_hash))], - ClDiffFile::Modified(path, _old, new_hash) => { - vec![(path.clone(), Some(*new_hash))] - } - ClDiffFile::Deleted(path, _old) => vec![(path.clone(), None)], - ClDiffFile::Renamed(old_path, new_path, _old_hash, new_hash, _similarity) - | ClDiffFile::Moved(old_path, new_path, _old_hash, new_hash, _similarity) => { - vec![ - (old_path.clone(), None), - (new_path.clone(), Some(*new_hash)), - ] - } - }; - - for (file_path, op) in operations { - // Reject absolute or parent-traversing paths to avoid writing outside repo root. - if file_path.is_absolute() - || file_path - .components() - .any(|c| matches!(c, std::path::Component::ParentDir)) - { - return Err(GitError::CustomError(format!( - "Invalid path (traversal/absolute) in CL diff: {:?}", - file_path - ))); - } - - let file_name = file_path - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| GitError::CustomError("Invalid file name".to_string()))?; - // Normalize root parent to "/". - let parent_path = match file_path.parent() { - Some(p) if !p.as_os_str().is_empty() => p, - _ => Path::new("/"), - }; - - // Build chain of trees from root to parent, using updated cache when available - let components: Vec = parent_path - .components() - .filter_map(|c| match c { - std::path::Component::RootDir => None, - other => other.as_os_str().to_str().map(|s| s.to_string()), - }) - .collect(); - - let mut chain_paths: Vec = vec![PathBuf::from("/")]; - let mut chain_trees: Vec = vec![ - tree_cache - .get(&PathBuf::from("/")) - .cloned() - .ok_or_else(|| { - GitError::CustomError("Root tree cache missing".to_string()) - })?, - ]; - - let mut cursor = PathBuf::from("/"); - let mut missing_components: Option> = None; - for (idx, comp) in components.iter().enumerate() { - let parent_tree = chain_trees - .last() - .ok_or_else(|| GitError::CustomError("Empty tree chain".to_string()))?; - - let maybe_child = parent_tree.tree_items.iter().find(|it| it.name == *comp); - let child_tree = if let Some(child_item) = maybe_child { - if child_item.mode != TreeItemMode::Tree { - return Err(GitError::CustomError(format!( - "Type conflict: '{}' is not a directory", - comp - ))); - } - cursor = cursor.join(comp); - let child_hash = child_item.id; - if let Some(cached) = tree_cache.get(&cursor) { - cached.clone() - } else { - let model = mono_storage - .get_tree_by_hash(&child_hash.to_string()) - .await? - .ok_or_else(|| { - GitError::CustomError(format!( - "Tree not found for path '{}' with hash {}", - cursor.to_string_lossy(), - child_hash - )) - })?; - Tree::from_mega_model(model) - } - } else { - missing_components = Some(components[idx..].to_vec()); - break; - }; - - chain_paths.push(cursor.clone()); - chain_trees.push(child_tree); - } - - if let Some(missing) = missing_components { - let mut ctx = ApplyChangeContext { - components: &components, - chain_paths: &chain_paths, - chain_trees: &chain_trees, - tree_cache: &mut tree_cache, - new_trees: &mut new_trees, - }; - if let Some(updated_root) = - Self::apply_missing_path_update(&cl.link, missing, op, file_name, &mut ctx)? - { - root_tree = updated_root; - } - continue; - } - - let parent_dir_abs = cursor.clone(); - - // Update parent tree with the file change - let parent_tree = chain_trees - .pop() - .ok_or_else(|| GitError::CustomError("Parent tree missing".to_string()))?; - chain_paths.pop(); - - let mut items = parent_tree.tree_items.clone(); - match op { - Some(new_hash) => { - if let Some(idx) = items.iter().position(|it| it.name == file_name) { - items[idx].id = new_hash; - } else { - items.push(TreeItem::new( - TreeItemMode::Blob, - new_hash, - file_name.to_string(), - )); - } - } - None => { - items.retain(|it| it.name != file_name); - } - } - - let updated_tree = Tree::from_tree_items(items) - .map_err(|e| GitError::CustomError(e.to_string()))?; - // If parent tree id did not change (no-op), skip propagation for this diff. - if updated_tree.id == parent_tree.id { - // keep cache consistent even for no-ops - tree_cache.insert(parent_dir_abs.clone(), parent_tree.clone()); - debug!( - cl_link = %cl.link, - parent_dir = %parent_dir_abs.to_string_lossy(), - "apply_changes: no-op diff skipped" - ); - continue; - } - Self::record_tree( - parent_dir_abs, - &updated_tree, - &mut tree_cache, - &mut new_trees, - ); - - // Propagate updated hashes up to root - root_tree = Self::propagate_up( - &cl.link, - updated_tree, - &components, - &chain_paths, - &chain_trees, - &mut tree_cache, - &mut new_trees, - )?; - } - } - - let result = TreeUpdateResult { - updated_trees: new_trees.values().cloned().collect(), - ref_updates: vec![RefUpdate { - path: cl.path.clone(), - tree_id: root_tree.id, - }], - }; - - self.apply_update_result_cl_only( - &result, - "update-branch: rebase", - &cl.link, - Some(ObjectHash::from_str(target_head).map_err(|e| { - GitError::CustomError(format!( - "Invalid target_head ObjectHash '{}': {}", - target_head, e - )) - })?), - ) - .await - } - - fn apply_missing_path_update( - cl_link: &str, - missing: Vec, - op: Option, - file_name: &str, - ctx: &mut ApplyChangeContext<'_>, - ) -> Result, GitError> { - debug_assert!( - !missing.iter().any(|c| c == file_name), - "missing path components should not include file name" - ); - if op.is_none() { - debug!( - cl_link, - missing_path = %missing.join("/"), - "apply_changes: delete on missing path (no-op)" - ); - return Ok(None); - } - - let new_hash = op.ok_or_else(|| { - GitError::CustomError("Missing blob hash for new/modified file".to_string()) - })?; - - if missing.is_empty() { - // No missing directories: update directly under the last existing parent. - let parent_path = ctx.chain_paths.last().cloned().unwrap_or_else(PathBuf::new); - let parent_tree = ctx - .chain_trees - .last() - .cloned() - .ok_or_else(|| GitError::CustomError("Root tree missing".to_string()))?; - let updated_tree = Self::update_parent_tree( - cl_link, - &parent_tree, - file_name, - TreeItemMode::Blob, - new_hash, - None, - )?; - Self::record_tree(parent_path, &updated_tree, ctx.tree_cache, ctx.new_trees); - - return Ok(Some(Self::propagate_up( - cl_link, - updated_tree, - ctx.components, - ctx.chain_paths, - ctx.chain_trees, - ctx.tree_cache, - ctx.new_trees, - )?)); - } - - // Build missing subtree from leaf (parent dir) upward without empty trees. - let file_item = TreeItem::new(TreeItemMode::Blob, new_hash, file_name.to_string()); - let mut updated_tree = Tree::from_tree_items(vec![file_item]) - .map_err(|e| GitError::CustomError(e.to_string()))?; - - let mut missing_paths: Vec = Vec::new(); - let mut base = ctx.chain_paths.last().cloned().unwrap_or_else(PathBuf::new); - for comp in &missing { - base = base.join(comp); - missing_paths.push(base.clone()); - } - - if let Some(parent_path) = missing_paths.last() { - Self::record_tree( - parent_path.clone(), - &updated_tree, - ctx.tree_cache, - ctx.new_trees, - ); - } else { - Self::record_tree(PathBuf::new(), &updated_tree, ctx.tree_cache, ctx.new_trees); - } - - for (child_name, path) in missing - .iter() - .rev() - .skip(1) - .zip(missing_paths.iter().rev().skip(1)) - { - let wrapper = Tree::from_tree_items(vec![TreeItem::new( - TreeItemMode::Tree, - updated_tree.id, - child_name.clone(), - )]) - .map_err(|e| GitError::CustomError(e.to_string()))?; - updated_tree = wrapper; - Self::record_tree(path.clone(), &updated_tree, ctx.tree_cache, ctx.new_trees); - } - - // Attach the newly built subtree to the last existing parent. - let parent_tree = ctx - .chain_trees - .last() - .cloned() - .ok_or_else(|| GitError::CustomError("Root tree missing".to_string()))?; - let attach_name = missing - .first() - .ok_or_else(|| GitError::CustomError("Missing component chain empty".to_string()))?; - updated_tree = Self::update_parent_tree( - cl_link, - &parent_tree, - attach_name, - TreeItemMode::Tree, - updated_tree.id, - None, - )?; - let parent_path = ctx.chain_paths.last().cloned().unwrap_or_else(PathBuf::new); - Self::record_tree(parent_path, &updated_tree, ctx.tree_cache, ctx.new_trees); - - Ok(Some(Self::propagate_up( - cl_link, - updated_tree, - ctx.components, - ctx.chain_paths, - ctx.chain_trees, - ctx.tree_cache, - ctx.new_trees, - )?)) - } - - fn update_parent_tree( - cl_link: &str, - parent_tree: &Tree, - name: &str, - mode: TreeItemMode, - id: ObjectHash, - debug_parent_path: Option<&PathBuf>, - ) -> Result { - let mut parent_items = parent_tree.tree_items.clone(); - if let Some(pos) = parent_items.iter().position(|it| it.name == name) { - parent_items[pos].id = id; - } else { - parent_items.push(TreeItem::new(mode, id, name.to_string())); - parent_items.sort_by(|a, b| a.name.cmp(&b.name)); - if let Some(path) = debug_parent_path { - debug!( - cl_link, - parent_path = %path.to_string_lossy(), - created_entry = %name, - "apply_changes: inserted missing parent entry" - ); - } - } - - Tree::from_tree_items(parent_items).map_err(|e| GitError::CustomError(e.to_string())) - } - - fn record_tree( - path: PathBuf, - tree: &Tree, - tree_cache: &mut HashMap, - new_trees: &mut HashMap, - ) { - tree_cache.insert(path, tree.clone()); - new_trees.insert(tree.id, tree.clone()); - } - - fn propagate_up( - cl_link: &str, - mut updated_tree: Tree, - components: &[String], - chain_paths: &[PathBuf], - chain_trees: &[Tree], - tree_cache: &mut HashMap, - new_trees: &mut HashMap, - ) -> Result { - debug_assert!( - components.len() >= chain_trees.len().saturating_sub(1), - "components length must cover parent chain" - ); - - for parent_index in (0..chain_trees.len().saturating_sub(1)).rev() { - let comp = components - .get(parent_index) - .ok_or_else(|| GitError::CustomError("Tree path chain underflow".to_string()))?; - - let parent_tree = Self::update_parent_tree( - cl_link, - &chain_trees[parent_index], - comp, - TreeItemMode::Tree, - updated_tree.id, - chain_paths.get(parent_index), - )?; - - let parent_path_idx = chain_paths - .get(parent_index) - .cloned() - .ok_or_else(|| GitError::CustomError("Tree path chain underflow".to_string()))?; - Self::record_tree(parent_path_idx, &parent_tree, tree_cache, new_trees); - updated_tree = parent_tree; - } - - Ok(updated_tree) - } - - /// Merges a CL without checking for conflicts. - /// Caller is responsible for ensuring no conflicts exist before calling this method. - async fn merge_cl_unchecked(&self, username: &str, cl: mega_cl::Model) -> Result<(), GitError> { - let storage = self.storage.mono_storage(); - - let commit_model = storage - .get_commit_by_hash(&cl.to_hash) - .await - .map_err(|e| GitError::CustomError(format!("Failed to get commit: {}", e)))? - .ok_or_else(|| GitError::CustomError(format!("Commit not found: {}", cl.to_hash)))?; - let commit: Commit = Commit::from_mega_model(commit_model); - - let normalized_path = MonoServiceLogic::clean_path_str(&cl.path); - let (path, update_chain) = if normalized_path == "/" { - (PathBuf::from("/"), Vec::new()) - } else { - let path = PathBuf::from(&normalized_path); - let parent = path.parent().ok_or_else(|| { - GitError::CustomError(format!("Invalid CL path: {}", normalized_path)) - })?; - let update_chain = self.search_tree_for_update(parent).await?; - (path, update_chain) - }; - let result = MonoServiceLogic::build_result_by_chain(path, update_chain, commit.tree_id)?; - self.apply_update_result(&result, "cl merge generated commit", Some(cl.link.as_str())) - .await?; - - if normalized_path != "/" { - storage - .remove_none_cl_refs(&normalized_path) - .await - .map_err(|e| GitError::CustomError(format!("Failed to remove refs: {}", e)))?; - // TODO: self.clean_dangling_commits().await; - } - // add conversation - self.storage - .conversation_storage() - .add_conversation(&cl.link, username, None, ConvTypeEnum::Merged) - .await - .map_err(|e| GitError::CustomError(format!("Failed to add conversation: {}", e)))?; - // update cl status last - self.storage - .cl_storage() - .merge_cl(cl.clone()) - .await - .map_err(|e| GitError::CustomError(format!("Failed to update CL status: {}", e)))?; - - // Invalidate admin cache when .mega_cedar.json is modified. - if let Ok(files) = self.get_sorted_changed_file_list(&cl.link, None).await { - let admin_file_modified = files.iter().any(|file| { - let normalized = file.replace('\\', "/"); - normalized.ends_with(crate::api_service::admin_ops::ADMIN_FILE) - }); - if admin_file_modified { - self.invalidate_admin_cache().await; - } - } - - Ok(()) - } - - pub async fn apply_update_result( - &self, - result: &TreeUpdateResult, - commit_msg: &str, - cl_link: Option<&str>, - ) -> Result { - let storage = self.storage.mono_storage(); - let mut new_commit_id = String::new(); - let mut commits: Vec = Vec::new(); - - let paths: Vec<&str> = result.ref_updates.iter().map(|r| r.path.as_str()).collect(); - - let cl_refs_formatted = cl_link.map(|cl| format!("refs/cl/{}", cl)); - let cl_refs: Option> = cl_refs_formatted - .as_ref() - .map(|formatted| vec![formatted.as_str(), MEGA_BRANCH_NAME]); - - let refs = storage - .get_refs_for_paths_and_cls(&paths, cl_refs.as_deref()) - .await?; - - let mut updates: Vec = Vec::new(); - - MonoServiceLogic::process_ref_updates( - result, - &refs, - commit_msg, - &mut commits, - &mut updates, - &mut new_commit_id, - )?; - - if new_commit_id.is_empty() { - return Err(GitError::CustomError( - "no commit_id generated: no matching refs found for the update paths".into(), - )); - } - - storage - .batch_update_by_path_concurrent(updates) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - storage - .save_mega_commits(commits, None) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - let save_trees: Vec = result - .updated_trees - .clone() - .into_iter() - .map(|save_t| { - let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new()); - tree_model.commit_id.clone_from(&new_commit_id); - tree_model.into() - }) - .collect(); - - storage - .batch_save_model(save_trees) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - Ok(new_commit_id) - } - - /// Apply update result but only update the CL ref (never main). - /// Optionally override the parent commit for the first created commit (used by rebase). - async fn apply_update_result_cl_only( - &self, - result: &TreeUpdateResult, - commit_msg: &str, - cl_link: &str, - parent_override: Option, - ) -> Result { - let storage = self.storage.mono_storage(); - let mut new_commit_id = String::new(); - let mut commits: Vec = Vec::new(); - - let cl_ref_name = format!("refs/cl/{}", cl_link); - let cl_ref = storage - .get_ref_by_name(&cl_ref_name) - .await - .map_err(|e| GitError::CustomError(e.to_string()))? - .ok_or_else(|| GitError::CustomError("CL ref not found".to_string()))?; - - let mut updates: Vec = Vec::new(); - - MonoServiceLogic::process_ref_updates_cl_only( - result, - &cl_ref, - commit_msg, - parent_override, - &mut commits, - &mut updates, - &mut new_commit_id, - )?; - - if new_commit_id.is_empty() { - debug!( - cl_link, - ref_name = %cl_ref.ref_name, - ref_path = %cl_ref.path, - commit_msg, - "apply_update_result_cl_only: no commit_id generated" - ); - return Err(GitError::CustomError( - "no commit_id generated: no matching refs found for the update paths".into(), - )); - } - - storage - .batch_update_by_path_concurrent(updates) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - storage - .save_mega_commits(commits, None) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - let save_trees: Vec = result - .updated_trees - .clone() - .into_iter() - .map(|save_t| { - let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new()); - tree_model.commit_id.clone_from(&new_commit_id); - tree_model.into() - }) - .collect(); - - storage - .batch_save_model(save_trees) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - Ok(new_commit_id) - } - - /// Fetches the content difference for a merge request, paginated by page_id and page_size. - /// # Arguments - /// * `cl_link` - The link to the merge request. - /// * `page_id` - The page number to fetch. (id out of bounds will return empty) - /// * `page_size` - The number of items per page. - /// # Returns - /// a `Result` containing `ClDiff` on success or a `GitError` on failure. - /// Build paged CL diff items with optional relocation metadata for CL views. - async fn paged_content_diff_items( - &self, - cl_link: &str, - page: Pagination, - ) -> Result<(Vec, u64), GitError> { - let per_page = page.per_page as usize; - let page_id = page.page as usize; - - let stg = self.storage.cl_storage(); - let cl = - stg.get_cl(cl_link).await.unwrap().ok_or_else(|| { - GitError::CustomError(format!("Merge request not found: {cl_link}")) - })?; - let old_blobs = self - .get_commit_blobs(&cl.from_hash) - .await - .map_err(|e| GitError::CustomError(format!("Failed to get old commit blobs: {e}")))?; - let new_blobs = self - .get_commit_blobs(&cl.to_hash) - .await - .map_err(|e| GitError::CustomError(format!("Failed to get new commit blobs: {e}")))?; - - let sorted_changed_files = self - .cl_files_list(old_blobs, new_blobs) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - let start = (page_id.saturating_sub(1)) * per_page; - let end = (start + per_page).min(sorted_changed_files.len()); - - let page_slice: &[ClDiffFile] = if start < sorted_changed_files.len() { - let start_idx = start; - let end_idx = end; - &sorted_changed_files[start_idx..end_idx] - } else { - &[] - }; - - let non_relocated_items: Vec = page_slice - .iter() - .filter(|item| { - !matches!( - item, - ClDiffFile::Renamed(_, _, _, _, _) | ClDiffFile::Moved(_, _, _, _, _) - ) - }) - .cloned() - .collect(); - - let mut page_old_blobs = Vec::new(); - let mut page_new_blobs = Vec::new(); - collect_page_blobs( - &non_relocated_items, - &mut page_old_blobs, - &mut page_new_blobs, - ); - - let raw_diff_output = if non_relocated_items.is_empty() { - Vec::new() - } else { - self.get_diff_by_blobs(page_old_blobs, page_new_blobs) - .await? - }; - - let mut raw_diff_by_path: HashMap> = HashMap::new(); - for item in raw_diff_output { - raw_diff_by_path - .entry(item.path.clone()) - .or_default() - .push(item); - } - - let mut diff_output: Vec = Vec::with_capacity(page_slice.len()); - for item in page_slice { - match item { - ClDiffFile::Renamed(old_path, new_path, old_hash, new_hash, similarity) - | ClDiffFile::Moved(old_path, new_path, old_hash, new_hash, similarity) => { - diff_output.push(PagedClDiffItem { - item: self - .format_relocated_diff_item( - old_path, - new_path, - *old_hash, - *new_hash, - *similarity, - ) - .await?, - old_path: Some(old_path.to_string_lossy().replace('\\', "/")), - }); - } - _ => { - let key = item.path().to_string_lossy().replace('\\', "/"); - if let Some(items) = raw_diff_by_path.get_mut(&key) - && !items.is_empty() - { - diff_output.push(PagedClDiffItem { - item: items.remove(0), - old_path: None, - }); - } - } - } - } - - let total = sorted_changed_files.len().div_ceil(per_page); - - Ok((diff_output, total as u64)) - } - - /// Return the legacy paged diff shape without CL-specific metadata. - pub async fn paged_content_diff( - &self, - cl_link: &str, - page: Pagination, - ) -> Result<(Vec, u64), GitError> { - let (items, total) = self.paged_content_diff_items(cl_link, page).await?; - Ok((items.into_iter().map(|item| item.item).collect(), total)) - } - - /// Return paged diff items tailored for the CL files-changed API. - pub async fn paged_content_diff_for_cl( - &self, - cl_link: &str, - page: Pagination, - ) -> Result<(Vec, u64), GitError> { - let (items, total) = self.paged_content_diff_items(cl_link, page).await?; - Ok(( - items - .into_iter() - .map(|item| ClFilesChangedItemSchema::new(item.item, item.old_path)) - .collect(), - total, - )) - } - - async fn get_diff_by_blobs( - &self, - old_blobs: Vec<(PathBuf, ObjectHash)>, - new_blobs: Vec<(PathBuf, ObjectHash)>, - ) -> Result, GitError> { - let mut blob_cache: HashMap> = HashMap::new(); - - // Collect all unique hashes - let mut all_hashes = HashSet::new(); - for (_, hash) in &old_blobs { - all_hashes.insert(*hash); - } - for (_, hash) in &new_blobs { - all_hashes.insert(*hash); - } - - // Fetch all blobs with better error handling and logging - let mut failed_hashes = Vec::new(); - for hash in all_hashes { - match self.get_raw_blob_by_hash(&hash.to_string()).await { - Ok(data) => { - blob_cache.insert(hash, data); - } - Err(e) => { - tracing::error!("Failed to fetch blob {}: {}", hash, e); - failed_hashes.push(hash); - blob_cache.insert(hash, Vec::new()); - } - } - } - - if !failed_hashes.is_empty() { - tracing::warn!( - "Failed to fetch {} blob(s): {:?}", - failed_hashes.len(), - failed_hashes - ); - } - - // Enhanced content reader with better error handling - let read_content = |file: &PathBuf, hash: &ObjectHash| -> Vec { - match blob_cache.get(hash) { - Some(content) => content.clone(), - None => { - tracing::warn!("Missing blob content for file: {:?}, hash: {}", file, hash); - Vec::new() - } - } - }; - - // Use the unified diff function with configurable algorithm - let diff_output = GitDiff::diff(old_blobs, new_blobs, Vec::new(), read_content) - .into_iter() - .map(Self::normalize_diff_item) - .collect(); - - Ok(diff_output) - } - - async fn format_relocated_diff_item( - &self, - old_path: &Path, - new_path: &Path, - old_hash: ObjectHash, - new_hash: ObjectHash, - similarity: u8, - ) -> Result { - let mut patch = Self::format_relocated_patch_header(old_path, new_path, similarity); - - if old_hash != new_hash { - let raw_items = self - .get_diff_by_blobs( - vec![(old_path.to_path_buf(), old_hash)], - vec![(old_path.to_path_buf(), new_hash)], - ) - .await?; - if let Some(item) = raw_items.into_iter().next() { - patch.push_str(&Self::relocate_patch_body(&item.data, old_path, new_path)); - } - } - - Ok(DiffItem { - path: new_path.to_string_lossy().replace('\\', "/"), - data: patch, - }) - } - - fn format_relocated_patch_header(old_path: &Path, new_path: &Path, similarity: u8) -> String { - let old_path = old_path.to_string_lossy().replace('\\', "/"); - let new_path = new_path.to_string_lossy().replace('\\', "/"); - format!( - "diff --git a/{old_path} b/{new_path}\nsimilarity index {similarity}%\nrename from {old_path}\nrename to {new_path}\n" - ) - } - - fn normalize_diff_item(mut item: DiffItem) -> DiffItem { - item.path = item.path.replace('\\', "/"); - item.data = Self::normalize_patch_header_paths(&item.data); - item - } - - fn normalize_patch_header_paths(raw_patch: &str) -> String { - let sections = Self::split_patch_sections(raw_patch); - let header_lines = sections - .header_lines - .into_iter() - .map(Self::normalize_patch_header_line) - .collect(); - let divider_line = sections.divider_line.map(|line| { - if line.starts_with("Binary files ") { - line.replace('\\', "/") - } else { - line.to_string() - } - }); - - Self::join_patch_sections( - header_lines, - divider_line, - sections.payload_lines, - sections.has_trailing_newline, - ) - } - - fn relocate_patch_body(raw_patch: &str, old_path: &Path, new_path: &Path) -> String { - let old_path = old_path.to_string_lossy().replace('\\', "/"); - let new_path = new_path.to_string_lossy().replace('\\', "/"); - let sections = Self::split_patch_sections(raw_patch); - let header_lines = sections - .header_lines - .into_iter() - .filter(|line| !line.starts_with("diff --git ")) - .map(|line| { - if line.starts_with("--- a/") { - format!("--- a/{old_path}") - } else if line.starts_with("+++ b/") { - format!("+++ b/{new_path}") - } else { - line.to_string() - } - }) - .collect(); - let divider_line = sections.divider_line.map(|line| { - if line.starts_with("Binary files ") { - format!("Binary files a/{old_path} and b/{new_path} differ") - } else { - line.to_string() - } - }); - - Self::join_patch_sections( - header_lines, - divider_line, - sections.payload_lines, - sections.has_trailing_newline, - ) - } - - fn normalize_patch_header_line(line: &str) -> String { - if line.starts_with("diff --git ") - || line.starts_with("--- ") - || line.starts_with("+++ ") - || line.starts_with("rename from ") - || line.starts_with("rename to ") - { - line.replace('\\', "/") - } else { - line.to_string() - } - } - - fn split_patch_sections(raw_patch: &str) -> PatchSections<'_> { - let mut header_lines = Vec::new(); - let mut divider_line = None; - let mut payload_lines = Vec::new(); - let mut in_payload = false; - - for line in raw_patch.lines() { - if in_payload { - payload_lines.push(line); - continue; - } - - if line.starts_with("@@") - || line.starts_with("Binary files ") - || line.starts_with("GIT binary patch") - { - divider_line = Some(line); - in_payload = true; - continue; - } - - header_lines.push(line); - } - - PatchSections { - header_lines, - divider_line, - payload_lines, - has_trailing_newline: raw_patch.ends_with('\n'), - } - } - - fn join_patch_sections( - header_lines: Vec, - divider_line: Option, - payload_lines: Vec<&str>, - has_trailing_newline: bool, - ) -> String { - let mut lines = header_lines; - if let Some(line) = divider_line { - lines.push(line); - } - lines.extend(payload_lines.into_iter().map(String::from)); - - let rendered = lines.join("\n"); - if rendered.is_empty() { - rendered - } else if has_trailing_newline { - format!("{rendered}\n") - } else { - rendered - } - } - - pub async fn get_sorted_changed_file_list( - &self, - cl_link: &str, - path: Option<&str>, - ) -> Result, MegaError> { - let normalized_prefix = path.map(|prefix| prefix.replace('\\', "/")); - let cl = self - .storage - .cl_storage() - .get_cl(cl_link) - .await - .unwrap() - .ok_or_else(|| MegaError::Other("Error getting ".to_string()))?; - - let old_files = self.get_commit_blobs(&cl.from_hash.clone()).await?; - let new_files = self.get_commit_blobs(&cl.to_hash.clone()).await?; - - // calculate pages - let sorted_changed_files = self.cl_files_list(old_files, new_files).await?; - let file_paths: Vec = sorted_changed_files - .iter() - .map(|f| f.path().to_string_lossy().replace('\\', "/")) - .filter(|file_path| { - if let Some(prefix) = &normalized_prefix { - file_path.starts_with(prefix) - } else { - true - } - }) - .collect(); - - Ok(file_paths) - } - - /// Return Update Branch status for a CL: only checks whether main/trunk moved past the CL base. - pub async fn update_branch_status( - &self, - cl_link: &str, - ) -> Result { - let stg = self.storage.cl_storage(); - let cl = stg - .get_cl(cl_link) - .await? - .ok_or_else(|| MegaError::Other("CL Not Found".to_string()))?; - - let main_ref = self - .storage - .mono_storage() - .get_main_ref(&cl.path) - .await? - .ok_or_else(|| MegaError::Other("Main ref not found".to_string()))?; - let target_head = main_ref.ref_commit_hash; - - Ok(UpdateBranchStatusRes { - base_commit: cl.from_hash.clone(), - target_head: target_head.clone(), - outdated: cl.from_hash != target_head, - }) - } - - /// Update Branch (rebase-like) for Open CL: applies CL file changes onto latest target head - /// and updates CL's base/head commits. Returns new head commit id on success. - pub async fn update_branch(&self, username: &str, cl_link: &str) -> Result { - let stg = self.storage.cl_storage(); - let conv_stg = self.storage.conversation_storage(); - - let cl = stg - .get_cl(cl_link) - .await - .map_err(|e| GitError::CustomError(e.to_string()))? - .ok_or_else(|| GitError::CustomError("CL Not Found".to_string()))?; - - if cl.status != MergeStatusEnum::Open { - return Err(GitError::CustomError( - "Only Open CL can update branch".to_string(), - )); - } - - let main_ref = self - .storage - .mono_storage() - .get_main_ref(&cl.path) - .await - .map_err(|e| GitError::CustomError(e.to_string()))? - .ok_or_else(|| GitError::CustomError("Main ref not found".to_string()))?; - let target_head = main_ref.ref_commit_hash; - - if target_head == cl.from_hash { - return Ok("Already up-to-date".to_string()); - } - - // Detect file-level conflicts - let conflicts = self.detect_update_conflicts(&cl, &target_head).await?; - - if !conflicts.is_empty() { - // Record conflict info on the CL conversation for visibility. - let conflict_msg = format!( - "{} failed to update branch: conflicts on {}", - username, - conflicts.join(", ") - ); - if let Err(e) = conv_stg - .add_conversation(cl_link, username, Some(conflict_msg), ConvTypeEnum::Comment) - .await - { - tracing::warn!("Failed to add conflict comment to conversation: {}", e); - } - return Err(GitError::CustomError(format!( - "Update conflict on files: {}", - conflicts.join(", ") - ))); - } - - // Apply CL diffs onto latest target head - let old_blobs = self - .get_commit_blobs(&cl.from_hash) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - let new_blobs = self - .get_commit_blobs(&cl.to_hash) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - let cl_changed = self - .cl_files_list(old_blobs.clone(), new_blobs.clone()) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - if cl_changed.is_empty() { - // No-op rebase: just advance base hash and log. - stg.update_cl_hash(cl.clone(), &target_head, &cl.to_hash) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - conv_stg - .add_conversation( - cl_link, - username, - Some(format!( - "{} updated branch (no changes) to {}", - username, - &target_head[..6] - )), - ConvTypeEnum::Comment, - ) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - return Ok(cl.to_hash); - } - - // Apply all changes in-memory atop target_head and emit a single commit for the CL ref. - let new_head = self - .apply_changes_as_single_commit(&cl, &cl_changed, &target_head) - .await?; - - // Update cl hashes and log - stg.update_cl_hash(cl.clone(), &target_head, &new_head) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - conv_stg - .add_conversation( - cl_link, - username, - Some(format!( - "{} updated branch to {}", - username, - &target_head[..6] - )), - ConvTypeEnum::Comment, - ) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - Ok(new_head) - } - - /// Detect file-level update conflicts between the CL changes and target head. - /// A conflict is reported if any file path modified by the CL is also changed - /// between `from_hash` and `target_head`. - async fn detect_update_conflicts( - &self, - cl: &mega_cl::Model, - target_head: &str, - ) -> Result, GitError> { - let old_blobs = self - .get_commit_blobs(&cl.from_hash) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - let new_blobs = self - .get_commit_blobs(&cl.to_hash) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - // Keep conflict checks path-based so renames cover both old and new paths. - let cl_changed = edit_utils::cl_files_list(old_blobs.clone(), new_blobs.clone()) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - let target_blobs = self - .get_commit_blobs(target_head) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - let base_vs_target = edit_utils::cl_files_list(old_blobs.clone(), target_blobs.clone()) - .await - .map_err(|e| GitError::CustomError(e.to_string()))?; - - let cl_paths: std::collections::HashSet = cl_changed - .iter() - .map(|f| f.path().to_string_lossy().replace('\\', "/")) - .collect(); - let target_paths: std::collections::HashSet = base_vs_target - .iter() - .map(|f| f.path().to_string_lossy().replace('\\', "/")) - .collect(); - - Ok(cl_paths.intersection(&target_paths).cloned().collect()) - } - - pub async fn cl_files_list( - &self, - old_files: Vec<(PathBuf, ObjectHash)>, - new_files: Vec<(PathBuf, ObjectHash)>, - ) -> Result, MegaError> { - let base_diff = tree_diff::calculate_tree_diff_basic(old_files.clone(), new_files.clone())?; - let mut blob_cache: HashMap> = HashMap::new(); - let mut failed_hashes = Vec::new(); - let candidate_hashes: HashSet = base_diff - .iter() - .filter_map(|item| match item { - ClDiffFile::Deleted(_, hash) | ClDiffFile::New(_, hash) => Some(*hash), - _ => None, - }) - .collect(); - - if base_diff.len() > LARGE_CL_RENAME_DETECTION_THRESHOLD - || candidate_hashes.len() > LARGE_CL_RENAME_DETECTION_THRESHOLD - { - tracing::info!( - diff_files = base_diff.len(), - candidate_hashes = candidate_hashes.len(), - threshold = LARGE_CL_RENAME_DETECTION_THRESHOLD, - "Skipping rename detection for large CL diff and returning path-level results." - ); - return Ok(base_diff); - } - - for hash in candidate_hashes { - match self.get_raw_blob_by_hash(&hash.to_string()).await { - Ok(data) => { - blob_cache.insert(hash, data); - } - Err(err) => { - failed_hashes.push(hash); - tracing::warn!( - "rename detection skipped blob {} and will fall back to path-level diff: {}", - hash, - err - ); - } - } - } - - if !failed_hashes.is_empty() { - tracing::warn!( - "rename detection degraded for {} candidate blob(s)", - failed_hashes.len() - ); - } - - let rename_config = self.storage.config().monorepo.rename.clone(); - tree_diff::calculate_tree_diff_with_blobs(old_files, new_files, &rename_config, &blob_cache) - } - - pub async fn get_commit_blobs( - &self, - commit_hash: &str, - ) -> Result, MegaError> { - let mut res = vec![]; - let mono_storage = self.storage.mono_storage(); - let commit = mono_storage.get_commit_by_hash(commit_hash).await?; - if let Some(commit) = commit { - let tree = mono_storage.get_tree_by_hash(&commit.tree).await?; - if let Some(tree) = tree { - let tree: Tree = Tree::from_mega_model(tree); - res = self.traverse_tree(tree).await?; - } - } - Ok(res) - } - - pub async fn sync_third_party_repo( - &self, - owner: &str, - repo: &str, - mega_path: PathBuf, - ) -> Result { - // Additional Path Parameter for Mega - let url = format!("https://github.com/{owner}/{repo}.git"); - let remote_client = ThirdPartyClient::new(&url); - - let (ref_name, ref_hash) = remote_client.fetch_refs().await?; - - let res = remote_client - .fetch_packs(std::slice::from_ref(&ref_hash)) - .await?; - let pack_data = remote_client - .process_pack_stream(res) - .await - .map_err(|e| MegaError::Other(format!("{e}")))?; - - let repo_path_str = mega_path - .to_str() - .ok_or_else(|| MegaError::Other("Invalid UTF-8 in mega_path".to_string()))? - .to_string(); - - let mut protocol = - SmartSession::new(mega_path, ServiceType::ReceivePack, TransportProtocol::Http); - - // Populate commands so `git_receive_pack_stream` can update import refs. - // old_id: current ref hash if exists; otherwise ZERO_ID (create). - let storage = self.storage.git_db_storage(); - let old_id = match storage.find_git_repo_exact_match(&repo_path_str).await? { - Some(repo_model) => storage - .get_ref(repo_model.id) - .await? - .into_iter() - .find(|r| r.ref_name == ref_name) - .map(|r| r.ref_git_id) - .unwrap_or_else(|| ZERO_ID.to_string()), - None => ZERO_ID.to_string(), - }; - - let commands = vec![crate::protocol::import_refs::RefCommand::new( - old_id, - ref_hash.clone(), - ref_name.clone(), - )]; - let state = ProtocolApiState { - storage: self.storage.clone(), - git_object_cache: self.git_object_cache.clone(), - }; - let bytes = protocol - .git_receive_pack_stream( - &state, - commands, - Box::pin(tokio_stream::once(Ok(Bytes::from(pack_data)))), - ) - .await - .map_err(|e| MegaError::Other(format!("{e}")))?; - - Ok(bytes) - } - - async fn traverse_tree( - &self, - root_tree: Tree, - ) -> Result, MegaError> { - let mut result = vec![]; - let mut stack = vec![(PathBuf::new(), root_tree)]; - - while let Some((base_path, tree)) = stack.pop() { - for item in tree.tree_items { - let path = base_path.join(&item.name); - if item.is_tree() { - let child = self - .storage - .mono_storage() - .get_tree_by_hash(&item.id.to_string()) - .await? - .unwrap(); - stack.push((path.clone(), Tree::from_mega_model(child))); - } else { - result.push((path, item.id)); - } - } - } - Ok(result) - } - - // ========== Merge Queue Methods ========== - - /// Queue polling interval in seconds when no items are processed - const QUEUE_POLL_INTERVAL_SECS: u64 = 5; - - /// Error backoff interval in seconds after processing failure - const ERROR_BACKOFF_SECS: u64 = 30; - - /// Adds a CL to the merge queue and ensures the background processor is running. - /// - /// This method validates the CL status before adding to queue and automatically - /// starts the background processor if not already running. - /// - /// # Arguments - /// * `cl_link` - The unique identifier of the CL to add to queue - /// - /// # Returns - /// * `Ok(i64)` - The position in queue on success - /// * `Err(MegaError)` - If validation fails or database error occurs - pub async fn add_to_merge_queue(&self, cl_link: String) -> Result { - // Validate CL exists and is in Open status - let cl = self.storage.cl_storage().get_cl(&cl_link).await?; - let model = cl.ok_or(MegaError::Other("CL not found".to_string()))?; - - if model.status != MergeStatusEnum::Open { - return Err(MegaError::Other(format!( - "CL is not in Open status, current status: {:?}", - model.status - ))); - } - - // Add to queue via jupiter layer service - let position = self - .storage - .merge_queue_service - .add_to_queue(cl_link) - .await?; - - // Ensure the background processor is running - self.ensure_merge_processor_running(); - - Ok(position) - } - - /// Retries a failed merge queue item and ensures the processor is running. - /// - /// # Arguments - /// * `cl_link` - The unique identifier of the CL to retry - /// - /// # Returns - /// * `Ok(true)` - If retry was successful - /// * `Ok(false)` - If item not found or cannot be retried - /// * `Err(MegaError)` - If database error occurs - pub async fn retry_merge_queue_item(&self, cl_link: &str) -> Result { - let result = self - .storage - .merge_queue_service - .retry_queue_item(cl_link) - .await?; - - if result { - // Ensure the background processor is running - self.ensure_merge_processor_running(); - } - - Ok(result) - } - - // ========== Buck Upload API Methods ========== - - /// Create buck upload session. - /// - /// # Arguments - /// * `username` - User creating the session - /// * `path` - Repository path (may be with or without leading `/`; normalized to match mega_refs format) - /// - /// # Returns - /// Returns `SessionResponse` on success - pub async fn create_buck_session( - &self, - username: &str, - path: &str, - ) -> Result { - let normalized_path = MonoServiceLogic::normalize_repo_path(path)?; - let refs = self - .storage - .mono_storage() - .get_main_ref(&normalized_path) - .await? - .ok_or_else(|| MegaError::NotFound(format!("Path not found: {}", normalized_path)))?; - let base_branch = refs - .ref_name - .strip_prefix("refs/heads/") - .unwrap_or(refs.ref_name.as_str()) - .to_string(); - // Use canonical path from mega_refs as the single source of truth for repository path - let canonical_path = refs.path.clone(); - let response = self - .storage - .buck_service - .create_session( - username, - &canonical_path, - &base_branch, - refs.ref_commit_hash, - ) - .await?; - - Ok(response) - } - - /// Process buck upload manifest. - /// - /// # Arguments - /// * `username` - User processing the manifest - /// * `cl_link` - CL link - /// * `payload` - Manifest payload - /// - /// # Returns - /// Returns `ManifestResponse` on success - pub async fn process_buck_manifest( - &self, - username: &str, - cl_link: &str, - payload: ManifestPayload, - ) -> Result { - let session = self - .storage - .buck_storage() - .get_session(cl_link) - .await? - .ok_or_else(|| MegaError::Buck(BuckError::SessionNotFound(cl_link.to_string())))?; - - if session.user_id != username { - return Err(MegaError::Buck(BuckError::Forbidden( - "Session belongs to another user".to_string(), - ))); - } - - let manifest_paths: Vec = payload - .files - .iter() - .map(|f| PathBuf::from(&f.path)) - .collect(); - - // Get content hashes (raw SHA-1) and blob IDs - let (existing_file_hashes, existing_blob_ids_map) = - crate::api_service::blob_ops::get_files_content_hashes_with_blob_ids( - self, - &manifest_paths, - session.from_hash.as_deref(), - ) - .await - .map_err(MegaError::Git)?; - - // Convert ObjectHash to String for storage - let existing_blob_ids: HashMap = existing_blob_ids_map - .into_iter() - .map(|(path, blob_hash)| (path, blob_hash.to_string())) - .collect(); - - // Convert payload to service layer type - let service_payload = jupiter::service::buck_service::ManifestPayload { - files: payload - .files - .iter() - .map(|f| jupiter::service::buck_service::ManifestFile { - path: f.path.clone(), - size: f.size, - hash: f.hash.clone(), - }) - .collect(), - commit_message: payload.commit_message.clone(), - }; - - let svc_resp = self - .storage - .buck_service - .process_manifest( - username, - cl_link, - service_payload, - existing_file_hashes, - existing_blob_ids, - ) - .await?; - - // Convert back to API layer response - let api_resp = ManifestResponse { - total_files: svc_resp.total_files, - total_size: svc_resp.total_size, - files_to_upload: svc_resp - .files_to_upload - .into_iter() - .map(|f| ApiFileToUpload { - path: f.path, - reason: f.reason, - }) - .collect(), - files_unchanged: svc_resp.files_unchanged, - upload_size: svc_resp.upload_size, - }; - - Ok(api_resp) - } - - /// Complete buck upload. - /// - /// Commit message is read from session.commit_message which is set during Manifest phase. - /// The payload is intentionally unused (empty struct). - /// - /// # Arguments - /// * `username` - User completing the upload - /// * `cl_link` - CL link - /// * `_payload` - Empty payload (unused). Commit message is read from session.commit_message - /// which is set during Manifest phase. - /// - /// # Returns - /// Returns `CompleteResponse` on success - pub async fn complete_buck_upload( - &self, - username: &str, - cl_link: &str, - _payload: CompletePayload, - ) -> Result { - let session = self - .storage - .buck_storage() - .get_session(cl_link) - .await? - .ok_or_else(|| MegaError::Buck(BuckError::SessionNotFound(cl_link.to_string())))?; - - if session.user_id != username { - return Err(MegaError::Buck(BuckError::Forbidden( - "Session belongs to another user".to_string(), - ))); - } - - if ![session_status::MANIFEST_UPLOADED, session_status::UPLOADING] - .contains(&session.status.as_str()) - { - return Err(MegaError::Buck(BuckError::InvalidSessionStatus { - expected: format!( - "{} or {}", - session_status::MANIFEST_UPLOADED, - session_status::UPLOADING - ), - actual: session.status.clone(), - })); - } - - let pending = self - .storage - .buck_storage() - .count_pending_files(cl_link) - .await?; - if pending > 0 { - return Err(MegaError::Buck(BuckError::FilesNotFullyUploaded { - missing_count: pending as u32, - })); - } - - let all_files = self.storage.buck_storage().get_all_files(cl_link).await?; - for file in &all_files { - if file.blob_id.is_none() { - return Err(MegaError::Buck(BuckError::ValidationError(format!( - "Missing blob_id for file: {} (status: {})", - file.file_path, file.upload_status - )))); - } - } - - // Build commit - let file_changes: Vec = all_files - .iter() - .filter(|f| f.upload_status == upload_status::UPLOADED) - .map(|f| { - let blob_id = f.blob_id.as_ref().unwrap(); - let normalized_blob_id = - format!("sha1:{}", blob_id.strip_prefix("sha1:").unwrap_or(blob_id)); - FileChange::new( - f.file_path.clone(), - normalized_blob_id, - f.file_mode - .clone() - .unwrap_or_else(|| DEFAULT_MODE.to_string()), - ) - }) - .collect(); - - // Use commit_message from session - let commit_message = session - .commit_message - .clone() - .unwrap_or_else(|| "Upload via buck push".to_string()); - - let commit_result = if file_changes.is_empty() { - None - } else { - let builder = BuckCommitBuilder::new(self.storage.mono_storage()); - let result = builder - .build_commit( - session.from_hash.as_deref().unwrap_or_default(), - &file_changes, - &commit_message, - ) - .await?; - Some(result) - }; - - // Convert to artifacts acceptable by BuckService - let artifacts = commit_result.map(|res| { - let commit_model: callisto::mega_commit::ActiveModel = res - .commit - .clone() - .into_mega_model(git_internal::internal::metadata::EntryMeta::default()) - .into(); - let new_tree_models: Vec = - res.new_tree_models.into_iter().map(|m| m.into()).collect(); - CommitArtifacts { - commit_id: res.commit_id, - tree_hash: res.tree_hash, - new_tree_models, - commit_model, - } - }); - - let svc_resp: SvcCompleteResponse = self - .storage - .buck_service - .complete_upload(username, cl_link, SvcCompletePayload {}, artifacts) - .await?; - - // Calculate uploaded files count - let uploaded_files_count = file_changes.len() as u32; - - let response = CompleteResponse { - cl_id: session.id, - cl_link: session.session_id.clone(), - commit_id: svc_resp.commit_id, - files_count: uploaded_files_count, - created_at: session.created_at.to_string(), - repo_path: session.repo_path.clone(), - from_hash: session.from_hash.clone().unwrap_or_default(), - }; - - self.trigger_build_for_buck_upload(&response, username); - - Ok(response) - } - - /// Ensures the background merge processor is running. - /// - /// Uses atomic flag to guarantee only one processor task runs at a time. - /// The processor automatically stops when no active items remain in queue. - fn ensure_merge_processor_running(&self) { - // Get the processor running flag from merge queue service - if self.storage.merge_queue_service.try_start_processor() { - let service = self.clone(); - tokio::spawn(async move { - tracing::info!("Merge queue processor started (from MonoApiService)"); - service.run_merge_processor_loop().await; - }); - } - } - - /// Main loop for the background merge processor. - /// - /// Continuously processes queue items until no active items remain. - async fn run_merge_processor_loop(&self) { - loop { - match self.process_next_queue_item().await { - Ok(processed) => { - if !processed { - // Check if there are active items - if let Ok(stats) = self.storage.merge_queue_service.get_queue_stats().await - { - let has_active = stats.waiting_count > 0 - || stats.testing_count > 0 - || stats.merging_count > 0; - - if !has_active { - // No active items, stop processor - self.storage.merge_queue_service.stop_processor(); - tracing::info!("Merge queue processor stopped (no active items)"); - break; - } - } - tokio::time::sleep(Duration::from_secs(Self::QUEUE_POLL_INTERVAL_SECS)) - .await; - } - } - Err(e) => { - tracing::error!("Merge queue processor error: {}", e); - tokio::time::sleep(Duration::from_secs(Self::ERROR_BACKOFF_SECS)).await; - } - } - } - } - - /// Processes the next item in the merge queue. - /// - /// # Returns - /// * `Ok(true)` - An item was processed (success or failure) - /// * `Ok(false)` - No items to process - /// * `Err(MegaError)` - System error occurred - async fn process_next_queue_item(&self) -> Result { - let queue_service = &self.storage.merge_queue_service; - - // Get next waiting item from queue - let next_item = queue_service.get_next_waiting_item().await?; - - if let Some(item) = next_item { - let cl_link = item.cl_link.clone(); - - // Update status to Testing - let updated = queue_service - .update_item_status(&cl_link, QueueStatusEnum::Testing) - .await?; - - // Item was cancelled before we could start processing - if !updated { - return Ok(false); - } - - // Execute the merge workflow - match self.execute_merge_workflow(&cl_link).await { - Ok(()) => { - // Success - status already updated to Merged in workflow - Ok(true) - } - Err((failure_type, message)) => { - if matches!(failure_type, QueueFailureTypeEnum::Conflict) { - // Conflict - move to tail of queue for retry - if let Err(e) = queue_service.move_item_to_tail(&cl_link).await { - tracing::warn!( - "Failed to move conflicting item {} to tail: {}", - cl_link, - e - ); - } - Ok(false) - } else { - // Other failure - mark as failed - if let Err(e) = queue_service - .update_item_status_with_error(&cl_link, failure_type, message) - .await - { - tracing::error!( - "Failed to update item {} status to failed: {}", - cl_link, - e - ); - } - Ok(true) - } - } - } - } else { - Ok(false) - } - } - - /// Executes the complete merge workflow for a CL. - /// - /// Workflow steps: - /// 1. Validate CL exists and is in valid status - /// 2. Run tests (TODO: Buck2 integration) - /// 3. Check for conflicts - /// 4. Execute merge - /// 5. Update statuses - async fn execute_merge_workflow( - &self, - cl_link: &str, - ) -> Result<(), (QueueFailureTypeEnum, String)> { - let queue_service = &self.storage.merge_queue_service; - - // Step 1: Validate CL still exists and is not closed - let cl = self - .storage - .cl_storage() - .get_cl(cl_link) - .await - .map_err(|e| { - ( - QueueFailureTypeEnum::SystemError, - format!("Failed to fetch CL: {}", e), - ) - })?; - - let cl_model = match cl { - Some(model) => { - if model.status == MergeStatusEnum::Closed { - return Err(( - QueueFailureTypeEnum::SystemError, - "CL has been closed, cannot merge".to_string(), - )); - } - if model.status == MergeStatusEnum::Draft { - return Err(( - QueueFailureTypeEnum::SystemError, - "CL is in draft status, cannot merge".to_string(), - )); - } - model - } - None => { - return Err(( - QueueFailureTypeEnum::SystemError, - "CL no longer exists, cannot merge".to_string(), - )); - } - }; - - // Step 2: Run tests (TODO: Buck2 integration) - // self.run_tests(&cl_model).await?; - - // Step 3: Check for conflicts - self.check_merge_conflicts(&cl_model).await?; - - // Step 4: Update status to Merging - let updated = queue_service - .update_item_status(cl_link, QueueStatusEnum::Merging) - .await - .map_err(|e| { - ( - QueueFailureTypeEnum::SystemError, - format!("Failed to update status to merging: {}", e), - ) - })?; - - if !updated { - return Err(( - QueueFailureTypeEnum::SystemError, - "Item was cancelled".to_string(), - )); - } - - // Step 5: Execute merge (conflict already checked in step 3) - self.merge_cl_unchecked("system", cl_model.clone()) - .await - .map_err(|e| { - ( - QueueFailureTypeEnum::MergeFailure, - format!("Merge failed: {}", e), - ) - })?; - - // Step 6: Update queue status to Merged - queue_service - .update_item_status(cl_link, QueueStatusEnum::Merged) - .await - .map_err(|e| { - ( - QueueFailureTypeEnum::SystemError, - format!("Failed to update status to merged: {}", e), - ) - })?; - - Ok(()) - } - - /// Checks for merge conflicts by comparing CL base hash with current main ref. - /// - /// A conflict occurs when the CL's from_hash differs from the current - /// main branch ref, indicating the base has changed since CL creation. - async fn check_merge_conflicts( - &self, - cl: &mega_cl::Model, - ) -> Result<(), (QueueFailureTypeEnum, String)> { - let storage = self.storage.mono_storage(); - - let refs = storage - .get_main_ref(&cl.path) - .await - .map_err(|e| { - ( - QueueFailureTypeEnum::SystemError, - format!("Failed to get main ref: {}", e), - ) - })? - .ok_or(( - QueueFailureTypeEnum::SystemError, - "Main ref not found".to_string(), - ))?; - - if cl.from_hash != refs.ref_commit_hash { - return Err(( - QueueFailureTypeEnum::Conflict, - format!( - "Conflict detected: CL base hash {} differs from current main ref {}", - cl.from_hash, refs.ref_commit_hash - ), - )); - } - - Ok(()) - } -} - -fn collect_page_blobs( - items: &[ClDiffFile], - old_out: &mut Vec<(PathBuf, ObjectHash)>, - new_out: &mut Vec<(PathBuf, ObjectHash)>, -) { - old_out.reserve(items.len()); - new_out.reserve(items.len()); - - for item in items { - match item { - ClDiffFile::New(p, h_new) => { - new_out.push((p.clone(), *h_new)); - } - ClDiffFile::Deleted(p, h_old) => { - old_out.push((p.clone(), *h_old)); - } - ClDiffFile::Modified(p, h_old, h_new) => { - old_out.push((p.clone(), *h_old)); - new_out.push((p.clone(), *h_new)); - } - ClDiffFile::Renamed(_, _, _, _, _) | ClDiffFile::Moved(_, _, _, _, _) => { - // Relocated items are filtered out before this helper is called. - debug_assert!(false, "collect_page_blobs only accepts non-relocated items"); - } - } - } -} -#[cfg(test)] -mod test { - use std::{path::PathBuf, str::FromStr, sync::Arc}; - - use git_internal::{ - hash::ObjectHash, - internal::object::{ - signature::{Signature, SignatureType}, - tree::{Tree, TreeItem, TreeItemMode}, - }, - }; - - use super::*; - use crate::model::change_list::ClDiffFile; - - #[test] - fn test_clean_path_str_edges() { - assert_eq!(MonoServiceLogic::clean_path_str(""), "/"); - assert_eq!(MonoServiceLogic::clean_path_str("/"), "/"); - assert_eq!(MonoServiceLogic::clean_path_str("abc/"), "abc"); - assert_eq!(MonoServiceLogic::clean_path_str("abc///"), "abc"); - } - - #[test] - fn test_normalize_repo_path() { - use common::errors::{BuckError, MegaError}; - - // Normalization: add leading slash, strip trailing - assert_eq!( - MonoServiceLogic::normalize_repo_path("project").unwrap(), - "/project" - ); - assert_eq!( - MonoServiceLogic::normalize_repo_path("/project").unwrap(), - "/project" - ); - assert_eq!( - MonoServiceLogic::normalize_repo_path("project/").unwrap(), - "/project" - ); - assert_eq!( - MonoServiceLogic::normalize_repo_path("/project/").unwrap(), - "/project" - ); - assert_eq!( - MonoServiceLogic::normalize_repo_path(" /project ").unwrap(), - "/project" - ); - assert_eq!(MonoServiceLogic::normalize_repo_path("/").unwrap(), "/"); - - // Empty / whitespace-only -> ValidationError - assert!(MonoServiceLogic::normalize_repo_path("").is_err()); - assert!(MonoServiceLogic::normalize_repo_path(" ").is_err()); - assert!(matches!( - MonoServiceLogic::normalize_repo_path(""), - Err(MegaError::Buck(BuckError::ValidationError(_))) - )); - - // Path traversal and invalid chars -> ValidationError - assert!(MonoServiceLogic::normalize_repo_path("project/../foo").is_err()); - assert!(MonoServiceLogic::normalize_repo_path("project\\foo").is_err()); - - // Middle slashes and "." segments are collapsed - assert_eq!( - MonoServiceLogic::normalize_repo_path("//project//foo//").unwrap(), - "/project/foo" - ); - assert_eq!( - MonoServiceLogic::normalize_repo_path("project/./foo").unwrap(), - "/project/foo" - ); - assert_eq!( - MonoServiceLogic::normalize_repo_path("/project/./foo").unwrap(), - "/project/foo" - ); - - // Dot-only paths are rejected (do not silently resolve to root) - assert!(matches!( - MonoServiceLogic::normalize_repo_path("."), - Err(MegaError::Buck(BuckError::ValidationError(_))) - )); - assert!(matches!( - MonoServiceLogic::normalize_repo_path("./"), - Err(MegaError::Buck(BuckError::ValidationError(_))) - )); - assert!(matches!( - MonoServiceLogic::normalize_repo_path("./."), - Err(MegaError::Buck(BuckError::ValidationError(_))) - )); - - // Leading colon is rejected - assert!(matches!( - MonoServiceLogic::normalize_repo_path(":/test"), - Err(MegaError::Buck(BuckError::ValidationError(_))) - )); - assert!(matches!( - MonoServiceLogic::normalize_repo_path(":"), - Err(MegaError::Buck(BuckError::ValidationError(_))) - )); - - // Windows drive letters are rejected - assert!(matches!( - MonoServiceLogic::normalize_repo_path("C:"), - Err(MegaError::Buck(BuckError::ValidationError(_))) - )); - assert!(matches!( - MonoServiceLogic::normalize_repo_path("D:/project"), - Err(MegaError::Buck(BuckError::ValidationError(_))) - )); - } - - #[test] - fn test_repo_root_candidates_walk_from_leaf_to_root() { - assert_eq!( - MonoServiceLogic::repo_root_candidates(Path::new("/project/buck2_test/src")), - vec![ - "/project/buck2_test/src".to_string(), - "/project/buck2_test".to_string(), - "/project".to_string(), - "/".to_string(), - ] - ); - } - - #[test] - fn test_repo_root_candidates_normalize_relative_paths() { - assert_eq!( - MonoServiceLogic::repo_root_candidates(Path::new("project/buck2_test/src")), - vec![ - "/project/buck2_test/src".to_string(), - "/project/buck2_test".to_string(), - "/project".to_string(), - "/".to_string(), - ] - ); - } - - #[test] - fn test_subtree_ref_path_keeps_parent_directory_for_file_edits() { - assert_eq!( - MonoServiceLogic::subtree_ref_path(Path::new("/project/buck2_test/src")).unwrap(), - "/project/buck2_test/src".to_string() - ); - } - - #[test] - fn test_subtree_ref_path_normalizes_relative_create_paths() { - assert_eq!( - MonoServiceLogic::subtree_ref_path(Path::new("project/buck2_test/src")).unwrap(), - "/project/buck2_test/src".to_string() - ); - } - - #[test] - fn test_save_file_edit_uses_build_repo_root_tree_from_ref_updates() { - let build_root_tree = - ObjectHash::from_str("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap(); - let nested_tree = ObjectHash::from_str("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb").unwrap(); - let result = TreeUpdateResult { - updated_trees: vec![], - ref_updates: vec![ - RefUpdate { - path: "/project/buck2_test/src".to_string(), - tree_id: nested_tree, - }, - RefUpdate { - path: "/project/buck2_test".to_string(), - tree_id: build_root_tree, - }, - ], - }; - - let selected = MonoApiService::ref_update_tree_id_for_path(&result, "/project/buck2_test"); - assert_eq!(selected, Some(build_root_tree)); - } - - #[test] - fn test_create_monorepo_entry_uses_normalized_build_repo_root_tree() { - let build_root_tree = - ObjectHash::from_str("cccccccccccccccccccccccccccccccccccccccc").unwrap(); - let result = TreeUpdateResult { - updated_trees: vec![], - ref_updates: vec![RefUpdate { - path: "/project/buck2_test".to_string(), - tree_id: build_root_tree, - }], - }; - - let selected = MonoApiService::ref_update_tree_id_for_path(&result, "/project/buck2_test/"); - assert_eq!(selected, Some(build_root_tree)); - } - - #[test] - fn test_update_tree_hash() { - let item = TreeItem::new( - TreeItemMode::Blob, - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - "path".to_string(), - ); - - let tree = Tree::from_tree_items(vec![item]).expect("tree should build"); - let tree = Arc::new(tree); - - let new_hash = ObjectHash::from_str("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap(); - - let new_tree = MonoServiceLogic::update_tree_hash(tree, "path", new_hash) - .expect("update_tree_hash should succeed"); - - assert_eq!(new_tree.tree_items.len(), 1); - assert_eq!(new_tree.tree_items[0].id, new_hash); - } - - #[test] - fn test_build_result_by_chain_logic() { - let item = TreeItem::new( - TreeItemMode::Blob, - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - "path".to_string(), - ); - - let tree = Tree::from_tree_items(vec![item]).expect("tree should build"); - let tree_id = tree.id; - - let update_chain = vec![Arc::new(tree)]; - let path = PathBuf::from("/test/path"); - - let result = MonoServiceLogic::build_result_by_chain(path, update_chain, tree_id) - .expect("build_result_by_chain should succeed"); - - assert_eq!(result.updated_trees.len(), 1); - assert_eq!(result.ref_updates.len(), 2); - - let paths: Vec<&str> = result.ref_updates.iter().map(|r| r.path.as_str()).collect(); - assert!(paths.contains(&"/test/path")); - assert!(paths.contains(&"/test")); - } - - #[test] - fn test_build_result_by_chain_normalizes_relative_paths_for_ref_updates() { - let old_hash = ObjectHash::from_str("1111111111111111111111111111111111111111").unwrap(); - let updated_child_hash = - ObjectHash::from_str("2222222222222222222222222222222222222222").unwrap(); - let item = TreeItem::new(TreeItemMode::Tree, old_hash, "src".to_string()); - let tree = Tree::from_tree_items(vec![item]).expect("tree should build"); - let update_chain = vec![Arc::new(tree)]; - - let result = MonoServiceLogic::build_result_by_chain( - PathBuf::from("project/buck2_test/src"), - update_chain, - updated_child_hash, - ) - .expect("build_result_by_chain should succeed"); - - let paths: Vec<&str> = result.ref_updates.iter().map(|r| r.path.as_str()).collect(); - assert!(paths.contains(&"/project/buck2_test/src")); - assert!(paths.contains(&"/project/buck2_test")); - - let selected = MonoApiService::ref_update_tree_id_for_path(&result, "/project/buck2_test"); - assert!(selected.is_some()); - } - - #[tokio::test] - async fn test_process_ref_updates_logic() { - let ref_update = RefUpdate { - path: "/test".to_string(), - tree_id: ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - }; - - let tree_update_result = TreeUpdateResult { - updated_trees: vec![], - ref_updates: vec![RefUpdate { - path: ref_update.path.clone(), - tree_id: ref_update.tree_id, - }], - }; - - let refs = vec![mega_refs::Model { - id: 1, - path: "/test".to_string(), - ref_name: "refs/heads/main".to_string(), - ref_commit_hash: "0987654321098765432109876543210987654321".to_string(), - ref_tree_hash: "1111111111111111111111111111111111111111".to_string(), - created_at: chrono::Utc::now().naive_utc(), - updated_at: chrono::Utc::now().naive_utc(), - is_cl: false, - }]; - - let mut commits: Vec = Vec::new(); - let mut updates: Vec = Vec::new(); - let mut new_commit_id = String::new(); - - let result = MonoServiceLogic::process_ref_updates( - &tree_update_result, - &refs, - "test commit message", - &mut commits, - &mut updates, - &mut new_commit_id, - ); - - assert!(result.is_ok()); - assert_eq!(commits.len(), 1); - assert_eq!(updates.len(), 1); - assert!(!new_commit_id.is_empty()); - - let created_commit = &commits[0]; - - assert_eq!( - created_commit.tree_id, - tree_update_result.ref_updates[0].tree_id - ); - let expected_parent = ObjectHash::from_str(&refs[0].ref_commit_hash).unwrap(); - assert_eq!(created_commit.parent_commit_ids, vec![expected_parent]); - - assert_eq!(updates[0].ref_name, refs[0].ref_name); - assert_eq!(updates[0].commit_id, new_commit_id); - } - - #[tokio::test] - async fn test_root_merge_flow_updates_cl_ref_and_main() { - let merged_tree_id = - ObjectHash::from_str("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap(); - let root_result = - MonoServiceLogic::build_result_by_chain(PathBuf::from("/"), vec![], merged_tree_id) - .expect("root path should build a valid update result"); - - assert_eq!(root_result.ref_updates.len(), 1); - assert_eq!(root_result.ref_updates[0].path, "/"); - - let refs = vec![ - mega_refs::Model { - id: 1, - path: "/".to_string(), - ref_name: "refs/cl/abcd1234".to_string(), - ref_commit_hash: "1111111111111111111111111111111111111111".to_string(), - ref_tree_hash: "2222222222222222222222222222222222222222".to_string(), - created_at: chrono::Utc::now().naive_utc(), - updated_at: chrono::Utc::now().naive_utc(), - is_cl: true, - }, - mega_refs::Model { - id: 2, - path: "/".to_string(), - ref_name: MEGA_BRANCH_NAME.to_string(), - ref_commit_hash: "3333333333333333333333333333333333333333".to_string(), - ref_tree_hash: "4444444444444444444444444444444444444444".to_string(), - created_at: chrono::Utc::now().naive_utc(), - updated_at: chrono::Utc::now().naive_utc(), - is_cl: false, - }, - ]; - - let mut commits = Vec::new(); - let mut updates = Vec::new(); - let mut new_commit_id = String::new(); - - MonoServiceLogic::process_ref_updates( - &root_result, - &refs, - "merge root cl", - &mut commits, - &mut updates, - &mut new_commit_id, - ) - .expect("root merge flow should produce ref updates"); - - assert_eq!(commits.len(), 1); - assert_eq!(updates.len(), 2); - assert!(!new_commit_id.is_empty()); - - assert_eq!(updates[0].ref_name, "refs/cl/abcd1234"); - assert_eq!(updates[1].ref_name, MEGA_BRANCH_NAME); - assert!(updates.iter().all(|u| u.path == "/")); - assert!( - updates - .iter() - .all(|u| u.tree_hash == merged_tree_id.to_string()) - ); - } - - #[test] - fn test_map_tree_items_to_commits() { - let id1 = ObjectHash::Sha1([1u8; 20]); - let id2 = ObjectHash::Sha1([2u8; 20]); - let commit_hash = ObjectHash::Sha1([3u8; 20]); - - let item1 = TreeItem { - id: id1, - name: "file1.txt".into(), - mode: TreeItemMode::Blob, - }; - let item2 = TreeItem { - id: id2, - name: "file2.txt".into(), - mode: TreeItemMode::Blob, - }; - - let tree = Tree { - id: ObjectHash::Sha1([9u8; 20]), - tree_items: vec![item1.clone(), item2.clone()], - }; - - let mut item_to_commit_id = HashMap::new(); - item_to_commit_id.insert(id1.to_string(), commit_hash.to_string()); - - let fake_sig = Signature { - signature_type: SignatureType::Committer, - name: "tester".into(), - email: "tester@example.com".into(), - timestamp: 0, - timezone: "+0000".into(), - }; - - let commit_a = Commit { - id: commit_hash, - tree_id: ObjectHash::Sha1([8u8; 20]), - parent_commit_ids: vec![], - author: fake_sig.clone(), - committer: fake_sig.clone(), - message: "test commit".into(), - }; - - let mut commit_map = HashMap::new(); - commit_map.insert(commit_hash.to_string(), commit_a.clone()); - - let result = - MonoServiceLogic::map_tree_items_to_commits(tree, &item_to_commit_id, &commit_map); - - assert_eq!(result.get(&item1), Some(&Some(commit_a))); - assert_eq!(result.get(&item2), Some(&None)); - } - - #[test] - fn test_path_traversal_with_pop() { - let mut full_path = PathBuf::from("/project/rust/mega"); - for _ in 0..3 { - let cloned_path = full_path.clone(); // Clone full_path - let name = cloned_path.file_name().unwrap().to_str().unwrap(); - full_path.pop(); - println!("name: {name}, path: {full_path:?}"); - } - } - - #[test] - fn test_paging_calculation_basic() { - let files: Vec = vec![ - ClDiffFile::New( - PathBuf::from("file1.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - ), - ClDiffFile::Modified( - PathBuf::from("file2.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - ObjectHash::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(), - ), - ClDiffFile::Deleted( - PathBuf::from("file3.txt"), - ObjectHash::from_str("1111111111111111111111111111111111111111").unwrap(), - ), - ]; - - let page_size = 2u32; - let page_id = 1u32; - - let start = (page_id.saturating_sub(1)) * page_size; - let end = (start + page_size).min(files.len() as u32); - - assert_eq!(start, 0); - assert_eq!(end, 2); - - let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { - let start_idx = start as usize; - let end_idx = end as usize; - &files[start_idx..end_idx] - } else { - &[] - }; - - assert_eq!(page_slice.len(), 2); - } - - #[test] - fn test_paging_calculation_second_page() { - let files: Vec = vec![ - ClDiffFile::New( - PathBuf::from("file1.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - ), - ClDiffFile::Modified( - PathBuf::from("file2.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - ObjectHash::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(), - ), - ClDiffFile::Deleted( - PathBuf::from("file3.txt"), - ObjectHash::from_str("1111111111111111111111111111111111111111").unwrap(), - ), - ClDiffFile::New( - PathBuf::from("file4.txt"), - ObjectHash::from_str("2222222222222222222222222222222222222222").unwrap(), - ), - ]; - - let page_size = 2u32; - let page_id = 2u32; - - let start = (page_id.saturating_sub(1)) * page_size; - let end = (start + page_size).min(files.len() as u32); - - assert_eq!(start, 2); - assert_eq!(end, 4); - - let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { - let start_idx = start as usize; - let end_idx = end as usize; - &files[start_idx..end_idx] - } else { - &[] - }; - - assert_eq!(page_slice.len(), 2); - assert_eq!(page_slice[0].path(), &PathBuf::from("file3.txt")); - assert_eq!(page_slice[1].path(), &PathBuf::from("file4.txt")); - } - - #[test] - fn test_paging_calculation_partial_page() { - let files: Vec = vec![ - ClDiffFile::New( - PathBuf::from("file1.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - ), - ClDiffFile::Modified( - PathBuf::from("file2.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - ObjectHash::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(), - ), - ClDiffFile::Deleted( - PathBuf::from("file3.txt"), - ObjectHash::from_str("1111111111111111111111111111111111111111").unwrap(), - ), - ]; - - let page_size = 5u32; - let page_id = 1u32; - - let start = (page_id.saturating_sub(1)) * page_size; - let end = (start + page_size).min(files.len() as u32); - - assert_eq!(start, 0); - assert_eq!(end, 3); - - let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { - let start_idx = start as usize; - let end_idx = end as usize; - &files[start_idx..end_idx] - } else { - &[] - }; - - assert_eq!(page_slice.len(), 3); - } - - #[test] - fn test_paging_calculation_out_of_bounds() { - let files: Vec = vec![ClDiffFile::New( - PathBuf::from("file1.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - )]; - - let page_size = 2u32; - let page_id = 3u32; // Page that doesn't exist - - let start = (page_id.saturating_sub(1)) * page_size; - let end = (start + page_size).min(files.len() as u32); - - assert_eq!(start, 4); - assert_eq!(end, 1); // end is clamped to files.len() - - let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { - let start_idx = start as usize; - let end_idx = end as usize; - &files[start_idx..end_idx] - } else { - &[] - }; - - assert_eq!(page_slice.len(), 0); - } - - #[test] - fn test_paging_calculation_edge_case_zero_page_size() { - let files: Vec = vec![ClDiffFile::New( - PathBuf::from("file1.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - )]; - - let page_size = 0u32; - let page_id = 1u32; - - let start = (page_id.saturating_sub(1)) * page_size; - let end = (start + page_size).min(files.len() as u32); - - assert_eq!(start, 0); - assert_eq!(end, 0); - - let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { - let start_idx = start as usize; - let end_idx = end as usize; - &files[start_idx..end_idx] - } else { - &[] - }; - - assert_eq!(page_slice.len(), 0); - } - - #[test] - fn test_paging_calculation_zero_page_id() { - let files: Vec = vec![ - ClDiffFile::New( - PathBuf::from("file1.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - ), - ClDiffFile::Modified( - PathBuf::from("file2.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - ObjectHash::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(), - ), - ]; - - let page_size = 2u32; - let page_id = 0u32; // Should be treated as page 1 due to saturating_sub - - let start = (page_id.saturating_sub(1)) * page_size; - let end = (start + page_size).min(files.len() as u32); - - assert_eq!(start, 0); - assert_eq!(end, 2); - - let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { - let start_idx = start as usize; - let end_idx = end as usize; - &files[start_idx..end_idx] - } else { - &[] - }; - - assert_eq!(page_slice.len(), 2); - } - - #[test] - fn test_paging_algorithm() { - let total_files = 10usize; - let current_page = 2u32; - let page_size = 3u32; - - let total_pages = total_files.div_ceil(page_size as usize); - let current_page = current_page as usize; - let page_size = page_size as usize; - - assert_eq!(total_pages, 4); - assert_eq!(current_page, 2); - assert_eq!(page_size, 3); - } - - #[test] - fn test_collect_page_blobs_new_files() { - let files = vec![ClDiffFile::New( - PathBuf::from("new_file.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - )]; - - let mut old_blobs = Vec::new(); - let mut new_blobs = Vec::new(); - - collect_page_blobs(&files, &mut old_blobs, &mut new_blobs); - - assert_eq!(old_blobs.len(), 0); - assert_eq!(new_blobs.len(), 1); - assert_eq!(new_blobs[0].0, PathBuf::from("new_file.txt")); - } - - #[test] - fn test_collect_page_blobs_deleted_files() { - let files = vec![ClDiffFile::Deleted( - PathBuf::from("deleted_file.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - )]; - - let mut old_blobs = Vec::new(); - let mut new_blobs = Vec::new(); - - collect_page_blobs(&files, &mut old_blobs, &mut new_blobs); - - assert_eq!(old_blobs.len(), 1); - assert_eq!(new_blobs.len(), 0); - assert_eq!(old_blobs[0].0, PathBuf::from("deleted_file.txt")); - } - - #[test] - fn test_file_lists_with_roots() { - let all_files = vec![ - "src/main.rs".to_string(), - "src/utils/math.rs".to_string(), - "src/utils/io.rs".to_string(), - "README.md".to_string(), - ]; - - let root: Option<&str> = None; - let filtered_none: Vec = all_files - .iter() - .filter(|file_path| { - if let Some(prefix) = root { - file_path.starts_with(prefix) - } else { - true - } - }) - .cloned() - .collect(); - - assert_eq!(filtered_none.len(), 4); - assert_eq!(filtered_none, all_files); - - let filtered_some: Vec = all_files - .iter() - .filter(|file_path| { - if let Some(prefix) = Some("src/utils") { - file_path.starts_with(prefix) - } else { - true - } - }) - .cloned() - .collect(); - - assert_eq!(filtered_some.len(), 2); - assert_eq!( - filtered_some, - vec![ - "src/utils/math.rs".to_string(), - "src/utils/io.rs".to_string() - ] - ); - } - - #[test] - fn test_collect_page_blobs_modified_files() { - let files = vec![ClDiffFile::Modified( - PathBuf::from("modified_file.txt"), - ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), - ObjectHash::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(), - )]; - - let mut old_blobs = Vec::new(); - let mut new_blobs = Vec::new(); - - collect_page_blobs(&files, &mut old_blobs, &mut new_blobs); - - assert_eq!(old_blobs.len(), 1); - assert_eq!(new_blobs.len(), 1); - assert_eq!(old_blobs[0].0, PathBuf::from("modified_file.txt")); - assert_eq!(new_blobs[0].0, PathBuf::from("modified_file.txt")); - } - - #[test] - fn test_collect_page_blobs_mixed_files() { - let files = vec![ - ClDiffFile::New( - PathBuf::from("new.txt"), - ObjectHash::from_str("1111111111111111111111111111111111111111").unwrap(), - ), - ClDiffFile::Deleted( - PathBuf::from("deleted.txt"), - ObjectHash::from_str("2222222222222222222222222222222222222222").unwrap(), - ), - ClDiffFile::Modified( - PathBuf::from("modified.txt"), - ObjectHash::from_str("3333333333333333333333333333333333333333").unwrap(), - ObjectHash::from_str("4444444444444444444444444444444444444444").unwrap(), - ), - ]; - - let mut old_blobs = Vec::new(); - let mut new_blobs = Vec::new(); - - collect_page_blobs(&files, &mut old_blobs, &mut new_blobs); - - assert_eq!(old_blobs.len(), 2); // deleted + modified - assert_eq!(new_blobs.len(), 2); // new + modified - - assert_eq!(old_blobs[0].0, PathBuf::from("deleted.txt")); - assert_eq!(old_blobs[1].0, PathBuf::from("modified.txt")); - assert_eq!(new_blobs[0].0, PathBuf::from("new.txt")); - assert_eq!(new_blobs[1].0, PathBuf::from("modified.txt")); - } - - #[test] - fn test_relocate_patch_body_rewrites_paths_and_keeps_hunk() { - let raw_patch = "\ -diff --git a/old/name.txt b/old/name.txt\n\ -index 1111111..2222222 100644\n\ ---- a/old/name.txt\n\ -+++ b/old/name.txt\n\ -@@ -1 +1 @@\n\ --old line\n\ -+new line\n"; - - let relocated = MonoApiService::relocate_patch_body( - raw_patch, - Path::new("old/name.txt"), - Path::new("new/name.txt"), - ); - - assert!(!relocated.contains("diff --git")); - assert!(relocated.contains("--- a/old/name.txt")); - assert!(relocated.contains("+++ b/new/name.txt")); - assert!(relocated.contains("@@ -1 +1 @@")); - assert!(relocated.contains("-old line")); - assert!(relocated.contains("+new line")); - assert!(!relocated.contains("deleted file mode")); - } - - #[test] - fn test_relocate_patch_body_preserves_hunk_backslashes() { - let raw_patch = "\ -diff --git a/old/name.txt b/old/name.txt\n\ -index 1111111..2222222 100644\n\ ---- a/old/name.txt\n\ -+++ b/old/name.txt\n\ -@@ -1 +1 @@\n\ --let path = \"C:\\\\temp\\\\old\";\n\ -+let path = \"C:\\\\temp\\\\new\";\n"; - - let relocated = MonoApiService::relocate_patch_body( - raw_patch, - Path::new("old/name.txt"), - Path::new("new/name.txt"), - ); - - assert!(relocated.contains("--- a/old/name.txt")); - assert!(relocated.contains("+++ b/new/name.txt")); - assert!(relocated.contains("-let path = \"C:\\\\temp\\\\old\";")); - assert!(relocated.contains("+let path = \"C:\\\\temp\\\\new\";")); - } - - #[test] - fn test_normalize_diff_item_path_uses_forward_slashes() { - let item = DiffItem { - path: "dir\\nested\\file.txt".to_string(), - data: "diff --git a/dir\\nested\\file.txt b/dir\\nested\\file.txt\n".to_string(), - }; - - let normalized = MonoApiService::normalize_diff_item(item); - assert_eq!(normalized.path, "dir/nested/file.txt"); - assert!( - normalized - .data - .contains("diff --git a/dir/nested/file.txt b/dir/nested/file.txt") - ); - } - - #[test] - fn test_normalize_patch_header_paths_preserves_hunk_content() { - let raw_patch = "\ -diff --git a/dir\\nested\\file.txt b/dir\\nested\\file.txt\n\ ---- a/dir\\nested\\file.txt\n\ -+++ b/dir\\nested\\file.txt\n\ -@@ -1 +1 @@\n\ --let path = \"C:\\\\temp\\\\old\";\n\ -+let path = \"C:\\\\temp\\\\new\";\n"; - - let normalized = MonoApiService::normalize_patch_header_paths(raw_patch); - - assert!(normalized.contains("diff --git a/dir/nested/file.txt b/dir/nested/file.txt")); - assert!(normalized.contains("--- a/dir/nested/file.txt")); - assert!(normalized.contains("+++ b/dir/nested/file.txt")); - assert!(normalized.contains("-let path = \"C:\\\\temp\\\\old\";")); - assert!(normalized.contains("+let path = \"C:\\\\temp\\\\new\";")); - } - - #[tokio::test] - async fn test_content_diff_functionality() { - use std::collections::HashMap; - - use git_internal::internal::object::blob::Blob; - - // Test basic diff generation with sample data - let old_content = "Hello World\nLine 2\nLine 3"; - let new_content = "Hello Universe\nLine 2\nLine 3 modified"; - - let old_blob = Blob::from_content(old_content); - let new_blob = Blob::from_content(new_content); - - let old_blobs = vec![(PathBuf::from("test_file.txt"), old_blob.id)]; - let new_blobs = vec![(PathBuf::from("test_file.txt"), new_blob.id)]; - - // Create a blob cache for the test - let mut blob_cache: HashMap> = HashMap::new(); - blob_cache.insert(old_blob.id, old_content.as_bytes().to_vec()); - blob_cache.insert(new_blob.id, new_content.as_bytes().to_vec()); - - // Test the diff engine directly - let read_content = |_file: &PathBuf, hash: &ObjectHash| -> Vec { - blob_cache.get(hash).cloned().unwrap_or_default() - }; - - let diff_output: Vec = - GitDiff::diff(old_blobs, new_blobs, Vec::new(), read_content) - .into_iter() - .collect(); - - // Verify diff output contains expected content - assert!(!diff_output.is_empty(), "Diff output should not be empty"); - assert_eq!(diff_output.len(), 1, "Should have diff for one file"); - - let diff_item = &diff_output[0]; - assert_eq!(diff_item.path, "test_file.txt"); - assert!( - diff_item.data.contains("diff --git"), - "Should contain git diff header" - ); - assert!( - diff_item.data.contains("-Hello World"), - "Should show removed line" - ); - assert!( - diff_item.data.contains("+Hello Universe"), - "Should show added line" - ); - assert!(diff_item.data.contains("-Line 3"), "Should show old line 3"); - assert!( - diff_item.data.contains("+Line 3 modified"), - "Should show new line 3" - ); - } - - #[tokio::test] - async fn test_get_diff_by_blobs_with_empty_content() { - // Test diff generation with empty content (simulating missing blobs) - let old_hash = ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(); - let new_hash = ObjectHash::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(); - - let old_blobs = vec![(PathBuf::from("empty_file.txt"), old_hash)]; - let new_blobs = vec![(PathBuf::from("empty_file.txt"), new_hash)]; - - // Create empty blob cache to simulate missing blobs - let blob_cache: HashMap> = HashMap::new(); - - let read_content = |_file: &PathBuf, hash: &ObjectHash| -> Vec { - blob_cache.get(hash).cloned().unwrap_or_default() - }; - - // Test the diff engine with empty content - let diff_output: Vec = - GitDiff::diff(old_blobs, new_blobs, Vec::new(), read_content) - .into_iter() - .collect(); - - assert!( - !diff_output.is_empty(), - "Should generate diff even with empty blobs" - ); - assert_eq!(diff_output[0].path, "empty_file.txt"); - assert!( - diff_output[0].data.contains("diff --git"), - "Should contain git diff header" - ); - } -} - -#[test] -fn test_parse_github_link() { - let url = "https://github.com/web3infra-foundation/libra/"; - let url = url - .trim_end_matches(".git") - .trim_end_matches("/") - .strip_prefix("https://github.com/") - .expect("Invalid GitHub URL"); - let (owner, repo) = url.rsplit_once('/').unwrap(); - assert_eq!(owner, "web3infra-foundation"); - assert_eq!(repo, "libra"); -} - -#[tokio::test] -async fn test_third_party_trait() { - let url = "https://github.com/aidcheng/mega.git"; - let third_party_client = ThirdPartyClient::new(url); - - let (_, refs) = match third_party_client.fetch_refs().await { - Ok(refs) => refs, - Err(err) => { - tracing::warn!( - "Skipping test_third_party_trait because remote refs are unavailable: {}", - err - ); - return; - } - }; - - let res = match third_party_client.fetch_packs(&[refs]).await { - Ok(res) => res, - Err(err) => { - tracing::warn!( - "Skipping test_third_party_trait because pack fetch failed: {}", - err - ); - return; - } - }; - - if let Err(err) = third_party_client.process_pack_stream(res).await { - tracing::warn!( - "Skipping test_third_party_trait because pack processing failed: {}", - err - ); - } -} diff --git a/ceres/src/api_service/state.rs b/ceres/src/api_service/state.rs deleted file mode 100644 index 15b96cfdb..000000000 --- a/ceres/src/api_service/state.rs +++ /dev/null @@ -1,22 +0,0 @@ -use std::sync::Arc; - -use jupiter::storage::Storage; - -use crate::api_service::cache::GitObjectCache; - -#[derive(Clone)] -/// Shared state for the protocol API service. -/// -/// `ProtocolApiState` provides access to the underlying storage backend and a shared -/// cache for Git objects. It is intended to be passed to API handlers and services -/// that require access to repository data and object caching. -/// -/// # Usage -/// Construct a `ProtocolApiState` with the required storage and cache, and share it -/// across API endpoints or service layers that need to interact with repository data. -pub struct ProtocolApiState { - /// The storage backend used for accessing repository data and objects. - pub storage: Storage, - /// Shared cache for Git objects to improve access performance and reduce backend load. - pub git_object_cache: Arc, -} diff --git a/ceres/src/api_service/blame_ops.rs b/ceres/src/application/api_service/blame_ops.rs similarity index 100% rename from ceres/src/api_service/blame_ops.rs rename to ceres/src/application/api_service/blame_ops.rs diff --git a/ceres/src/api_service/blob_ops.rs b/ceres/src/application/api_service/blob_ops.rs similarity index 100% rename from ceres/src/api_service/blob_ops.rs rename to ceres/src/application/api_service/blob_ops.rs diff --git a/ceres/src/api_service/buck_tree_builder.rs b/ceres/src/application/api_service/buck_tree_builder.rs similarity index 100% rename from ceres/src/api_service/buck_tree_builder.rs rename to ceres/src/application/api_service/buck_tree_builder.rs diff --git a/ceres/src/application/api_service/cache.rs b/ceres/src/application/api_service/cache.rs new file mode 100644 index 000000000..77c6e0409 --- /dev/null +++ b/ceres/src/application/api_service/cache.rs @@ -0,0 +1,2 @@ +//! Re-export; implementation lives in [`crate::infra::cache`]. +pub use crate::infra::cache::*; diff --git a/ceres/src/api_service/commit_ops.rs b/ceres/src/application/api_service/commit_ops.rs similarity index 100% rename from ceres/src/api_service/commit_ops.rs rename to ceres/src/application/api_service/commit_ops.rs diff --git a/ceres/src/api_service/history.rs b/ceres/src/application/api_service/history.rs similarity index 100% rename from ceres/src/api_service/history.rs rename to ceres/src/application/api_service/history.rs diff --git a/ceres/src/api_service/import_api_service.rs b/ceres/src/application/api_service/import_api_service.rs similarity index 74% rename from ceres/src/api_service/import_api_service.rs rename to ceres/src/application/api_service/import_api_service.rs index c37b7b9f1..5aab9d28f 100644 --- a/ceres/src/api_service/import_api_service.rs +++ b/ceres/src/application/api_service/import_api_service.rs @@ -24,7 +24,16 @@ use git_internal::{ use jupiter::{storage::Storage, utils::converter::FromGitModel}; use crate::{ - api_service::{ApiHandler, cache::GitObjectCache, history}, + api_service::{ + ApiHandler, + cache::GitObjectCache, + history, + mono::MonoServiceLogic, + tag_ops::{ + self, build_git_internal_tag, db_error, format_tagger_info, is_annotated_tag, + lightweight_commit_tag, merge_paginated_tags, tag_already_exists, tags_full_ref, + }, + }, model::{ git::{CreateEntryInfo, CreateEntryResult, EditFilePayload, EditFileResult}, tag::TagInfo, @@ -167,51 +176,35 @@ impl ApiHandler for ImportApiService { message: Option, ) -> Result { let git_storage = self.storage.git_db_storage(); - let is_annotated = message.as_ref().map(|s| !s.is_empty()).unwrap_or(false); - let tagger_info = match (tagger_name, tagger_email) { - (Some(n), Some(e)) => format!("{} <{}>", n, e), - (Some(n), None) => n, - (None, Some(e)) => e, - (None, None) => "unknown".to_string(), - }; + let tagger_info = format_tagger_info(tagger_name, tagger_email); - // validate target commit if provided self.validate_target_commit(target.as_ref()).await?; - let full_ref = format!("refs/tags/{}", name.clone()); - // Prevent duplicate tag/ref creation: check annotated table and refs first. + let full_ref = tags_full_ref(&name); match git_storage .get_tag_by_repo_and_name(self.repo.repo_id, &name) .await { - Ok(Some(_)) => { - return Err(GitError::CustomError(format!( - "[code:400] Tag '{}' already exists", - name - ))); - } + Ok(Some(_)) => return Err(tag_already_exists(&name)), Ok(None) => {} Err(e) => { tracing::error!("DB error while checking git_tag existence: {}", e); - return Err(GitError::CustomError("[code:500] DB error".to_string())); + return Err(db_error()); } } if let Ok(refs) = git_storage.get_ref(self.repo.repo_id).await && refs.iter().any(|r| r.ref_name == full_ref) { - return Err(GitError::CustomError(format!( - "[code:400] Tag '{}' already exists", - name - ))); + return Err(tag_already_exists(&name)); } - if is_annotated { + + if is_annotated_tag(&message) { return self .create_annotated_tag(&git_storage, full_ref, name, target, tagger_info, message) .await; } - // lightweight self.create_lightweight_tag(&git_storage, full_ref, name, target, tagger_info) .await } @@ -234,8 +227,7 @@ impl ApiHandler for ImportApiService { } }; - // map annotated page into TagInfo - let mut result: Vec = annotated_tags_page + let result: Vec = annotated_tags_page .into_iter() .map(|t| TagInfo { name: t.tag_name, @@ -248,48 +240,30 @@ impl ApiHandler for ImportApiService { }) .collect(); - // lightweight refs let mut lightweight_refs: Vec = vec![]; if let Ok(refs) = git_storage.get_ref(self.repo.repo_id).await { for r in refs { if r.ref_name.starts_with("refs/tags/") { let tag_name = r.ref_name.trim_start_matches("refs/tags/").to_string(); - // skip if annotated exists (anywhere) - // Note: we only have the annotated page in memory; to avoid duplicate names we check by tag_name against annotated page and will accept duplicates only if not present. if result.iter().any(|t| t.name == tag_name) { continue; } - let created_at = r.created_at.and_utc().to_rfc3339(); - lightweight_refs.push(TagInfo { - name: tag_name.clone(), - tag_id: r.ref_git_id.clone(), - object_id: r.ref_git_id.clone(), - object_type: "commit".to_string(), - tagger: "".to_string(), - message: "".to_string(), - created_at, - }); + lightweight_refs.push(lightweight_commit_tag( + tag_name, + r.ref_git_id.clone(), + "", + r.created_at.and_utc().to_rfc3339(), + )); } } } - // total is annotated_total + lightweight_refs.len() - let total = annotated_total + lightweight_refs.len() as u64; - - // fill page: annotated page items come first, then lightweight refs to make up page size - let per_page = if pagination.per_page == 0 { - 20 - } else { - pagination.per_page - } as usize; - if result.len() < per_page { - let need = per_page - result.len(); - for r in lightweight_refs.into_iter().take(need) { - result.push(r); - } - } - - Ok((result, total)) + Ok(merge_paginated_tags( + result, + lightweight_refs, + annotated_total, + pagination.per_page, + )) } async fn get_tag( @@ -320,21 +294,16 @@ impl ApiHandler for ImportApiService { return Err(GitError::CustomError("[code:500] DB error".to_string())); } } - // check import_refs for lightweight - let full_ref = format!("refs/tags/{}", name.clone()); + let full_ref = tags_full_ref(&name); if let Ok(refs) = git_storage.get_ref(self.repo.repo_id).await { for r in refs { if r.ref_name == full_ref { - let created_at = r.created_at.and_utc().to_rfc3339(); - return Ok(Some(TagInfo { - name: name.clone(), - tag_id: r.ref_git_id.clone(), - object_id: r.ref_git_id.clone(), - object_type: "commit".to_string(), - tagger: "".to_string(), - message: "".to_string(), - created_at, - })); + return Ok(Some(lightweight_commit_tag( + name, + r.ref_git_id.clone(), + "", + r.created_at.and_utc().to_rfc3339(), + ))); } } } @@ -349,8 +318,7 @@ impl ApiHandler for ImportApiService { .await { Ok(Some(_tag)) => { - // remove import ref if exists - let full_ref = format!("refs/tags/{}", name.clone()); + let full_ref = tags_full_ref(&name); git_storage .remove_ref(self.repo.repo_id, &full_ref) .await @@ -371,8 +339,7 @@ impl ApiHandler for ImportApiService { Ok(()) } Ok(None) => { - // remove lightweight ref if exists - let full_ref = format!("refs/tags/{}", name.clone()); + let full_ref = tags_full_ref(&name); git_storage .remove_ref(self.repo.repo_id, &full_ref) .await @@ -418,7 +385,7 @@ impl ApiHandler for ImportApiService { // Create new blob and rebuild tree up to root let new_blob = Blob::from_content(&payload.content); let (updated_trees, new_root_id) = - self.build_updated_trees(path.clone(), update_chain, new_blob.id)?; + MonoServiceLogic::propagate_tree_chain(path.clone(), update_chain, new_blob.id)?; // Save commit and objects under import repo tables let git_storage = self.storage.git_db_storage(); @@ -488,12 +455,8 @@ impl ImportApiService { message: Option, ) -> Result { // build git_internal tag and models - let (tag_id_hex, object_id) = self.build_git_internal_tag( - name.clone(), - target.clone(), - tagger_info.clone(), - message.clone(), - )?; + let (tag_id_hex, object_id) = + build_git_internal_tag(name.clone(), target, tagger_info.clone(), message.clone())?; let new_model = self.build_git_tag_model( tag_id_hex.clone(), @@ -559,66 +522,32 @@ impl ImportApiService { tracing::error!("Failed to write import ref for lightweight tag: {}", e); GitError::CustomError("[code:500] Failed to write import ref".to_string()) })?; - Ok(TagInfo { - name: name.clone(), - tag_id: object_id.clone(), - object_id: object_id.clone(), - object_type: "commit".to_string(), - tagger: tagger_info.clone(), - message: String::new(), + Ok(lightweight_commit_tag( + name, + object_id, + tagger_info, created_at, - }) + )) } async fn validate_target_commit(&self, target: Option<&String>) -> Result<(), GitError> { - if let Some(ref t) = target { + if let Some(t) = target { let git_storage = self.storage.git_db_storage(); match git_storage.get_commit_by_hash(self.repo.repo_id, t).await { Ok(c) => { if c.is_none() { - return Err(GitError::CustomError(format!( - "[code:404] Target commit '{}' not found", - t - ))); + return Err(tag_ops::commit_not_found(t)); } } Err(e) => { tracing::error!("DB error while fetching commit by hash: {}", e); - return Err(GitError::CustomError("[code:500] DB error".to_string())); + return Err(db_error()); } } } Ok(()) } - fn build_git_internal_tag( - &self, - name: String, - target: Option, - tagger_info: String, - message: Option, - ) -> Result<(String, String), GitError> { - let tag_target = target - .as_ref() - .ok_or(GitError::InvalidCommitObject) - .and_then(|t| ObjectHash::from_str(t).map_err(|_| GitError::InvalidCommitObject))?; - let git_internal_tag = git_internal::internal::object::tag::Tag::new( - tag_target, - git_internal::internal::object::types::ObjectType::Commit, - name.clone(), - git_internal::internal::object::signature::Signature::new( - git_internal::internal::object::signature::SignatureType::Tagger, - tagger_info.clone(), - String::new(), - ), - message.clone().unwrap_or_default(), - ); - Ok(( - git_internal_tag.id.to_string(), - target.unwrap_or_else(|| "HEAD".to_string()), - )) - } - fn build_git_tag_model( &self, tag_id_hex: String, @@ -675,43 +604,4 @@ impl ImportApiService { } Ok(()) } - - fn update_tree_hash( - &self, - tree: Arc, - name: &str, - target_hash: ObjectHash, - ) -> Result { - let index = tree - .tree_items - .iter() - .position(|item| item.name == name) - .ok_or_else(|| GitError::CustomError(format!("Tree item '{}' not found", name)))?; - let mut items = tree.tree_items.clone(); - items[index].id = target_hash; - Tree::from_tree_items(items).map_err(|_| GitError::CustomError("Invalid tree".to_string())) - } - - /// Build updated trees chain and return (updated_trees, new_root_tree_id) - fn build_updated_trees( - &self, - mut path: PathBuf, - mut update_chain: Vec>, - mut updated_tree_hash: ObjectHash, - ) -> Result<(Vec, ObjectHash), GitError> { - let mut updated_trees = Vec::new(); - while let Some(tree) = update_chain.pop() { - let cloned_path = path.clone(); - let name = cloned_path - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| GitError::CustomError("Invalid path".into()))?; - path.pop(); - - let new_tree = self.update_tree_hash(tree, name, updated_tree_hash)?; - updated_tree_hash = new_tree.id; - updated_trees.push(new_tree); - } - Ok((updated_trees, updated_tree_hash)) - } } diff --git a/ceres/src/api_service/mod.rs b/ceres/src/application/api_service/mod.rs similarity index 97% rename from ceres/src/api_service/mod.rs rename to ceres/src/application/api_service/mod.rs index cca4c11fa..844793cf7 100644 --- a/ceres/src/api_service/mod.rs +++ b/ceres/src/application/api_service/mod.rs @@ -30,20 +30,23 @@ use crate::{ }, }; -pub mod admin_ops; pub mod blame_ops; pub mod blob_ops; -pub mod bot_ops; pub mod buck_tree_builder; pub mod cache; pub mod commit_ops; -pub mod group_ops; pub mod history; pub mod import_api_service; -pub mod mono_api_service; +pub mod mono; pub mod state; +pub mod tag_ops; pub mod tree_ops; +pub use mono::{ + ADMIN_FILE, EffectiveResourcePermission, MonoApiService, MonoServiceLogic, RefUpdate, + TreeUpdateResult, cl_merge, +}; + #[async_trait] pub trait ApiHandler: Send + Sync { fn get_context(&self) -> Storage; diff --git a/ceres/src/api_service/bot_ops.rs b/ceres/src/application/api_service/mono/admin/bot.rs similarity index 97% rename from ceres/src/api_service/bot_ops.rs rename to ceres/src/application/api_service/mono/admin/bot.rs index d2e6b5092..e2c828461 100644 --- a/ceres/src/api_service/bot_ops.rs +++ b/ceres/src/application/api_service/mono/admin/bot.rs @@ -3,7 +3,7 @@ use callisto::sea_orm_active_enums::{ }; use common::errors::MegaError; -use crate::api_service::mono_api_service::MonoApiService; +use crate::api_service::mono::MonoApiService; impl MonoApiService { /// Check whether a bot has sufficient permission on a given resource. diff --git a/ceres/src/api_service/group_ops.rs b/ceres/src/application/api_service/mono/admin/group.rs similarity index 79% rename from ceres/src/api_service/group_ops.rs rename to ceres/src/application/api_service/mono/admin/group.rs index 2612951ba..1b2ef9a58 100644 --- a/ceres/src/api_service/group_ops.rs +++ b/ceres/src/application/api_service/mono/admin/group.rs @@ -8,7 +8,7 @@ use jupiter::model::group_dto::{ CreateGroupPayload, DeleteGroupStats, ResourcePermissionBinding, UpdateGroupPayload, }; -use crate::api_service::mono_api_service::MonoApiService; +use crate::{api_service::mono::MonoApiService, model::group::ResourceTypeValue}; #[derive(Debug, Clone)] pub struct EffectiveResourcePermission { @@ -202,6 +202,57 @@ impl MonoApiService { }) } + pub async fn resolve_resource_id( + &self, + resource_type: ResourceTypeValue, + resource_id: &str, + ) -> Result { + let normalized_resource_id = resource_id.trim(); + if normalized_resource_id.is_empty() { + return Err(MegaError::Other( + "resource_id must not be empty".to_string(), + )); + } + + match resource_type { + ResourceTypeValue::Note => { + let note = self + .storage + .note_storage() + .get_note_by_public_id(normalized_resource_id) + .await?; + match note { + Some(note) => Ok(note.public_id), + None => { + tracing::warn!( + resource_id = normalized_resource_id, + "note resource missing in mono notes table; falling back to raw public_id" + ); + Ok(normalized_resource_id.to_string()) + } + } + } + } + } + + /// Validates resource type and resolves the canonical resource id (e.g. note public_id). + pub async fn resolve_resource_context( + &self, + resource_type: &str, + resource_id: &str, + ) -> Result<(ResourceTypeEnum, ResourceTypeValue, String), MegaError> { + let resource_type_value = ResourceTypeValue::try_from(resource_type) + .map_err(|err| MegaError::Other(err.to_string()))?; + let validated_resource_id = self + .resolve_resource_id(resource_type_value, resource_id) + .await?; + Ok(( + resource_type_value.into(), + resource_type_value, + validated_resource_id, + )) + } + /// Reserved for future business-route authorization integration. pub async fn check_resource_permission( &self, diff --git a/ceres/src/application/api_service/mono/admin/mod.rs b/ceres/src/application/api_service/mono/admin/mod.rs new file mode 100644 index 000000000..fdc5dc49b --- /dev/null +++ b/ceres/src/application/api_service/mono/admin/mod.rs @@ -0,0 +1,6 @@ +pub mod bot; +pub mod group; +pub mod permissions; + +pub use group::EffectiveResourcePermission; +pub use permissions::ADMIN_FILE; diff --git a/ceres/src/api_service/admin_ops.rs b/ceres/src/application/api_service/mono/admin/permissions.rs similarity index 98% rename from ceres/src/api_service/admin_ops.rs rename to ceres/src/application/api_service/mono/admin/permissions.rs index 2e3187547..8e4210f24 100644 --- a/ceres/src/api_service/admin_ops.rs +++ b/ceres/src/application/api_service/mono/admin/permissions.rs @@ -13,7 +13,7 @@ use common::errors::MegaError; use git_internal::internal::object::tree::Tree; use jupiter::{redis::AsyncCommands, utils::converter::FromMegaModel}; -use crate::api_service::mono_api_service::MonoApiService; +use crate::api_service::mono::MonoApiService; /// Cache TTL for admin list (10 minutes). pub const ADMIN_CACHE_TTL: u64 = 600; diff --git a/ceres/src/application/api_service/mono/buck/mod.rs b/ceres/src/application/api_service/mono/buck/mod.rs new file mode 100644 index 000000000..4c8a0a3b0 --- /dev/null +++ b/ceres/src/application/api_service/mono/buck/mod.rs @@ -0,0 +1,3 @@ +//! Buck upload sessions. + +pub mod upload; diff --git a/ceres/src/application/api_service/mono/buck/upload.rs b/ceres/src/application/api_service/mono/buck/upload.rs new file mode 100644 index 000000000..1a8ebba14 --- /dev/null +++ b/ceres/src/application/api_service/mono/buck/upload.rs @@ -0,0 +1,371 @@ +//! Buck upload operations for [`MonoApiService`](super::service::MonoApiService). + +use std::{collections::HashMap, path::PathBuf, sync::Arc}; + +use callisto; +use common::errors::{BuckError, MegaError}; +use jupiter::{ + service::buck_service::{ + CommitArtifacts, CompletePayload as SvcCompletePayload, + CompleteResponse as SvcCompleteResponse, + }, + storage::buck_storage::{session_status, upload_status}, + utils::converter::IntoMegaModel, +}; +use orion_client::OrionBuildClient; + +use crate::{ + api_service::{ + buck_tree_builder::BuckCommitBuilder, + mono::{MonoApiService, MonoServiceLogic}, + }, + build_trigger::{BuildTriggerService, TriggerContext}, + model::buck::{ + CompletePayload, CompleteResponse, DEFAULT_MODE, FileChange, + FileToUpload as ApiFileToUpload, ManifestPayload, ManifestResponse, + }, +}; + +impl MonoApiService { + /// Triggers a build for Buck upload completion + fn trigger_build_for_buck_upload(&self, response: &CompleteResponse, username: &str) { + let config = self.storage.config(); + let orion_client = Arc::new(OrionBuildClient::new(config.build.clone())); + if !orion_client.enable_build() { + return; + } + let storage = self.storage.clone(); + let git_cache = self.git_object_cache.clone(); + let mut context = TriggerContext::from_buck_upload( + response.repo_path.clone(), + response.from_hash.clone(), + response.commit_id.clone(), + response.cl_link.clone(), + Some(response.cl_id), + Some(username.to_string()), + ); + context.ref_name = Some("main".to_string()); + context.ref_type = Some("branch".to_string()); + tokio::spawn(async move { + if let Err(e) = + BuildTriggerService::build_by_context(storage, git_cache, orion_client, context) + .await + { + tracing::error!("Failed to create build trigger for buck upload: {}", e); + } + }); + } + pub async fn create_buck_session( + &self, + username: &str, + path: &str, + ) -> Result { + let normalized_path = MonoServiceLogic::normalize_repo_path(path)?; + let refs = self + .storage + .mono_storage() + .get_main_ref(&normalized_path) + .await? + .ok_or_else(|| MegaError::NotFound(format!("Path not found: {}", normalized_path)))?; + let base_branch = refs + .ref_name + .strip_prefix("refs/heads/") + .unwrap_or(refs.ref_name.as_str()) + .to_string(); + // Use canonical path from mega_refs as the single source of truth for repository path + let canonical_path = refs.path.clone(); + let response = self + .storage + .buck_service + .create_session( + username, + &canonical_path, + &base_branch, + refs.ref_commit_hash, + ) + .await?; + + Ok(response) + } + + /// Process buck upload manifest. + /// + /// # Arguments + /// * `username` - User processing the manifest + /// * `cl_link` - CL link + /// * `payload` - Manifest payload + /// + /// # Returns + /// Returns `ManifestResponse` on success + pub async fn process_buck_manifest( + &self, + username: &str, + cl_link: &str, + payload: ManifestPayload, + ) -> Result { + let session = self + .storage + .buck_storage() + .get_session(cl_link) + .await? + .ok_or_else(|| MegaError::Buck(BuckError::SessionNotFound(cl_link.to_string())))?; + + if session.user_id != username { + return Err(MegaError::Buck(BuckError::Forbidden( + "Session belongs to another user".to_string(), + ))); + } + + let manifest_paths: Vec = payload + .files + .iter() + .map(|f| PathBuf::from(&f.path)) + .collect(); + + // Get content hashes (raw SHA-1) and blob IDs + let (existing_file_hashes, existing_blob_ids_map) = + crate::api_service::blob_ops::get_files_content_hashes_with_blob_ids( + self, + &manifest_paths, + session.from_hash.as_deref(), + ) + .await + .map_err(MegaError::Git)?; + + // Convert ObjectHash to String for storage + let existing_blob_ids: HashMap = existing_blob_ids_map + .into_iter() + .map(|(path, blob_hash)| (path, blob_hash.to_string())) + .collect(); + + // Convert payload to service layer type + let service_payload = jupiter::service::buck_service::ManifestPayload { + files: payload + .files + .iter() + .map(|f| jupiter::service::buck_service::ManifestFile { + path: f.path.clone(), + size: f.size, + hash: f.hash.clone(), + }) + .collect(), + commit_message: payload.commit_message.clone(), + }; + + let svc_resp = self + .storage + .buck_service + .process_manifest( + username, + cl_link, + service_payload, + existing_file_hashes, + existing_blob_ids, + ) + .await?; + + // Convert back to API layer response + let api_resp = ManifestResponse { + total_files: svc_resp.total_files, + total_size: svc_resp.total_size, + files_to_upload: svc_resp + .files_to_upload + .into_iter() + .map(|f| ApiFileToUpload { + path: f.path, + reason: f.reason, + }) + .collect(), + files_unchanged: svc_resp.files_unchanged, + upload_size: svc_resp.upload_size, + }; + + Ok(api_resp) + } + + pub fn buck_max_file_size(&self) -> u64 { + self.storage.buck_service.max_file_size() + } + + pub fn buck_try_acquire_upload_permits( + &self, + file_size: u64, + ) -> Result< + ( + tokio::sync::OwnedSemaphorePermit, + Option, + ), + MegaError, + > { + self.storage + .buck_service + .try_acquire_upload_permits(file_size) + } + + pub async fn upload_buck_file( + &self, + username: &str, + cl_link: &str, + file_path: &str, + file_size: u64, + file_hash: Option<&str>, + file_content: bytes::Bytes, + ) -> Result { + self.storage + .buck_service + .upload_file( + username, + cl_link, + file_path, + file_size, + file_hash, + file_content, + ) + .await + } + + /// Complete buck upload. + /// + /// Commit message is read from session.commit_message which is set during Manifest phase. + /// The payload is intentionally unused (empty struct). + /// + /// # Arguments + /// * `username` - User completing the upload + /// * `cl_link` - CL link + /// * `_payload` - Empty payload (unused). Commit message is read from session.commit_message + /// which is set during Manifest phase. + /// + /// # Returns + /// Returns `CompleteResponse` on success + pub async fn complete_buck_upload( + &self, + username: &str, + cl_link: &str, + _payload: CompletePayload, + ) -> Result { + let session = self + .storage + .buck_storage() + .get_session(cl_link) + .await? + .ok_or_else(|| MegaError::Buck(BuckError::SessionNotFound(cl_link.to_string())))?; + + if session.user_id != username { + return Err(MegaError::Buck(BuckError::Forbidden( + "Session belongs to another user".to_string(), + ))); + } + + if ![session_status::MANIFEST_UPLOADED, session_status::UPLOADING] + .contains(&session.status.as_str()) + { + return Err(MegaError::Buck(BuckError::InvalidSessionStatus { + expected: format!( + "{} or {}", + session_status::MANIFEST_UPLOADED, + session_status::UPLOADING + ), + actual: session.status.clone(), + })); + } + + let pending = self + .storage + .buck_storage() + .count_pending_files(cl_link) + .await?; + if pending > 0 { + return Err(MegaError::Buck(BuckError::FilesNotFullyUploaded { + missing_count: pending as u32, + })); + } + + let all_files = self.storage.buck_storage().get_all_files(cl_link).await?; + for file in &all_files { + if file.blob_id.is_none() { + return Err(MegaError::Buck(BuckError::ValidationError(format!( + "Missing blob_id for file: {} (status: {})", + file.file_path, file.upload_status + )))); + } + } + + // Build commit + let file_changes: Vec = all_files + .iter() + .filter(|f| f.upload_status == upload_status::UPLOADED) + .map(|f| { + let blob_id = f.blob_id.as_ref().unwrap(); + let normalized_blob_id = + format!("sha1:{}", blob_id.strip_prefix("sha1:").unwrap_or(blob_id)); + FileChange::new( + f.file_path.clone(), + normalized_blob_id, + f.file_mode + .clone() + .unwrap_or_else(|| DEFAULT_MODE.to_string()), + ) + }) + .collect(); + + // Use commit_message from session + let commit_message = session + .commit_message + .clone() + .unwrap_or_else(|| "Upload via buck push".to_string()); + + let commit_result = if file_changes.is_empty() { + None + } else { + let builder = BuckCommitBuilder::new(self.storage.mono_storage()); + let result = builder + .build_commit( + session.from_hash.as_deref().unwrap_or_default(), + &file_changes, + &commit_message, + ) + .await?; + Some(result) + }; + + // Convert to artifacts acceptable by BuckService + let artifacts = commit_result.map(|res| { + let commit_model: callisto::mega_commit::ActiveModel = res + .commit + .clone() + .into_mega_model(git_internal::internal::metadata::EntryMeta::default()) + .into(); + let new_tree_models: Vec = + res.new_tree_models.into_iter().map(|m| m.into()).collect(); + CommitArtifacts { + commit_id: res.commit_id, + tree_hash: res.tree_hash, + new_tree_models, + commit_model, + } + }); + + let svc_resp: SvcCompleteResponse = self + .storage + .buck_service + .complete_upload(username, cl_link, SvcCompletePayload {}, artifacts) + .await?; + + // Calculate uploaded files count + let uploaded_files_count = file_changes.len() as u32; + + let response = CompleteResponse { + cl_id: session.id, + cl_link: session.session_id.clone(), + commit_id: svc_resp.commit_id, + files_count: uploaded_files_count, + created_at: session.created_at.to_string(), + repo_path: session.repo_path.clone(), + from_hash: session.from_hash.clone().unwrap_or_default(), + }; + + self.trigger_build_for_buck_upload(&response, username); + + Ok(response) + } +} diff --git a/ceres/src/application/api_service/mono/cl/branch.rs b/ceres/src/application/api_service/mono/cl/branch.rs new file mode 100644 index 000000000..4e69b4506 --- /dev/null +++ b/ceres/src/application/api_service/mono/cl/branch.rs @@ -0,0 +1,640 @@ +//! Branch update / rebase operations for [`MonoApiService`](super::service::MonoApiService). + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + str::FromStr, +}; + +use callisto::{ + mega_cl, + sea_orm_active_enums::{ConvTypeEnum, MergeStatusEnum}, +}; +use common::{errors::MegaError, utils::ZERO_ID}; +use git_internal::{ + errors::GitError, + hash::ObjectHash, + internal::object::tree::{Tree, TreeItem, TreeItemMode}, +}; +use jupiter::utils::converter::FromMegaModel; +use tracing::debug; + +use crate::{ + api_service::mono::{ + MonoApiService, + types::{ApplyChangeContext, RefUpdate, TreeUpdateResult}, + }, + code_edit::utils as edit_utils, + model::change_list::{ClDiffFile, UpdateBranchStatusRes}, +}; + +impl MonoApiService { + pub(crate) async fn apply_changes_as_single_commit( + &self, + cl: &mega_cl::Model, + changes: &[ClDiffFile], + target_head: &str, + ) -> Result { + let mono_storage = self.storage.mono_storage(); + + // Load base commit and its root tree + let base_commit = mono_storage + .get_commit_by_hash(target_head) + .await? + .ok_or_else(|| GitError::CustomError(format!("Commit not found: {target_head}")))?; + + let base_tree_model = mono_storage + .get_tree_by_hash(&base_commit.tree) + .await? + .ok_or_else(|| GitError::CustomError("Root tree not found".to_string()))?; + let mut root_tree = Tree::from_mega_model(base_tree_model); + + // Cache trees by path to reuse updated versions + let mut tree_cache: HashMap = HashMap::new(); + tree_cache.insert(PathBuf::from("/"), root_tree.clone()); + + // Collect all new trees we generate (dedup by hash) + let mut new_trees: HashMap = HashMap::new(); + + for diff in changes { + let operations: Vec<(PathBuf, Option)> = match diff { + ClDiffFile::New(path, new_hash) => vec![(path.clone(), Some(*new_hash))], + ClDiffFile::Modified(path, _old, new_hash) => { + vec![(path.clone(), Some(*new_hash))] + } + ClDiffFile::Deleted(path, _old) => vec![(path.clone(), None)], + ClDiffFile::Renamed(old_path, new_path, _old_hash, new_hash, _similarity) + | ClDiffFile::Moved(old_path, new_path, _old_hash, new_hash, _similarity) => { + vec![ + (old_path.clone(), None), + (new_path.clone(), Some(*new_hash)), + ] + } + }; + + for (file_path, op) in operations { + // Reject absolute or parent-traversing paths to avoid writing outside repo root. + if file_path.is_absolute() + || file_path + .components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return Err(GitError::CustomError(format!( + "Invalid path (traversal/absolute) in CL diff: {:?}", + file_path + ))); + } + + let file_name = file_path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| GitError::CustomError("Invalid file name".to_string()))?; + // Normalize root parent to "/". + let parent_path = match file_path.parent() { + Some(p) if !p.as_os_str().is_empty() => p, + _ => Path::new("/"), + }; + + // Build chain of trees from root to parent, using updated cache when available + let components: Vec = parent_path + .components() + .filter_map(|c| match c { + std::path::Component::RootDir => None, + other => other.as_os_str().to_str().map(|s| s.to_string()), + }) + .collect(); + + let mut chain_paths: Vec = vec![PathBuf::from("/")]; + let mut chain_trees: Vec = vec![ + tree_cache + .get(&PathBuf::from("/")) + .cloned() + .ok_or_else(|| { + GitError::CustomError("Root tree cache missing".to_string()) + })?, + ]; + + let mut cursor = PathBuf::from("/"); + let mut missing_components: Option> = None; + for (idx, comp) in components.iter().enumerate() { + let parent_tree = chain_trees + .last() + .ok_or_else(|| GitError::CustomError("Empty tree chain".to_string()))?; + + let maybe_child = parent_tree.tree_items.iter().find(|it| it.name == *comp); + let child_tree = if let Some(child_item) = maybe_child { + if child_item.mode != TreeItemMode::Tree { + return Err(GitError::CustomError(format!( + "Type conflict: '{}' is not a directory", + comp + ))); + } + cursor = cursor.join(comp); + let child_hash = child_item.id; + if let Some(cached) = tree_cache.get(&cursor) { + cached.clone() + } else { + let model = mono_storage + .get_tree_by_hash(&child_hash.to_string()) + .await? + .ok_or_else(|| { + GitError::CustomError(format!( + "Tree not found for path '{}' with hash {}", + cursor.to_string_lossy(), + child_hash + )) + })?; + Tree::from_mega_model(model) + } + } else { + missing_components = Some(components[idx..].to_vec()); + break; + }; + + chain_paths.push(cursor.clone()); + chain_trees.push(child_tree); + } + + if let Some(missing) = missing_components { + let mut ctx = ApplyChangeContext { + components: &components, + chain_paths: &chain_paths, + chain_trees: &chain_trees, + tree_cache: &mut tree_cache, + new_trees: &mut new_trees, + }; + if let Some(updated_root) = + Self::apply_missing_path_update(&cl.link, missing, op, file_name, &mut ctx)? + { + root_tree = updated_root; + } + continue; + } + + let parent_dir_abs = cursor.clone(); + + // Update parent tree with the file change + let parent_tree = chain_trees + .pop() + .ok_or_else(|| GitError::CustomError("Parent tree missing".to_string()))?; + chain_paths.pop(); + + let mut items = parent_tree.tree_items.clone(); + match op { + Some(new_hash) => { + if let Some(idx) = items.iter().position(|it| it.name == file_name) { + items[idx].id = new_hash; + } else { + items.push(TreeItem::new( + TreeItemMode::Blob, + new_hash, + file_name.to_string(), + )); + } + } + None => { + items.retain(|it| it.name != file_name); + } + } + + let updated_tree = Tree::from_tree_items(items) + .map_err(|e| GitError::CustomError(e.to_string()))?; + // If parent tree id did not change (no-op), skip propagation for this diff. + if updated_tree.id == parent_tree.id { + // keep cache consistent even for no-ops + tree_cache.insert(parent_dir_abs.clone(), parent_tree.clone()); + debug!( + cl_link = %cl.link, + parent_dir = %parent_dir_abs.to_string_lossy(), + "apply_changes: no-op diff skipped" + ); + continue; + } + Self::record_tree( + parent_dir_abs, + &updated_tree, + &mut tree_cache, + &mut new_trees, + ); + + // Propagate updated hashes up to root + root_tree = Self::propagate_up( + &cl.link, + updated_tree, + &components, + &chain_paths, + &chain_trees, + &mut tree_cache, + &mut new_trees, + )?; + } + } + + let result = TreeUpdateResult { + updated_trees: new_trees.values().cloned().collect(), + ref_updates: vec![RefUpdate { + path: cl.path.clone(), + tree_id: root_tree.id, + }], + }; + + self.apply_update_result_cl_only( + &result, + "update-branch: rebase", + &cl.link, + Some(ObjectHash::from_str(target_head).map_err(|e| { + GitError::CustomError(format!( + "Invalid target_head ObjectHash '{}': {}", + target_head, e + )) + })?), + ) + .await + } + + fn apply_missing_path_update( + cl_link: &str, + missing: Vec, + op: Option, + file_name: &str, + ctx: &mut ApplyChangeContext<'_>, + ) -> Result, GitError> { + debug_assert!( + !missing.iter().any(|c| c == file_name), + "missing path components should not include file name" + ); + if op.is_none() { + debug!( + cl_link, + missing_path = %missing.join("/"), + "apply_changes: delete on missing path (no-op)" + ); + return Ok(None); + } + + let new_hash = op.ok_or_else(|| { + GitError::CustomError("Missing blob hash for new/modified file".to_string()) + })?; + + if missing.is_empty() { + // No missing directories: update directly under the last existing parent. + let parent_path = ctx.chain_paths.last().cloned().unwrap_or_else(PathBuf::new); + let parent_tree = ctx + .chain_trees + .last() + .cloned() + .ok_or_else(|| GitError::CustomError("Root tree missing".to_string()))?; + let updated_tree = Self::update_parent_tree( + cl_link, + &parent_tree, + file_name, + TreeItemMode::Blob, + new_hash, + None, + )?; + Self::record_tree(parent_path, &updated_tree, ctx.tree_cache, ctx.new_trees); + + return Ok(Some(Self::propagate_up( + cl_link, + updated_tree, + ctx.components, + ctx.chain_paths, + ctx.chain_trees, + ctx.tree_cache, + ctx.new_trees, + )?)); + } + + // Build missing subtree from leaf (parent dir) upward without empty trees. + let file_item = TreeItem::new(TreeItemMode::Blob, new_hash, file_name.to_string()); + let mut updated_tree = Tree::from_tree_items(vec![file_item]) + .map_err(|e| GitError::CustomError(e.to_string()))?; + + let mut missing_paths: Vec = Vec::new(); + let mut base = ctx.chain_paths.last().cloned().unwrap_or_else(PathBuf::new); + for comp in &missing { + base = base.join(comp); + missing_paths.push(base.clone()); + } + + if let Some(parent_path) = missing_paths.last() { + Self::record_tree( + parent_path.clone(), + &updated_tree, + ctx.tree_cache, + ctx.new_trees, + ); + } else { + Self::record_tree(PathBuf::new(), &updated_tree, ctx.tree_cache, ctx.new_trees); + } + + for (child_name, path) in missing + .iter() + .rev() + .skip(1) + .zip(missing_paths.iter().rev().skip(1)) + { + let wrapper = Tree::from_tree_items(vec![TreeItem::new( + TreeItemMode::Tree, + updated_tree.id, + child_name.clone(), + )]) + .map_err(|e| GitError::CustomError(e.to_string()))?; + updated_tree = wrapper; + Self::record_tree(path.clone(), &updated_tree, ctx.tree_cache, ctx.new_trees); + } + + // Attach the newly built subtree to the last existing parent. + let parent_tree = ctx + .chain_trees + .last() + .cloned() + .ok_or_else(|| GitError::CustomError("Root tree missing".to_string()))?; + let attach_name = missing + .first() + .ok_or_else(|| GitError::CustomError("Missing component chain empty".to_string()))?; + updated_tree = Self::update_parent_tree( + cl_link, + &parent_tree, + attach_name, + TreeItemMode::Tree, + updated_tree.id, + None, + )?; + let parent_path = ctx.chain_paths.last().cloned().unwrap_or_else(PathBuf::new); + Self::record_tree(parent_path, &updated_tree, ctx.tree_cache, ctx.new_trees); + + Ok(Some(Self::propagate_up( + cl_link, + updated_tree, + ctx.components, + ctx.chain_paths, + ctx.chain_trees, + ctx.tree_cache, + ctx.new_trees, + )?)) + } + + fn update_parent_tree( + cl_link: &str, + parent_tree: &Tree, + name: &str, + mode: TreeItemMode, + id: ObjectHash, + debug_parent_path: Option<&PathBuf>, + ) -> Result { + let mut parent_items = parent_tree.tree_items.clone(); + if let Some(pos) = parent_items.iter().position(|it| it.name == name) { + parent_items[pos].id = id; + } else { + parent_items.push(TreeItem::new(mode, id, name.to_string())); + parent_items.sort_by(|a, b| a.name.cmp(&b.name)); + if let Some(path) = debug_parent_path { + debug!( + cl_link, + parent_path = %path.to_string_lossy(), + created_entry = %name, + "apply_changes: inserted missing parent entry" + ); + } + } + + Tree::from_tree_items(parent_items).map_err(|e| GitError::CustomError(e.to_string())) + } + + fn record_tree( + path: PathBuf, + tree: &Tree, + tree_cache: &mut HashMap, + new_trees: &mut HashMap, + ) { + tree_cache.insert(path, tree.clone()); + new_trees.insert(tree.id, tree.clone()); + } + + fn propagate_up( + cl_link: &str, + mut updated_tree: Tree, + components: &[String], + chain_paths: &[PathBuf], + chain_trees: &[Tree], + tree_cache: &mut HashMap, + new_trees: &mut HashMap, + ) -> Result { + debug_assert!( + components.len() >= chain_trees.len().saturating_sub(1), + "components length must cover parent chain" + ); + + for parent_index in (0..chain_trees.len().saturating_sub(1)).rev() { + let comp = components + .get(parent_index) + .ok_or_else(|| GitError::CustomError("Tree path chain underflow".to_string()))?; + + let parent_tree = Self::update_parent_tree( + cl_link, + &chain_trees[parent_index], + comp, + TreeItemMode::Tree, + updated_tree.id, + chain_paths.get(parent_index), + )?; + + let parent_path_idx = chain_paths + .get(parent_index) + .cloned() + .ok_or_else(|| GitError::CustomError("Tree path chain underflow".to_string()))?; + Self::record_tree(parent_path_idx, &parent_tree, tree_cache, new_trees); + updated_tree = parent_tree; + } + + Ok(updated_tree) + } + /// Return Update Branch status for a CL: only checks whether main/trunk moved past the CL base. + pub async fn update_branch_status( + &self, + cl_link: &str, + ) -> Result { + let stg = self.storage.cl_storage(); + let cl = stg + .get_cl(cl_link) + .await? + .ok_or_else(|| MegaError::Other("CL Not Found".to_string()))?; + + let main_ref = match self.storage.mono_storage().get_main_ref(&cl.path).await? { + Some(r) => r, + None if crate::api_service::mono::cl_merge::path_lacks_main_ref(self, &cl.path) + .await? => + { + return Ok(UpdateBranchStatusRes { + base_commit: cl.from_hash.clone(), + target_head: ZERO_ID.to_string(), + outdated: false, + }); + } + None => return Err(MegaError::Other("Main ref not found".to_string())), + }; + let target_head = main_ref.ref_commit_hash; + + Ok(UpdateBranchStatusRes { + base_commit: cl.from_hash.clone(), + target_head: target_head.clone(), + outdated: cl.from_hash != target_head, + }) + } + + /// Update Branch (rebase-like) for Open CL: applies CL file changes onto latest target head + /// and updates CL's base/head commits. Returns new head commit id on success. + pub async fn update_branch(&self, username: &str, cl_link: &str) -> Result { + let stg = self.storage.cl_storage(); + let conv_stg = self.storage.conversation_storage(); + + let cl = stg + .get_cl(cl_link) + .await + .map_err(|e| GitError::CustomError(e.to_string()))? + .ok_or_else(|| GitError::CustomError("CL Not Found".to_string()))?; + + if cl.status != MergeStatusEnum::Open { + return Err(GitError::CustomError( + "Only Open CL can update branch".to_string(), + )); + } + + let main_ref = self + .storage + .mono_storage() + .get_main_ref(&cl.path) + .await + .map_err(|e| GitError::CustomError(e.to_string()))? + .ok_or_else(|| GitError::CustomError("Main ref not found".to_string()))?; + let target_head = main_ref.ref_commit_hash; + + if target_head == cl.from_hash { + return Ok("Already up-to-date".to_string()); + } + + // Detect file-level conflicts + let conflicts = self.detect_update_conflicts(&cl, &target_head).await?; + + if !conflicts.is_empty() { + // Record conflict info on the CL conversation for visibility. + let conflict_msg = format!( + "{} failed to update branch: conflicts on {}", + username, + conflicts.join(", ") + ); + if let Err(e) = conv_stg + .add_conversation(cl_link, username, Some(conflict_msg), ConvTypeEnum::Comment) + .await + { + tracing::warn!("Failed to add conflict comment to conversation: {}", e); + } + return Err(GitError::CustomError(format!( + "Update conflict on files: {}", + conflicts.join(", ") + ))); + } + + // Apply CL diffs onto latest target head + let old_blobs = self + .get_commit_blobs(&cl.from_hash) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + let new_blobs = self + .get_commit_blobs(&cl.to_hash) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + let cl_changed = self + .cl_files_list(old_blobs.clone(), new_blobs.clone()) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + if cl_changed.is_empty() { + // No-op rebase: just advance base hash and log. + stg.update_cl_hash(cl.clone(), &target_head, &cl.to_hash) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + conv_stg + .add_conversation( + cl_link, + username, + Some(format!( + "{} updated branch (no changes) to {}", + username, + &target_head[..6] + )), + ConvTypeEnum::Comment, + ) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + return Ok(cl.to_hash); + } + + // Apply all changes in-memory atop target_head and emit a single commit for the CL ref. + let new_head = self + .apply_changes_as_single_commit(&cl, &cl_changed, &target_head) + .await?; + + // Update cl hashes and log + stg.update_cl_hash(cl.clone(), &target_head, &new_head) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + conv_stg + .add_conversation( + cl_link, + username, + Some(format!( + "{} updated branch to {}", + username, + &target_head[..6] + )), + ConvTypeEnum::Comment, + ) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + Ok(new_head) + } + + /// Detect file-level update conflicts between the CL changes and target head. + /// A conflict is reported if any file path modified by the CL is also changed + /// between `from_hash` and `target_head`. + pub(crate) async fn detect_update_conflicts( + &self, + cl: &mega_cl::Model, + target_head: &str, + ) -> Result, GitError> { + let old_blobs = self + .get_commit_blobs(&cl.from_hash) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + let new_blobs = self + .get_commit_blobs(&cl.to_hash) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + // Keep conflict checks path-based so renames cover both old and new paths. + let cl_changed = edit_utils::cl_files_list(old_blobs.clone(), new_blobs.clone()) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + let target_blobs = self + .get_commit_blobs(target_head) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + let base_vs_target = edit_utils::cl_files_list(old_blobs.clone(), target_blobs.clone()) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + let cl_paths: std::collections::HashSet = cl_changed + .iter() + .map(|f| f.path().to_string_lossy().replace('\\', "/")) + .collect(); + let target_paths: std::collections::HashSet = base_vs_target + .iter() + .map(|f| f.path().to_string_lossy().replace('\\', "/")) + .collect(); + + Ok(cl_paths.intersection(&target_paths).cloned().collect()) + } +} diff --git a/ceres/src/application/api_service/mono/cl/diff.rs b/ceres/src/application/api_service/mono/cl/diff.rs new file mode 100644 index 000000000..800d8c26d --- /dev/null +++ b/ceres/src/application/api_service/mono/cl/diff.rs @@ -0,0 +1,989 @@ +//! Diff and patch operations for [`MonoApiService`](super::service::MonoApiService). + +use std::{ + collections::{HashMap, HashSet}, + path::{Path, PathBuf}, +}; + +use api_model::common::Pagination; +use common::errors::MegaError; +use git_internal::{ + DiffItem, diff::Diff as GitDiff, errors::GitError, hash::ObjectHash, + internal::object::tree::Tree, +}; +use jupiter::utils::converter::FromMegaModel; + +use crate::{ + api_service::{ApiHandler, mono::MonoApiService}, + diff::tree_diff, + model::change_list::{ClDiffFile, ClFilesChangedItemSchema}, +}; + +const LARGE_CL_RENAME_DETECTION_THRESHOLD: usize = 1000; + +struct PatchSections<'a> { + header_lines: Vec<&'a str>, + divider_line: Option<&'a str>, + payload_lines: Vec<&'a str>, + has_trailing_newline: bool, +} + +struct PagedClDiffItem { + item: DiffItem, + old_path: Option, +} + +impl MonoApiService { + /// Fetches the content difference for a merge request, paginated by page_id and page_size. + /// # Arguments + /// * `cl_link` - The link to the merge request. + /// * `page_id` - The page number to fetch. (id out of bounds will return empty) + /// * `page_size` - The number of items per page. + /// # Returns + /// a `Result` containing `ClDiff` on success or a `GitError` on failure. + /// Build paged CL diff items with optional relocation metadata for CL views. + async fn paged_content_diff_items( + &self, + cl_link: &str, + page: Pagination, + ) -> Result<(Vec, u64), GitError> { + let per_page = page.per_page as usize; + let page_id = page.page as usize; + + let stg = self.storage.cl_storage(); + let cl = + stg.get_cl(cl_link).await.unwrap().ok_or_else(|| { + GitError::CustomError(format!("Merge request not found: {cl_link}")) + })?; + let old_blobs = self + .get_commit_blobs(&cl.from_hash) + .await + .map_err(|e| GitError::CustomError(format!("Failed to get old commit blobs: {e}")))?; + let new_blobs = self + .get_commit_blobs(&cl.to_hash) + .await + .map_err(|e| GitError::CustomError(format!("Failed to get new commit blobs: {e}")))?; + + let sorted_changed_files = self + .cl_files_list(old_blobs, new_blobs) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + let start = (page_id.saturating_sub(1)) * per_page; + let end = (start + per_page).min(sorted_changed_files.len()); + + let page_slice: &[ClDiffFile] = if start < sorted_changed_files.len() { + let start_idx = start; + let end_idx = end; + &sorted_changed_files[start_idx..end_idx] + } else { + &[] + }; + + let non_relocated_items: Vec = page_slice + .iter() + .filter(|item| { + !matches!( + item, + ClDiffFile::Renamed(_, _, _, _, _) | ClDiffFile::Moved(_, _, _, _, _) + ) + }) + .cloned() + .collect(); + + let mut page_old_blobs = Vec::new(); + let mut page_new_blobs = Vec::new(); + collect_page_blobs( + &non_relocated_items, + &mut page_old_blobs, + &mut page_new_blobs, + ); + + let raw_diff_output = if non_relocated_items.is_empty() { + Vec::new() + } else { + self.get_diff_by_blobs(page_old_blobs, page_new_blobs) + .await? + }; + + let mut raw_diff_by_path: HashMap> = HashMap::new(); + for item in raw_diff_output { + raw_diff_by_path + .entry(item.path.clone()) + .or_default() + .push(item); + } + + let mut diff_output: Vec = Vec::with_capacity(page_slice.len()); + for item in page_slice { + match item { + ClDiffFile::Renamed(old_path, new_path, old_hash, new_hash, similarity) + | ClDiffFile::Moved(old_path, new_path, old_hash, new_hash, similarity) => { + diff_output.push(PagedClDiffItem { + item: self + .format_relocated_diff_item( + old_path, + new_path, + *old_hash, + *new_hash, + *similarity, + ) + .await?, + old_path: Some(old_path.to_string_lossy().replace('\\', "/")), + }); + } + _ => { + let key = item.path().to_string_lossy().replace('\\', "/"); + if let Some(items) = raw_diff_by_path.get_mut(&key) + && !items.is_empty() + { + diff_output.push(PagedClDiffItem { + item: items.remove(0), + old_path: None, + }); + } + } + } + } + + let total = sorted_changed_files.len().div_ceil(per_page); + + Ok((diff_output, total as u64)) + } + + /// Return the legacy paged diff shape without CL-specific metadata. + pub async fn paged_content_diff( + &self, + cl_link: &str, + page: Pagination, + ) -> Result<(Vec, u64), GitError> { + let (items, total) = self.paged_content_diff_items(cl_link, page).await?; + Ok((items.into_iter().map(|item| item.item).collect(), total)) + } + + /// Return paged diff items tailored for the CL files-changed API. + pub async fn paged_content_diff_for_cl( + &self, + cl_link: &str, + page: Pagination, + ) -> Result<(Vec, u64), GitError> { + let (items, total) = self.paged_content_diff_items(cl_link, page).await?; + Ok(( + items + .into_iter() + .map(|item| ClFilesChangedItemSchema::new(item.item, item.old_path)) + .collect(), + total, + )) + } + + async fn get_diff_by_blobs( + &self, + old_blobs: Vec<(PathBuf, ObjectHash)>, + new_blobs: Vec<(PathBuf, ObjectHash)>, + ) -> Result, GitError> { + let mut blob_cache: HashMap> = HashMap::new(); + + // Collect all unique hashes + let mut all_hashes = HashSet::new(); + for (_, hash) in &old_blobs { + all_hashes.insert(*hash); + } + for (_, hash) in &new_blobs { + all_hashes.insert(*hash); + } + + // Fetch all blobs with better error handling and logging + let mut failed_hashes = Vec::new(); + for hash in all_hashes { + match self.get_raw_blob_by_hash(&hash.to_string()).await { + Ok(data) => { + blob_cache.insert(hash, data); + } + Err(e) => { + tracing::error!("Failed to fetch blob {}: {}", hash, e); + failed_hashes.push(hash); + blob_cache.insert(hash, Vec::new()); + } + } + } + + if !failed_hashes.is_empty() { + tracing::warn!( + "Failed to fetch {} blob(s): {:?}", + failed_hashes.len(), + failed_hashes + ); + } + + // Enhanced content reader with better error handling + let read_content = |file: &PathBuf, hash: &ObjectHash| -> Vec { + match blob_cache.get(hash) { + Some(content) => content.clone(), + None => { + tracing::warn!("Missing blob content for file: {:?}, hash: {}", file, hash); + Vec::new() + } + } + }; + + // Use the unified diff function with configurable algorithm + let diff_output = GitDiff::diff(old_blobs, new_blobs, Vec::new(), read_content) + .into_iter() + .map(Self::normalize_diff_item) + .collect(); + + Ok(diff_output) + } + + async fn format_relocated_diff_item( + &self, + old_path: &Path, + new_path: &Path, + old_hash: ObjectHash, + new_hash: ObjectHash, + similarity: u8, + ) -> Result { + let mut patch = Self::format_relocated_patch_header(old_path, new_path, similarity); + + if old_hash != new_hash { + let raw_items = self + .get_diff_by_blobs( + vec![(old_path.to_path_buf(), old_hash)], + vec![(old_path.to_path_buf(), new_hash)], + ) + .await?; + if let Some(item) = raw_items.into_iter().next() { + patch.push_str(&Self::relocate_patch_body(&item.data, old_path, new_path)); + } + } + + Ok(DiffItem { + path: new_path.to_string_lossy().replace('\\', "/"), + data: patch, + }) + } + + fn format_relocated_patch_header(old_path: &Path, new_path: &Path, similarity: u8) -> String { + let old_path = old_path.to_string_lossy().replace('\\', "/"); + let new_path = new_path.to_string_lossy().replace('\\', "/"); + format!( + "diff --git a/{old_path} b/{new_path}\nsimilarity index {similarity}%\nrename from {old_path}\nrename to {new_path}\n" + ) + } + + pub(crate) fn normalize_diff_item(mut item: DiffItem) -> DiffItem { + item.path = item.path.replace('\\', "/"); + item.data = Self::normalize_patch_header_paths(&item.data); + item + } + + pub(crate) fn normalize_patch_header_paths(raw_patch: &str) -> String { + let sections = Self::split_patch_sections(raw_patch); + let header_lines = sections + .header_lines + .into_iter() + .map(Self::normalize_patch_header_line) + .collect(); + let divider_line = sections.divider_line.map(|line| { + if line.starts_with("Binary files ") { + line.replace('\\', "/") + } else { + line.to_string() + } + }); + + Self::join_patch_sections( + header_lines, + divider_line, + sections.payload_lines, + sections.has_trailing_newline, + ) + } + + pub(crate) fn relocate_patch_body(raw_patch: &str, old_path: &Path, new_path: &Path) -> String { + let old_path = old_path.to_string_lossy().replace('\\', "/"); + let new_path = new_path.to_string_lossy().replace('\\', "/"); + let sections = Self::split_patch_sections(raw_patch); + let header_lines = sections + .header_lines + .into_iter() + .filter(|line| !line.starts_with("diff --git ")) + .map(|line| { + if line.starts_with("--- a/") { + format!("--- a/{old_path}") + } else if line.starts_with("+++ b/") { + format!("+++ b/{new_path}") + } else { + line.to_string() + } + }) + .collect(); + let divider_line = sections.divider_line.map(|line| { + if line.starts_with("Binary files ") { + format!("Binary files a/{old_path} and b/{new_path} differ") + } else { + line.to_string() + } + }); + + Self::join_patch_sections( + header_lines, + divider_line, + sections.payload_lines, + sections.has_trailing_newline, + ) + } + + fn normalize_patch_header_line(line: &str) -> String { + if line.starts_with("diff --git ") + || line.starts_with("--- ") + || line.starts_with("+++ ") + || line.starts_with("rename from ") + || line.starts_with("rename to ") + { + line.replace('\\', "/") + } else { + line.to_string() + } + } + + fn split_patch_sections(raw_patch: &str) -> PatchSections<'_> { + let mut header_lines = Vec::new(); + let mut divider_line = None; + let mut payload_lines = Vec::new(); + let mut in_payload = false; + + for line in raw_patch.lines() { + if in_payload { + payload_lines.push(line); + continue; + } + + if line.starts_with("@@") + || line.starts_with("Binary files ") + || line.starts_with("GIT binary patch") + { + divider_line = Some(line); + in_payload = true; + continue; + } + + header_lines.push(line); + } + + PatchSections { + header_lines, + divider_line, + payload_lines, + has_trailing_newline: raw_patch.ends_with('\n'), + } + } + + fn join_patch_sections( + header_lines: Vec, + divider_line: Option, + payload_lines: Vec<&str>, + has_trailing_newline: bool, + ) -> String { + let mut lines = header_lines; + if let Some(line) = divider_line { + lines.push(line); + } + lines.extend(payload_lines.into_iter().map(String::from)); + + let rendered = lines.join("\n"); + if rendered.is_empty() { + rendered + } else if has_trailing_newline { + format!("{rendered}\n") + } else { + rendered + } + } + + pub async fn get_sorted_changed_file_list( + &self, + cl_link: &str, + path: Option<&str>, + ) -> Result, MegaError> { + let normalized_prefix = path.map(|prefix| prefix.replace('\\', "/")); + let cl = self + .storage + .cl_storage() + .get_cl(cl_link) + .await + .unwrap() + .ok_or_else(|| MegaError::Other("Error getting ".to_string()))?; + + let old_files = self.get_commit_blobs(&cl.from_hash.clone()).await?; + let new_files = self.get_commit_blobs(&cl.to_hash.clone()).await?; + + // calculate pages + let sorted_changed_files = self.cl_files_list(old_files, new_files).await?; + let file_paths: Vec = sorted_changed_files + .iter() + .map(|f| f.path().to_string_lossy().replace('\\', "/")) + .filter(|file_path| { + if let Some(prefix) = &normalized_prefix { + file_path.starts_with(prefix) + } else { + true + } + }) + .collect(); + + Ok(file_paths) + } + pub async fn cl_files_list( + &self, + old_files: Vec<(PathBuf, ObjectHash)>, + new_files: Vec<(PathBuf, ObjectHash)>, + ) -> Result, MegaError> { + let base_diff = tree_diff::calculate_tree_diff_basic(old_files.clone(), new_files.clone())?; + let mut blob_cache: HashMap> = HashMap::new(); + let mut failed_hashes = Vec::new(); + let candidate_hashes: HashSet = base_diff + .iter() + .filter_map(|item| match item { + ClDiffFile::Deleted(_, hash) | ClDiffFile::New(_, hash) => Some(*hash), + _ => None, + }) + .collect(); + + if base_diff.len() > LARGE_CL_RENAME_DETECTION_THRESHOLD + || candidate_hashes.len() > LARGE_CL_RENAME_DETECTION_THRESHOLD + { + tracing::info!( + diff_files = base_diff.len(), + candidate_hashes = candidate_hashes.len(), + threshold = LARGE_CL_RENAME_DETECTION_THRESHOLD, + "Skipping rename detection for large CL diff and returning path-level results." + ); + return Ok(base_diff); + } + + for hash in candidate_hashes { + match self.get_raw_blob_by_hash(&hash.to_string()).await { + Ok(data) => { + blob_cache.insert(hash, data); + } + Err(err) => { + failed_hashes.push(hash); + tracing::warn!( + "rename detection skipped blob {} and will fall back to path-level diff: {}", + hash, + err + ); + } + } + } + + if !failed_hashes.is_empty() { + tracing::warn!( + "rename detection degraded for {} candidate blob(s)", + failed_hashes.len() + ); + } + + let rename_config = self.storage.config().monorepo.rename.clone(); + tree_diff::calculate_tree_diff_with_blobs(old_files, new_files, &rename_config, &blob_cache) + } + + pub async fn get_commit_blobs( + &self, + commit_hash: &str, + ) -> Result, MegaError> { + let mut res = vec![]; + let mono_storage = self.storage.mono_storage(); + let commit = mono_storage.get_commit_by_hash(commit_hash).await?; + if let Some(commit) = commit { + let tree = mono_storage.get_tree_by_hash(&commit.tree).await?; + if let Some(tree) = tree { + let tree: Tree = Tree::from_mega_model(tree); + res = self.traverse_tree(tree).await?; + } + } + Ok(res) + } +} + +pub(crate) fn collect_page_blobs( + items: &[ClDiffFile], + old_out: &mut Vec<(PathBuf, ObjectHash)>, + new_out: &mut Vec<(PathBuf, ObjectHash)>, +) { + old_out.reserve(items.len()); + new_out.reserve(items.len()); + + for item in items { + match item { + ClDiffFile::New(p, h_new) => { + new_out.push((p.clone(), *h_new)); + } + ClDiffFile::Deleted(p, h_old) => { + old_out.push((p.clone(), *h_old)); + } + ClDiffFile::Modified(p, h_old, h_new) => { + old_out.push((p.clone(), *h_old)); + new_out.push((p.clone(), *h_new)); + } + ClDiffFile::Renamed(_, _, _, _, _) | ClDiffFile::Moved(_, _, _, _, _) => { + // Relocated items are filtered out before this helper is called. + debug_assert!(false, "collect_page_blobs only accepts non-relocated items"); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + path::{Path, PathBuf}, + str::FromStr, + }; + + use git_internal::{DiffItem, hash::ObjectHash}; + + use super::collect_page_blobs; + use crate::{api_service::mono::MonoApiService, model::change_list::ClDiffFile}; + + #[test] + fn test_paging_calculation_basic() { + let files: Vec = vec![ + ClDiffFile::New( + PathBuf::from("file1.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + ), + ClDiffFile::Modified( + PathBuf::from("file2.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + ObjectHash::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(), + ), + ClDiffFile::Deleted( + PathBuf::from("file3.txt"), + ObjectHash::from_str("1111111111111111111111111111111111111111").unwrap(), + ), + ]; + + let page_size = 2u32; + let page_id = 1u32; + + let start = (page_id.saturating_sub(1)) * page_size; + let end = (start + page_size).min(files.len() as u32); + + assert_eq!(start, 0); + assert_eq!(end, 2); + + let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { + let start_idx = start as usize; + let end_idx = end as usize; + &files[start_idx..end_idx] + } else { + &[] + }; + + assert_eq!(page_slice.len(), 2); + } + + #[test] + fn test_paging_calculation_second_page() { + let files: Vec = vec![ + ClDiffFile::New( + PathBuf::from("file1.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + ), + ClDiffFile::Modified( + PathBuf::from("file2.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + ObjectHash::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(), + ), + ClDiffFile::Deleted( + PathBuf::from("file3.txt"), + ObjectHash::from_str("1111111111111111111111111111111111111111").unwrap(), + ), + ClDiffFile::New( + PathBuf::from("file4.txt"), + ObjectHash::from_str("2222222222222222222222222222222222222222").unwrap(), + ), + ]; + + let page_size = 2u32; + let page_id = 2u32; + + let start = (page_id.saturating_sub(1)) * page_size; + let end = (start + page_size).min(files.len() as u32); + + assert_eq!(start, 2); + assert_eq!(end, 4); + + let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { + let start_idx = start as usize; + let end_idx = end as usize; + &files[start_idx..end_idx] + } else { + &[] + }; + + assert_eq!(page_slice.len(), 2); + assert_eq!(page_slice[0].path(), &PathBuf::from("file3.txt")); + assert_eq!(page_slice[1].path(), &PathBuf::from("file4.txt")); + } + + #[test] + fn test_paging_calculation_partial_page() { + let files: Vec = vec![ + ClDiffFile::New( + PathBuf::from("file1.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + ), + ClDiffFile::Modified( + PathBuf::from("file2.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + ObjectHash::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(), + ), + ClDiffFile::Deleted( + PathBuf::from("file3.txt"), + ObjectHash::from_str("1111111111111111111111111111111111111111").unwrap(), + ), + ]; + + let page_size = 5u32; + let page_id = 1u32; + + let start = (page_id.saturating_sub(1)) * page_size; + let end = (start + page_size).min(files.len() as u32); + + assert_eq!(start, 0); + assert_eq!(end, 3); + + let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { + let start_idx = start as usize; + let end_idx = end as usize; + &files[start_idx..end_idx] + } else { + &[] + }; + + assert_eq!(page_slice.len(), 3); + } + + #[test] + fn test_paging_calculation_out_of_bounds() { + let files: Vec = vec![ClDiffFile::New( + PathBuf::from("file1.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + )]; + + let page_size = 2u32; + let page_id = 3u32; // Page that doesn't exist + + let start = (page_id.saturating_sub(1)) * page_size; + let end = (start + page_size).min(files.len() as u32); + + assert_eq!(start, 4); + assert_eq!(end, 1); // end is clamped to files.len() + + let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { + let start_idx = start as usize; + let end_idx = end as usize; + &files[start_idx..end_idx] + } else { + &[] + }; + + assert_eq!(page_slice.len(), 0); + } + + #[test] + fn test_paging_calculation_edge_case_zero_page_size() { + let files: Vec = vec![ClDiffFile::New( + PathBuf::from("file1.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + )]; + + let page_size = 0u32; + let page_id = 1u32; + + let start = (page_id.saturating_sub(1)) * page_size; + let end = (start + page_size).min(files.len() as u32); + + assert_eq!(start, 0); + assert_eq!(end, 0); + + let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { + let start_idx = start as usize; + let end_idx = end as usize; + &files[start_idx..end_idx] + } else { + &[] + }; + + assert_eq!(page_slice.len(), 0); + } + + #[test] + fn test_paging_calculation_zero_page_id() { + let files: Vec = vec![ + ClDiffFile::New( + PathBuf::from("file1.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + ), + ClDiffFile::Modified( + PathBuf::from("file2.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + ObjectHash::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(), + ), + ]; + + let page_size = 2u32; + let page_id = 0u32; // Should be treated as page 1 due to saturating_sub + + let start = (page_id.saturating_sub(1)) * page_size; + let end = (start + page_size).min(files.len() as u32); + + assert_eq!(start, 0); + assert_eq!(end, 2); + + let page_slice: &[ClDiffFile] = if (start as usize) < files.len() { + let start_idx = start as usize; + let end_idx = end as usize; + &files[start_idx..end_idx] + } else { + &[] + }; + + assert_eq!(page_slice.len(), 2); + } + + #[test] + fn test_paging_algorithm() { + let total_files = 10usize; + let current_page = 2u32; + let page_size = 3u32; + + let total_pages = total_files.div_ceil(page_size as usize); + let current_page = current_page as usize; + let page_size = page_size as usize; + + assert_eq!(total_pages, 4); + assert_eq!(current_page, 2); + assert_eq!(page_size, 3); + } + + #[test] + fn test_collect_page_blobs_new_files() { + let files = vec![ClDiffFile::New( + PathBuf::from("new_file.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + )]; + + let mut old_blobs = Vec::new(); + let mut new_blobs = Vec::new(); + + collect_page_blobs(&files, &mut old_blobs, &mut new_blobs); + + assert_eq!(old_blobs.len(), 0); + assert_eq!(new_blobs.len(), 1); + assert_eq!(new_blobs[0].0, PathBuf::from("new_file.txt")); + } + + #[test] + fn test_collect_page_blobs_deleted_files() { + let files = vec![ClDiffFile::Deleted( + PathBuf::from("deleted_file.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + )]; + + let mut old_blobs = Vec::new(); + let mut new_blobs = Vec::new(); + + collect_page_blobs(&files, &mut old_blobs, &mut new_blobs); + + assert_eq!(old_blobs.len(), 1); + assert_eq!(new_blobs.len(), 0); + assert_eq!(old_blobs[0].0, PathBuf::from("deleted_file.txt")); + } + + #[test] + fn test_file_lists_with_roots() { + let all_files = vec![ + "src/main.rs".to_string(), + "src/utils/math.rs".to_string(), + "src/utils/io.rs".to_string(), + "README.md".to_string(), + ]; + + let root: Option<&str> = None; + let filtered_none: Vec = all_files + .iter() + .filter(|file_path| { + if let Some(prefix) = root { + file_path.starts_with(prefix) + } else { + true + } + }) + .cloned() + .collect(); + + assert_eq!(filtered_none.len(), 4); + assert_eq!(filtered_none, all_files); + + let filtered_some: Vec = all_files + .iter() + .filter(|file_path| { + if let Some(prefix) = Some("src/utils") { + file_path.starts_with(prefix) + } else { + true + } + }) + .cloned() + .collect(); + + assert_eq!(filtered_some.len(), 2); + assert_eq!( + filtered_some, + vec![ + "src/utils/math.rs".to_string(), + "src/utils/io.rs".to_string() + ] + ); + } + + #[test] + fn test_collect_page_blobs_modified_files() { + let files = vec![ClDiffFile::Modified( + PathBuf::from("modified_file.txt"), + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + ObjectHash::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(), + )]; + + let mut old_blobs = Vec::new(); + let mut new_blobs = Vec::new(); + + collect_page_blobs(&files, &mut old_blobs, &mut new_blobs); + + assert_eq!(old_blobs.len(), 1); + assert_eq!(new_blobs.len(), 1); + assert_eq!(old_blobs[0].0, PathBuf::from("modified_file.txt")); + assert_eq!(new_blobs[0].0, PathBuf::from("modified_file.txt")); + } + + #[test] + fn test_collect_page_blobs_mixed_files() { + let files = vec![ + ClDiffFile::New( + PathBuf::from("new.txt"), + ObjectHash::from_str("1111111111111111111111111111111111111111").unwrap(), + ), + ClDiffFile::Deleted( + PathBuf::from("deleted.txt"), + ObjectHash::from_str("2222222222222222222222222222222222222222").unwrap(), + ), + ClDiffFile::Modified( + PathBuf::from("modified.txt"), + ObjectHash::from_str("3333333333333333333333333333333333333333").unwrap(), + ObjectHash::from_str("4444444444444444444444444444444444444444").unwrap(), + ), + ]; + + let mut old_blobs = Vec::new(); + let mut new_blobs = Vec::new(); + + collect_page_blobs(&files, &mut old_blobs, &mut new_blobs); + + assert_eq!(old_blobs.len(), 2); // deleted + modified + assert_eq!(new_blobs.len(), 2); // new + modified + + assert_eq!(old_blobs[0].0, PathBuf::from("deleted.txt")); + assert_eq!(old_blobs[1].0, PathBuf::from("modified.txt")); + assert_eq!(new_blobs[0].0, PathBuf::from("new.txt")); + assert_eq!(new_blobs[1].0, PathBuf::from("modified.txt")); + } + + #[test] + fn test_relocate_patch_body_rewrites_paths_and_keeps_hunk() { + let raw_patch = "\ +diff --git a/old/name.txt b/old/name.txt\n\ +index 1111111..2222222 100644\n\ +--- a/old/name.txt\n\ ++++ b/old/name.txt\n\ +@@ -1 +1 @@\n\ +-old line\n\ ++new line\n"; + + let relocated = MonoApiService::relocate_patch_body( + raw_patch, + Path::new("old/name.txt"), + Path::new("new/name.txt"), + ); + + assert!(!relocated.contains("diff --git")); + assert!(relocated.contains("--- a/old/name.txt")); + assert!(relocated.contains("+++ b/new/name.txt")); + assert!(relocated.contains("@@ -1 +1 @@")); + assert!(relocated.contains("-old line")); + assert!(relocated.contains("+new line")); + assert!(!relocated.contains("deleted file mode")); + } + + #[test] + fn test_relocate_patch_body_preserves_hunk_backslashes() { + let raw_patch = "\ +diff --git a/old/name.txt b/old/name.txt\n\ +index 1111111..2222222 100644\n\ +--- a/old/name.txt\n\ ++++ b/old/name.txt\n\ +@@ -1 +1 @@\n\ +-let path = \"C:\\\\temp\\\\old\";\n\ ++let path = \"C:\\\\temp\\\\new\";\n"; + + let relocated = MonoApiService::relocate_patch_body( + raw_patch, + Path::new("old/name.txt"), + Path::new("new/name.txt"), + ); + + assert!(relocated.contains("--- a/old/name.txt")); + assert!(relocated.contains("+++ b/new/name.txt")); + assert!(relocated.contains("-let path = \"C:\\\\temp\\\\old\";")); + assert!(relocated.contains("+let path = \"C:\\\\temp\\\\new\";")); + } + + #[test] + fn test_normalize_diff_item_path_uses_forward_slashes() { + let item = DiffItem { + path: "dir\\nested\\file.txt".to_string(), + data: "diff --git a/dir\\nested\\file.txt b/dir\\nested\\file.txt\n".to_string(), + }; + + let normalized = MonoApiService::normalize_diff_item(item); + assert_eq!(normalized.path, "dir/nested/file.txt"); + assert!( + normalized + .data + .contains("diff --git a/dir/nested/file.txt b/dir/nested/file.txt") + ); + } + + #[test] + fn test_normalize_patch_header_paths_preserves_hunk_content() { + let raw_patch = "\ +diff --git a/dir\\nested\\file.txt b/dir\\nested\\file.txt\n\ +--- a/dir\\nested\\file.txt\n\ ++++ b/dir\\nested\\file.txt\n\ +@@ -1 +1 @@\n\ +-let path = \"C:\\\\temp\\\\old\";\n\ ++let path = \"C:\\\\temp\\\\new\";\n"; + + let normalized = MonoApiService::normalize_patch_header_paths(raw_patch); + + assert!(normalized.contains("diff --git a/dir/nested/file.txt b/dir/nested/file.txt")); + assert!(normalized.contains("--- a/dir/nested/file.txt")); + assert!(normalized.contains("+++ b/dir/nested/file.txt")); + assert!(normalized.contains("-let path = \"C:\\\\temp\\\\old\";")); + assert!(normalized.contains("+let path = \"C:\\\\temp\\\\new\";")); + } +} diff --git a/ceres/src/application/api_service/mono/cl/lifecycle.rs b/ceres/src/application/api_service/mono/cl/lifecycle.rs new file mode 100644 index 000000000..c1d66ce6d --- /dev/null +++ b/ceres/src/application/api_service/mono/cl/lifecycle.rs @@ -0,0 +1,394 @@ +//! CL lifecycle orchestration (detail, status transitions, comments, merge box). + +use std::collections::HashSet; + +use callisto::sea_orm_active_enums::{ConvTypeEnum, MergeStatusEnum}; +use common::errors::MegaError; +use jupiter::model::cl_dto::CLDetails; + +use crate::{ + api_service::mono::MonoApiService, + application::webhook::{WebhookEvent, dispatch_cl_webhook}, + model::change_list::{CLDetailRes, Condition, MergeBoxRes, UpdateClStatusPayload}, +}; + +impl MonoApiService { + pub async fn get_cl_details( + &self, + link: &str, + username: String, + ) -> Result { + let cl_storage = self.storage.cl_storage(); + let conversation_storage = self.storage.conversation_storage(); + + let (cl, labels) = cl_storage + .get_cl_labels(link) + .await? + .ok_or_else(|| MegaError::Other("CL not found".to_string()))?; + + let conversations = conversation_storage + .get_comments_with_reactions(link) + .await?; + + let (_, assignees) = cl_storage + .get_cl_assignees(link) + .await? + .unwrap_or((cl.clone(), vec![])); + + Ok(CLDetails { + cl, + labels, + conversations, + assignees, + username, + } + .into()) + } + + pub async fn reopen_cl(&self, link: &str, username: &str) -> Result<(), MegaError> { + let cl_storage = self.storage.cl_storage(); + let model = cl_storage + .get_cl(link) + .await? + .ok_or(MegaError::Other("Not Found".to_string()))?; + + if model.status != MergeStatusEnum::Closed { + return Ok(()); + } + + let link = model.link.clone(); + cl_storage.reopen_cl(model.clone()).await?; + self.storage + .conversation_storage() + .add_conversation( + &link, + username, + Some(format!("{username} reopen this")), + ConvTypeEnum::Reopen, + ) + .await?; + + if let Some(updated_model) = cl_storage.get_cl(&link).await? { + dispatch_cl_webhook(&self.storage, WebhookEvent::ClReopened, &updated_model); + } + Ok(()) + } + + pub async fn close_cl(&self, link: &str, username: &str) -> Result<(), MegaError> { + let cl_storage = self.storage.cl_storage(); + let model = cl_storage + .get_cl(link) + .await? + .ok_or(MegaError::Other("Not Found".to_string()))?; + + if !matches!(model.status, MergeStatusEnum::Open | MergeStatusEnum::Draft) { + return Ok(()); + } + + let link = model.link.clone(); + cl_storage.close_cl(model.clone()).await?; + self.storage + .conversation_storage() + .add_conversation( + &link, + username, + Some(format!("{username} closed this")), + ConvTypeEnum::Closed, + ) + .await?; + + if let Some(updated_model) = cl_storage.get_cl(&link).await? { + dispatch_cl_webhook(&self.storage, WebhookEvent::ClClosed, &updated_model); + } + Ok(()) + } + + pub async fn merge_open_cl(&self, username: &str, link: &str) -> Result<(), MegaError> { + let cl_storage = self.storage.cl_storage(); + let model = cl_storage + .get_cl(link) + .await? + .ok_or(MegaError::Other("Not Found".to_string()))?; + + if model.status == MergeStatusEnum::Draft { + return Err(MegaError::Other("CL is not ready for review".to_owned())); + } + + if model.status == MergeStatusEnum::Open { + self.merge_cl(username, model.clone()).await?; + if let Some(updated_model) = cl_storage.get_cl(link).await? { + dispatch_cl_webhook(&self.storage, WebhookEvent::ClMerged, &updated_model); + } + } + Ok(()) + } + + pub async fn merge_open_cl_no_auth(&self, link: &str) -> Result<(), MegaError> { + let cl_storage = self.storage.cl_storage(); + let model = cl_storage + .get_cl(link) + .await? + .ok_or(MegaError::Other("CL Not Found".to_string()))?; + + if model.status != MergeStatusEnum::Open { + return Err(MegaError::Other(format!( + "CL is not in Open status, current status: {:?}", + model.status + ))); + } + + self.merge_cl("system", model.clone()).await?; + if let Some(updated_model) = cl_storage.get_cl(link).await? { + dispatch_cl_webhook(&self.storage, WebhookEvent::ClMerged, &updated_model); + } + Ok(()) + } + + pub async fn get_merge_box(&self, link: &str) -> Result { + let cl_storage = self.storage.cl_storage(); + let cl = cl_storage + .get_cl(link) + .await? + .ok_or(MegaError::Other("CL Not Found".to_string()))?; + + let res = match cl.status { + MergeStatusEnum::Open => { + let check_res: Vec = cl_storage + .get_check_result(link) + .await? + .into_iter() + .map(|m| m.into()) + .collect(); + MergeBoxRes::from_condition(check_res) + } + MergeStatusEnum::Draft | MergeStatusEnum::Merged | MergeStatusEnum::Closed => { + MergeBoxRes { + merge_requirements: None, + } + } + }; + Ok(res) + } + + pub async fn save_cl_comment( + &self, + link: &str, + username: &str, + content: &str, + ) -> Result<(), MegaError> { + let conv_type = if self + .storage + .reviewer_storage() + .is_reviewer(link, username) + .await? + { + ConvTypeEnum::Review + } else { + ConvTypeEnum::Comment + }; + + self.storage + .conversation_storage() + .add_conversation(link, username, Some(content.to_string()), conv_type) + .await?; + + if let Err(e) = enqueue_cl_comment_notifications(self, username, link, content).await { + tracing::warn!("failed to enqueue cl comment notifications: {e}"); + } + + if let Some(cl_model) = self.storage.cl_storage().get_cl(link).await? { + dispatch_cl_webhook(&self.storage, WebhookEvent::ClCommentCreated, &cl_model); + } + Ok(()) + } + + pub async fn edit_cl_title(&self, link: &str, content: &str) -> Result<(), MegaError> { + self.storage.cl_storage().edit_title(link, content).await?; + if let Some(cl_model) = self.storage.cl_storage().get_cl(link).await? { + dispatch_cl_webhook(&self.storage, WebhookEvent::ClUpdated, &cl_model); + } + Ok(()) + } + + pub async fn update_cl_status( + &self, + link: &str, + username: &str, + payload: &UpdateClStatusPayload, + ) -> Result<(), MegaError> { + let cl_storage = self.storage.cl_storage(); + let model = cl_storage + .get_cl(link) + .await? + .ok_or(MegaError::Other("Not Found".to_string()))?; + + let new_status = match payload.status.to_lowercase().as_str() { + "draft" => MergeStatusEnum::Draft, + "open" => MergeStatusEnum::Open, + _ => { + return Err(MegaError::Other( + "Invalid status. Only 'draft' and 'open' are supported".to_string(), + )); + } + }; + + match (&model.status, &new_status) { + (MergeStatusEnum::Draft, MergeStatusEnum::Open) => { + cl_storage + .update_cl_status(model.clone(), new_status.clone()) + .await?; + self.storage + .conversation_storage() + .add_conversation( + link, + username, + Some(format!("{username} marked this as ready for review")), + ConvTypeEnum::Review, + ) + .await?; + if let Some(updated_model) = cl_storage.get_cl(link).await? { + dispatch_cl_webhook(&self.storage, WebhookEvent::ClCreated, &updated_model); + } + } + (MergeStatusEnum::Open, MergeStatusEnum::Draft) => { + cl_storage + .update_cl_status(model.clone(), new_status.clone()) + .await?; + self.storage + .conversation_storage() + .add_conversation( + link, + username, + Some(format!("{username} marked this as draft")), + ConvTypeEnum::Draft, + ) + .await?; + if let Some(updated_model) = cl_storage.get_cl(link).await? { + dispatch_cl_webhook(&self.storage, WebhookEvent::ClUpdated, &updated_model); + } + } + _ => { + return Err(MegaError::Other( + "Invalid status transition. Only Draft ↔ Open is allowed".to_string(), + )); + } + } + Ok(()) + } + + pub async fn update_branch_with_webhook( + &self, + username: &str, + link: &str, + ) -> Result { + let new_head = self.update_branch(username, link).await?; + if let Some(cl_model) = self.storage.cl_storage().get_cl(link).await? { + dispatch_cl_webhook(&self.storage, WebhookEvent::ClUpdated, &cl_model); + } + Ok(new_head) + } +} + +const EVENT_CL_COMMENT_CREATED: &str = "cl.comment.created"; + +async fn enqueue_cl_comment_notifications( + service: &MonoApiService, + actor_username: &str, + cl_link: &str, + comment_text: &str, +) -> Result<(), MegaError> { + let notif_stg = service.storage.notification_storage(); + ensure_cl_comment_event_type(¬if_stg).await?; + + let cl_stg = service.storage.cl_storage(); + let cl = cl_stg + .get_cl(cl_link) + .await? + .ok_or_else(|| MegaError::NotFound(format!("CL {cl_link} not found")))?; + + let reviewers = service + .storage + .reviewer_storage() + .list_reviewers(cl_link) + .await?; + + let mut recipients: HashSet = HashSet::new(); + recipients.insert(cl.username); + for r in reviewers { + recipients.insert(r.username); + } + recipients.remove(actor_username); + + for username in recipients { + if !notif_stg + .should_send(&username, EVENT_CL_COMMENT_CREATED) + .await? + { + continue; + } + + let settings = match notif_stg.get_user_settings(&username).await? { + Some(s) => s, + None => continue, + }; + + let subject = format!("New comment on CL {cl_link}"); + let body_text = format!("{actor_username} commented on {cl_link}: {comment_text}"); + let body_html = format!( + "

{actor_username} commented on {cl_link}:

{}

", + escape_html(comment_text) + ); + + notif_stg + .enqueue_email_job( + &username, + &settings.email, + EVENT_CL_COMMENT_CREATED, + &subject, + &body_html, + Some(&body_text), + ) + .await?; + } + + Ok(()) +} + +async fn ensure_cl_comment_event_type( + notif_stg: &jupiter::storage::NotificationStorage, +) -> Result<(), MegaError> { + use callisto::notification_event_types; + use jupiter::sea_orm::{ActiveModelTrait, Set}; + + if notif_stg + .get_event_type(EVENT_CL_COMMENT_CREATED) + .await? + .is_some() + { + return Ok(()); + } + + let now = chrono::Utc::now().naive_utc(); + notification_event_types::ActiveModel { + code: Set(EVENT_CL_COMMENT_CREATED.to_owned()), + category: Set("cl".to_owned()), + description: Set("New comment on a Change List".to_owned()), + system_required: Set(false), + default_enabled: Set(true), + created_at: Set(now), + updated_at: Set(now), + } + .insert(notif_stg.db()) + .await?; + + Ok(()) +} + +fn escape_html(input: &str) -> String { + input + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} diff --git a/ceres/src/application/api_service/mono/cl/merge.rs b/ceres/src/application/api_service/mono/cl/merge.rs new file mode 100644 index 000000000..649435f18 --- /dev/null +++ b/ceres/src/application/api_service/mono/cl/merge.rs @@ -0,0 +1,377 @@ +//! CL merge operations for [`MonoApiService`](super::service::MonoApiService). + +use std::{path::PathBuf, sync::Arc}; + +use callisto::{mega_cl, mega_tree, sea_orm_active_enums::ConvTypeEnum}; +use common::{errors::MegaError, utils::MEGA_BRANCH_NAME}; +use git_internal::{ + errors::GitError, + hash::ObjectHash, + internal::{metadata::EntryMeta, object::commit::Commit}, +}; +use jupiter::{ + storage::{base_storage::StorageConnector, mono_storage::RefUpdateData}, + utils::converter::IntoMegaModel, +}; +use orion_client::OrionBuildClient; +use tracing::debug; + +use crate::{ + api_service::{ + ApiHandler, + mono::{MonoApiService, logic::MonoServiceLogic, types::TreeUpdateResult}, + }, + code_edit::on_edit::OneditCodeEdit, + merge_checker::CheckerRegistry, +}; + +impl MonoApiService { + // This function is intended to be called before merging a CL to ensure it meets all required checks. + pub(crate) async fn ensure_cl_mergeable(&self, cl: &mega_cl::Model) -> Result<(), MegaError> { + let check_reg = CheckerRegistry::new(self.storage.clone().into(), cl.username.clone()); + check_reg.run_checks(cl.clone().into()).await?; + + let required_check_types = self + .storage + .cl_storage() + .get_checks_config_by_path(&cl.path) + .await? + .into_iter() + .filter(|cfg| cfg.required) + .map(|cfg| cfg.check_type_code) + .collect::>(); + + let failed_checks = self + .storage + .cl_storage() + .get_check_result(&cl.link) + .await? + .into_iter() + .filter(|result| { + result.status == "FAILED" + && required_check_types + .iter() + .any(|required_type| required_type == &result.check_type_code) + }) + .map(|result| format!("{:?}", result.check_type_code)) + .collect::>(); + + if failed_checks.is_empty() { + Ok(()) + } else { + Err(MegaError::Other(format!( + "CL is unmergeable, failed checks: {}", + failed_checks.join(", ") + ))) + } + } + + pub(crate) async fn ensure_merge_no_file_conflicts( + &self, + cl: &mega_cl::Model, + ) -> Result<(), GitError> { + let main_ref = self + .storage + .mono_storage() + .get_main_ref(&cl.path) + .await + .map_err(|e| GitError::CustomError(e.to_string()))? + .ok_or_else(|| GitError::CustomError("Main ref not found".to_string()))?; + + let conflicts = self + .detect_update_conflicts(cl, &main_ref.ref_commit_hash) + .await?; + if conflicts.is_empty() { + Ok(()) + } else { + Err(GitError::CustomError(format!( + "Merge conflict on files: {}", + conflicts.join(", ") + ))) + } + } + + pub(crate) async fn trigger_build_for_cl( + &self, + editor: &OneditCodeEdit, + cl: &mega_cl::Model, + username: &str, + ) -> Result<(), GitError> { + let config = self.storage.config(); + let orion_client = OrionBuildClient::new(config.build.clone()); + let git_cache = self.git_object_cache.clone(); + editor + .trigger_build_and_check( + self.storage.clone(), + git_cache, + Arc::new(orion_client), + cl, + username, + ) + .await?; + + Ok(()) + } + pub async fn merge_cl(&self, username: &str, mut cl: mega_cl::Model) -> Result<(), GitError> { + crate::api_service::mono::cl_merge::prepare_cl_path_for_merge(self, &mut cl) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + self.ensure_cl_mergeable(&cl) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + let storage = self.storage.mono_storage(); + let refs = storage + .get_main_ref(&cl.path) + .await + .map_err(|e| GitError::CustomError(format!("Failed to get main ref: {}", e)))? + .ok_or_else(|| GitError::CustomError("Main ref not found".to_string()))?; + + if cl.from_hash != refs.ref_commit_hash { + return Err(GitError::CustomError("ref hash conflict".to_owned())); + } + + self.ensure_merge_no_file_conflicts(&cl).await?; + + self.merge_cl_unchecked(username, cl).await + } + + /// Apply all CL changes onto the target_head in-memory and emit a single commit on the CL ref. + /// Merges a CL without checking for conflicts. + /// Caller is responsible for ensuring no conflicts exist before calling this method. + pub(crate) async fn merge_cl_unchecked( + &self, + username: &str, + cl: mega_cl::Model, + ) -> Result<(), GitError> { + let storage = self.storage.mono_storage(); + + let strategy = crate::api_service::mono::cl_merge::resolve_merge_strategy(self, &cl) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + tracing::info!( + cl_link = %cl.link, + cl_path = %cl.path, + strategy = strategy.as_str(), + "Applying CL merge" + ); + + let normalized_path = MonoServiceLogic::clean_path_str(&cl.path); + let (path, update_chain) = if normalized_path == "/" { + (PathBuf::from("/"), Vec::new()) + } else { + let path = PathBuf::from(&normalized_path); + let parent = path.parent().ok_or_else(|| { + GitError::CustomError(format!("Invalid CL path: {}", normalized_path)) + })?; + if crate::api_service::mono::cl_merge::needs_path_tree_attach(self, &path) + .await + .map_err(|e| GitError::CustomError(e.to_string()))? + { + self.attach_project_path_to_monorepo_root(&normalized_path) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + crate::api_service::mono::cl_merge::sync_path_prefix_main_refs( + self, + &normalized_path, + ) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + } + let update_chain = self.search_tree_for_update(parent).await?; + (path, update_chain) + }; + + let leaf_tree_id = + crate::api_service::mono::cl_merge::resolve_merge_leaf_tree_id(self, &cl, strategy) + .await?; + let result = MonoServiceLogic::build_result_by_chain(path, update_chain, leaf_tree_id)?; + self.apply_update_result(&result, "cl merge generated commit", Some(cl.link.as_str())) + .await?; + + if normalized_path != "/" { + storage + .remove_none_cl_refs(&normalized_path) + .await + .map_err(|e| GitError::CustomError(format!("Failed to remove refs: {}", e)))?; + // TODO: self.clean_dangling_commits().await; + } + // add conversation + self.storage + .conversation_storage() + .add_conversation(&cl.link, username, None, ConvTypeEnum::Merged) + .await + .map_err(|e| GitError::CustomError(format!("Failed to add conversation: {}", e)))?; + // update cl status last + self.storage + .cl_storage() + .merge_cl(cl.clone()) + .await + .map_err(|e| GitError::CustomError(format!("Failed to update CL status: {}", e)))?; + + // Invalidate admin cache when .mega_cedar.json is modified. + if let Ok(files) = self.get_sorted_changed_file_list(&cl.link, None).await { + let admin_file_modified = files.iter().any(|file| { + let normalized = file.replace('\\', "/"); + normalized.ends_with(crate::api_service::mono::ADMIN_FILE) + }); + if admin_file_modified { + self.invalidate_admin_cache().await; + } + } + + Ok(()) + } + + pub async fn apply_update_result( + &self, + result: &TreeUpdateResult, + commit_msg: &str, + cl_link: Option<&str>, + ) -> Result { + let storage = self.storage.mono_storage(); + let mut new_commit_id = String::new(); + let mut commits: Vec = Vec::new(); + + let paths: Vec<&str> = result.ref_updates.iter().map(|r| r.path.as_str()).collect(); + + let cl_refs_formatted = cl_link.map(|cl| format!("refs/cl/{}", cl)); + let cl_refs: Option> = cl_refs_formatted + .as_ref() + .map(|formatted| vec![formatted.as_str(), MEGA_BRANCH_NAME]); + + let refs = storage + .get_refs_for_paths_and_cls(&paths, cl_refs.as_deref()) + .await?; + + let mut updates: Vec = Vec::new(); + + MonoServiceLogic::process_ref_updates( + result, + &refs, + commit_msg, + &mut commits, + &mut updates, + &mut new_commit_id, + )?; + + if new_commit_id.is_empty() { + return Err(GitError::CustomError( + "no commit_id generated: no matching refs found for the update paths".into(), + )); + } + + let txn = self + .storage + .begin_db_transaction() + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + let save_trees: Vec = result + .updated_trees + .clone() + .into_iter() + .map(|save_t| { + let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new()); + tree_model.commit_id.clone_from(&new_commit_id); + tree_model.into() + }) + .collect(); + + storage + .save_mega_commits(commits, Some(&txn)) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + storage + .batch_save_model_with_txn(save_trees, Some(&txn)) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + storage + .batch_upsert_ref_updates_in_txn(updates, &txn) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + txn.commit() + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + Ok(new_commit_id) + } + + /// Apply update result but only update the CL ref (never main). + /// Optionally override the parent commit for the first created commit (used by rebase). + pub(crate) async fn apply_update_result_cl_only( + &self, + result: &TreeUpdateResult, + commit_msg: &str, + cl_link: &str, + parent_override: Option, + ) -> Result { + let storage = self.storage.mono_storage(); + let mut new_commit_id = String::new(); + let mut commits: Vec = Vec::new(); + + let cl_ref_name = format!("refs/cl/{}", cl_link); + let cl_ref = storage + .get_ref_by_name(&cl_ref_name) + .await + .map_err(|e| GitError::CustomError(e.to_string()))? + .ok_or_else(|| GitError::CustomError("CL ref not found".to_string()))?; + + let mut updates: Vec = Vec::new(); + + MonoServiceLogic::process_ref_updates_cl_only( + result, + &cl_ref, + commit_msg, + parent_override, + &mut commits, + &mut updates, + &mut new_commit_id, + )?; + + if new_commit_id.is_empty() { + debug!( + cl_link, + ref_name = %cl_ref.ref_name, + ref_path = %cl_ref.path, + commit_msg, + "apply_update_result_cl_only: no commit_id generated" + ); + return Err(GitError::CustomError( + "no commit_id generated: no matching refs found for the update paths".into(), + )); + } + + storage + .batch_update_by_path_concurrent(updates) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + storage + .save_mega_commits(commits, None) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + let save_trees: Vec = result + .updated_trees + .clone() + .into_iter() + .map(|save_t| { + let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new()); + tree_model.commit_id.clone_from(&new_commit_id); + tree_model.into() + }) + .collect(); + + storage + .batch_save_model(save_trees) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + Ok(new_commit_id) + } +} diff --git a/ceres/src/application/api_service/mono/cl/merge_strategy.rs b/ceres/src/application/api_service/mono/cl/merge_strategy.rs new file mode 100644 index 000000000..1970faba9 --- /dev/null +++ b/ceres/src/application/api_service/mono/cl/merge_strategy.rs @@ -0,0 +1,269 @@ +//! CL merge strategy and monorepo path bootstrap helpers. + +use std::path::{Component, Path}; + +use callisto::mega_cl; +use common::{errors::MegaError, utils::ZERO_ID}; +use git_internal::{errors::GitError, internal::object::commit::Commit}; +use jupiter::utils::converter::FromMegaModel; + +use crate::api_service::{mono::MonoApiService, tree_ops}; + +/// How a CL should be applied onto monorepo main. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClMergeStrategy { + /// Apply file-level diff onto the path main baseline (web edits, incremental pushes). + FileDiff, + /// Replace the CL path subtree with `cl.to_hash` root tree (GitHub import / new directory). + SubtreeReplace, +} + +impl ClMergeStrategy { + pub fn as_str(self) -> &'static str { + match self { + Self::FileDiff => "file_diff", + Self::SubtreeReplace => "subtree_replace", + } + } +} + +/// Returns true when the path has no `refs/heads/main` row yet. +pub async fn path_lacks_main_ref(service: &MonoApiService, path: &str) -> Result { + Ok(service + .storage + .mono_storage() + .get_main_ref(path) + .await? + .is_none()) +} + +/// Enumerate strict prefixes for a normalized repo path, e.g. `/project/mega` → [`/project`]. +pub fn path_prefixes(path: &str) -> Vec { + let normalized = path.trim_end_matches('/'); + if normalized.is_empty() || normalized == "/" { + return Vec::new(); + } + let components: Vec<&str> = Path::new(normalized) + .components() + .filter_map(|c| match c { + Component::Normal(s) => s.to_str(), + _ => None, + }) + .collect(); + let mut out = Vec::new(); + let mut buf = String::from("/"); + for (idx, comp) in components.iter().enumerate() { + if idx == components.len() - 1 { + break; + } + if buf == "/" { + buf.push_str(comp); + } else { + buf.push('/'); + buf.push_str(comp); + } + out.push(buf.clone()); + } + out +} + +/// Sync path-level main refs for each strict prefix after the root tree changed. +pub async fn sync_path_prefix_main_refs( + service: &MonoApiService, + path: &str, +) -> Result<(), MegaError> { + for prefix in path_prefixes(path) { + let hash = crate::code_edit::utils::create_repo_commit(&service.storage, &prefix).await?; + if hash == ZERO_ID { + return Err(MegaError::Other(format!( + "Failed to sync main ref for prefix {prefix}" + ))); + } + } + Ok(()) +} + +/// Bootstrap a new monorepo path: attach under `/`, sync prefix refs, create path main baseline. +pub async fn bootstrap_monorepo_path( + service: &MonoApiService, + path: &str, + cl: Option<&mega_cl::Model>, +) -> Result { + let mono_storage = service.storage.mono_storage(); + if let Some(existing) = mono_storage.get_main_ref(path).await? { + if let Some(cl) = cl + && cl.from_hash == ZERO_ID + { + service + .storage + .cl_storage() + .update_cl_hash(cl.clone(), &existing.ref_commit_hash, &cl.to_hash) + .await?; + } + return Ok(existing.ref_commit_hash); + } + + service.attach_project_path_to_monorepo_root(path).await?; + sync_path_prefix_main_refs(service, path).await?; + + let baseline_hash = crate::code_edit::utils::create_repo_commit(&service.storage, path).await?; + if baseline_hash == ZERO_ID { + return Err(MegaError::Other(format!( + "Failed to create main ref baseline for {path}" + ))); + } + + if let Some(cl) = cl + && cl.from_hash == ZERO_ID + { + service + .storage + .cl_storage() + .update_cl_hash(cl.clone(), &baseline_hash, &cl.to_hash) + .await?; + } + + Ok(baseline_hash) +} + +/// Prepare a CL path for merge (bootstrap new `/project/*` directories). +pub async fn prepare_cl_path_for_merge( + service: &MonoApiService, + cl: &mut mega_cl::Model, +) -> Result<(), MegaError> { + if !cl.path.starts_with("/project/") { + return Ok(()); + } + + bootstrap_monorepo_path(service, &cl.path, Some(cl)).await?; + + if let Some(fresh) = service.storage.cl_storage().get_cl(&cl.link).await? { + *cl = fresh; + } + Ok(()) +} + +pub async fn resolve_merge_strategy( + service: &MonoApiService, + cl: &mega_cl::Model, +) -> Result { + if cl.from_hash == ZERO_ID { + return Ok(ClMergeStrategy::SubtreeReplace); + } + + if path_lacks_main_ref(service, &cl.path).await? { + return Ok(ClMergeStrategy::SubtreeReplace); + } + + if is_gitkeep_baseline(service, &cl.from_hash).await? { + return Ok(ClMergeStrategy::SubtreeReplace); + } + + Ok(ClMergeStrategy::FileDiff) +} + +async fn is_gitkeep_baseline( + service: &MonoApiService, + commit_hash: &str, +) -> Result { + let blobs = service.get_commit_blobs(commit_hash).await?; + if blobs.is_empty() { + return Ok(true); + } + Ok(blobs.len() == 1 + && blobs[0] + .0 + .file_name() + .is_some_and(|name| name == ".gitkeep")) +} + +/// Returns true when the final path segment is not yet present in the monorepo tree. +pub async fn needs_path_tree_attach( + service: &MonoApiService, + path: &Path, +) -> Result { + let parent = match path.parent() { + Some(p) if p.as_os_str().is_empty() || p == Path::new("/") => Path::new("/"), + Some(p) => p, + None => return Ok(false), + }; + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + return Ok(false); + }; + let parent_tree = match tree_ops::search_tree_by_path(service, parent, None).await? { + Some(tree) => tree, + None => return Ok(true), + }; + Ok(!parent_tree + .tree_items + .iter() + .any(|item| item.name == name && item.is_tree())) +} + +/// Leaf tree hash to mount at `cl.path` for merge. +pub async fn resolve_merge_leaf_tree_id( + service: &MonoApiService, + cl: &mega_cl::Model, + strategy: ClMergeStrategy, +) -> Result { + let storage = service.storage.mono_storage(); + + match strategy { + ClMergeStrategy::SubtreeReplace => { + let commit_model = storage + .get_commit_by_hash(&cl.to_hash) + .await + .map_err(|e| GitError::CustomError(format!("Failed to get commit: {e}")))? + .ok_or_else(|| { + GitError::CustomError(format!("Commit not found: {}", cl.to_hash)) + })?; + let commit = Commit::from_mega_model(commit_model); + Ok(commit.tree_id) + } + ClMergeStrategy::FileDiff => { + let main_ref = storage + .get_main_ref(&cl.path) + .await + .map_err(|e| GitError::CustomError(e.to_string()))? + .ok_or_else(|| GitError::CustomError("Main ref not found".to_string()))?; + + let old_blobs = service + .get_commit_blobs(&cl.from_hash) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + let new_blobs = service + .get_commit_blobs(&cl.to_hash) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + let cl_changed = service + .cl_files_list(old_blobs, new_blobs) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + let merged_commit_hash = service + .apply_changes_as_single_commit(cl, &cl_changed, &main_ref.ref_commit_hash) + .await?; + + let merged = storage + .get_commit_by_hash(&merged_commit_hash) + .await + .map_err(|e| GitError::CustomError(e.to_string()))? + .ok_or_else(|| { + GitError::CustomError(format!("Merged commit not found: {merged_commit_hash}")) + })?; + Ok(Commit::from_mega_model(merged).tree_id) + } + } +} + +#[cfg(test)] +mod tests { + use super::path_prefixes; + + #[test] + fn path_prefixes_returns_strict_prefixes() { + assert_eq!(path_prefixes("/project/mega"), vec!["/project".to_string()]); + assert_eq!(path_prefixes("/project"), Vec::::new()); + assert_eq!(path_prefixes("/"), Vec::::new()); + } +} diff --git a/ceres/src/application/api_service/mono/cl/mod.rs b/ceres/src/application/api_service/mono/cl/mod.rs new file mode 100644 index 000000000..4b0c79848 --- /dev/null +++ b/ceres/src/application/api_service/mono/cl/mod.rs @@ -0,0 +1,8 @@ +//! Change-list domain: merge, branch update, diff, queue. + +pub mod branch; +pub mod diff; +pub mod lifecycle; +pub mod merge; +pub mod merge_strategy; +pub mod queue; diff --git a/ceres/src/application/api_service/mono/cl/queue.rs b/ceres/src/application/api_service/mono/cl/queue.rs new file mode 100644 index 000000000..16c467ed9 --- /dev/null +++ b/ceres/src/application/api_service/mono/cl/queue.rs @@ -0,0 +1,389 @@ +//! Merge queue background processor for [`MonoApiService`](super::service::MonoApiService). + +use std::time::Duration; + +use callisto::sea_orm_active_enums::{MergeStatusEnum, QueueFailureTypeEnum, QueueStatusEnum}; +use common::errors::MegaError; +use tracing; + +use crate::{ + api_service::mono::MonoApiService, + model::merge_queue::{ + AddToQueueResponse, QueueItem, QueueListResponse, QueueStatsResponse, QueueStatus, + QueueStatusResponse, + }, +}; + +impl MonoApiService { + // ========== Merge Queue Methods ========== + + /// Queue polling interval in seconds when no items are processed + const QUEUE_POLL_INTERVAL_SECS: u64 = 5; + + /// Error backoff interval in seconds after processing failure + const ERROR_BACKOFF_SECS: u64 = 30; + + /// Adds a CL to the merge queue and ensures the background processor is running. + /// + /// This method validates the CL status before adding to queue and automatically + /// starts the background processor if not already running. + /// + /// # Arguments + /// * `cl_link` - The unique identifier of the CL to add to queue + /// + /// # Returns + /// * `Ok(i64)` - The position in queue on success + /// * `Err(MegaError)` - If validation fails or database error occurs + pub async fn add_to_merge_queue(&self, cl_link: String) -> Result { + // Validate CL exists and is in Open status + let cl = self.storage.cl_storage().get_cl(&cl_link).await?; + let model = cl.ok_or(MegaError::Other("CL not found".to_string()))?; + + if model.status != MergeStatusEnum::Open { + return Err(MegaError::Other(format!( + "CL is not in Open status, current status: {:?}", + model.status + ))); + } + + // Add to queue via jupiter layer service + let position = self + .storage + .merge_queue_service + .add_to_queue(cl_link) + .await?; + + // Ensure the background processor is running + self.ensure_merge_processor_running(); + + Ok(position) + } + + /// Retries a failed merge queue item and ensures the processor is running. + /// + /// # Arguments + /// * `cl_link` - The unique identifier of the CL to retry + /// + /// # Returns + /// * `Ok(true)` - If retry was successful + /// * `Ok(false)` - If item not found or cannot be retried + /// * `Err(MegaError)` - If database error occurs + pub async fn retry_merge_queue_item(&self, cl_link: &str) -> Result { + let result = self + .storage + .merge_queue_service + .retry_queue_item(cl_link) + .await?; + + if result { + // Ensure the background processor is running + self.ensure_merge_processor_running(); + } + + Ok(result) + } + + /// Adds a CL to the merge queue and returns API response fields including display position. + pub async fn add_to_merge_queue_response( + &self, + cl_link: String, + ) -> Result { + let position = self.add_to_merge_queue(cl_link.clone()).await?; + let display_position = self + .storage + .merge_queue_service + .get_display_position_by_position(position) + .await + .map(Some) + .unwrap_or_else(|e| { + tracing::warn!( + "Failed to get display position after add for {}: {}", + cl_link, + e + ); + None + }); + + Ok(AddToQueueResponse { + success: true, + position, + display_position, + message: "Added to queue".to_string(), + }) + } + + pub async fn remove_from_merge_queue(&self, cl_link: &str) -> Result { + self.storage + .merge_queue_service + .remove_from_queue(cl_link) + .await + } + + pub async fn get_merge_queue_list(&self) -> Result { + let items = self.storage.merge_queue_service.get_queue_list().await?; + Ok(QueueListResponse::from(items)) + } + + pub async fn get_cl_queue_status( + &self, + cl_link: &str, + ) -> Result { + let item_model = self + .storage + .merge_queue_service + .get_cl_queue_status(cl_link) + .await?; + + let mut item_opt: Option = item_model.map(|m| m.into()); + + if let Some(ref mut item) = item_opt { + match item.status { + QueueStatus::Waiting | QueueStatus::Testing | QueueStatus::Merging => { + match self + .storage + .merge_queue_service + .get_display_position(&item.cl_link) + .await + { + Ok(index) => item.display_position = index, + Err(e) => { + tracing::warn!( + "Failed to get display position for {}: {}", + item.cl_link, + e + ); + item.display_position = None; + } + } + } + _ => {} + } + } + + Ok(QueueStatusResponse { + in_queue: item_opt.is_some(), + item: item_opt, + }) + } + + pub async fn get_merge_queue_stats(&self) -> Result { + let stats = self.storage.merge_queue_service.get_queue_stats().await?; + Ok(QueueStatsResponse::from(stats)) + } + + pub async fn cancel_all_pending_merge_queue(&self) -> Result<(), MegaError> { + self.storage + .merge_queue_service + .cancel_all_pending() + .await?; + Ok(()) + } + + /// Ensures the background merge processor is running. + /// + /// Uses atomic flag to guarantee only one processor task runs at a time. + /// The processor automatically stops when no active items remain in queue. + fn ensure_merge_processor_running(&self) { + // Get the processor running flag from merge queue service + if self.storage.merge_queue_service.try_start_processor() { + let service = self.clone(); + tokio::spawn(async move { + tracing::info!("Merge queue processor started (from MonoApiService)"); + service.run_merge_processor_loop().await; + }); + } + } + + /// Main loop for the background merge processor. + /// + /// Continuously processes queue items until no active items remain. + async fn run_merge_processor_loop(&self) { + loop { + match self.process_next_queue_item().await { + Ok(processed) => { + if !processed { + // Check if there are active items + if let Ok(stats) = self.storage.merge_queue_service.get_queue_stats().await + { + let has_active = stats.waiting_count > 0 + || stats.testing_count > 0 + || stats.merging_count > 0; + + if !has_active { + // No active items, stop processor + self.storage.merge_queue_service.stop_processor(); + tracing::info!("Merge queue processor stopped (no active items)"); + break; + } + } + tokio::time::sleep(Duration::from_secs(Self::QUEUE_POLL_INTERVAL_SECS)) + .await; + } + } + Err(e) => { + tracing::error!("Merge queue processor error: {}", e); + tokio::time::sleep(Duration::from_secs(Self::ERROR_BACKOFF_SECS)).await; + } + } + } + } + + /// Processes the next item in the merge queue. + /// + /// # Returns + /// * `Ok(true)` - An item was processed (success or failure) + /// * `Ok(false)` - No items to process + /// * `Err(MegaError)` - System error occurred + async fn process_next_queue_item(&self) -> Result { + let queue_service = &self.storage.merge_queue_service; + + // Get next waiting item from queue + let next_item = queue_service.get_next_waiting_item().await?; + + if let Some(item) = next_item { + let cl_link = item.cl_link.clone(); + + // Update status to Testing + let updated = queue_service + .update_item_status(&cl_link, QueueStatusEnum::Testing) + .await?; + + // Item was cancelled before we could start processing + if !updated { + return Ok(false); + } + + // Execute the merge workflow + match self.execute_merge_workflow(&cl_link).await { + Ok(()) => { + // Success - status already updated to Merged in workflow + Ok(true) + } + Err((failure_type, message)) => { + if matches!(failure_type, QueueFailureTypeEnum::Conflict) { + // Conflict - move to tail of queue for retry + if let Err(e) = queue_service.move_item_to_tail(&cl_link).await { + tracing::warn!( + "Failed to move conflicting item {} to tail: {}", + cl_link, + e + ); + } + Ok(false) + } else { + // Other failure - mark as failed + if let Err(e) = queue_service + .update_item_status_with_error(&cl_link, failure_type, message) + .await + { + tracing::error!( + "Failed to update item {} status to failed: {}", + cl_link, + e + ); + } + Ok(true) + } + } + } + } else { + Ok(false) + } + } + + /// Executes the complete merge workflow for a CL. + /// + /// Workflow steps: + /// 1. Validate CL exists and is in valid status + /// 2. Run tests (TODO: Buck2 integration) + /// 3. Check for conflicts + /// 4. Execute merge + /// 5. Update statuses + async fn execute_merge_workflow( + &self, + cl_link: &str, + ) -> Result<(), (QueueFailureTypeEnum, String)> { + let queue_service = &self.storage.merge_queue_service; + + // Step 1: Validate CL still exists and is not closed + let cl = self + .storage + .cl_storage() + .get_cl(cl_link) + .await + .map_err(|e| { + ( + QueueFailureTypeEnum::SystemError, + format!("Failed to fetch CL: {}", e), + ) + })?; + + let cl_model = match cl { + Some(model) => { + if model.status == MergeStatusEnum::Closed { + return Err(( + QueueFailureTypeEnum::SystemError, + "CL has been closed, cannot merge".to_string(), + )); + } + if model.status == MergeStatusEnum::Draft { + return Err(( + QueueFailureTypeEnum::SystemError, + "CL is in draft status, cannot merge".to_string(), + )); + } + model + } + None => { + return Err(( + QueueFailureTypeEnum::SystemError, + "CL no longer exists, cannot merge".to_string(), + )); + } + }; + + let updated = queue_service + .update_item_status(cl_link, QueueStatusEnum::Merging) + .await + .map_err(|e| { + ( + QueueFailureTypeEnum::SystemError, + format!("Failed to update status to merging: {}", e), + ) + })?; + + if !updated { + return Err(( + QueueFailureTypeEnum::SystemError, + "Item was cancelled".to_string(), + )); + } + + let merge_result = self.merge_cl("system", cl_model).await; + + if let Err(e) = merge_result { + let message = e.to_string(); + let failure_type = if message.contains("conflict") || message.contains("Conflict") { + QueueFailureTypeEnum::Conflict + } else if message.contains("unmergeable") || message.contains("FAILED") { + QueueFailureTypeEnum::SystemError + } else { + QueueFailureTypeEnum::MergeFailure + }; + return Err((failure_type, format!("Merge failed: {message}"))); + } + + // Step 6: Update queue status to Merged + queue_service + .update_item_status(cl_link, QueueStatusEnum::Merged) + .await + .map_err(|e| { + ( + QueueFailureTypeEnum::SystemError, + format!("Failed to update status to merged: {}", e), + ) + })?; + + Ok(()) + } +} diff --git a/ceres/src/application/api_service/mono/cla.rs b/ceres/src/application/api_service/mono/cla.rs new file mode 100644 index 000000000..4d7d0d737 --- /dev/null +++ b/ceres/src/application/api_service/mono/cla.rs @@ -0,0 +1,108 @@ +//! CLA (Contributor License Agreement) operations for [`MonoApiService`](super::service::MonoApiService). + +use bytes::Bytes; +use common::errors::MegaError; +use futures::{StreamExt, stream}; +use io_orbit::object_storage::{ObjectKey, ObjectMeta, ObjectNamespace}; + +use crate::{api_service::mono::MonoApiService, merge_checker::CheckerRegistry}; + +const CLA_CONTENT_OBJECT_KEY: &str = "cla/content/current.txt"; + +impl MonoApiService { + pub async fn get_or_init_cla_sign_status( + &self, + username: &str, + ) -> Result<(bool, Option), MegaError> { + let model = self + .storage + .cla_storage() + .get_or_create_status(username) + .await?; + Ok((model.cla_signed, model.cla_signed_at)) + } + + pub async fn get_cla_content(&self) -> Result { + let key = ObjectKey { + namespace: ObjectNamespace::Log, + key: CLA_CONTENT_OBJECT_KEY.to_string(), + }; + + let stream = self + .storage + .git_service + .obj_storage + .inner + .get_stream(&key) + .await; + let (mut stream, _meta) = match stream { + Ok(result) => result, + Err(MegaError::ObjStorageNotFound(_)) => return Ok(String::new()), + Err(e) => return Err(e), + }; + + let mut data = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + data.extend_from_slice(&chunk); + } + + String::from_utf8(data).map_err(|e| { + MegaError::Other(format!( + "Invalid UTF-8 in CLA content from object storage: {e}" + )) + }) + } + + pub async fn update_cla_content(&self, content: &str) -> Result<(), MegaError> { + let key = ObjectKey { + namespace: ObjectNamespace::Log, + key: CLA_CONTENT_OBJECT_KEY.to_string(), + }; + + let bytes = Bytes::from(content.as_bytes().to_vec()); + let stream = stream::once(async move { Ok::(bytes) }); + let meta = ObjectMeta { + size: content.len() as i64, + content_type: Some("text/plain; charset=utf-8".to_string()), + ..Default::default() + }; + + self.storage + .git_service + .obj_storage + .inner + .put_stream(&key, Box::pin(stream), meta) + .await + } + + pub async fn change_cla_sign_status( + &self, + username: &str, + ) -> Result<(bool, Option), MegaError> { + let model = self.storage.cla_storage().sign(username).await?; + self.refresh_checks_for_open_cls_by_author(username).await?; + Ok((model.cla_signed, model.cla_signed_at)) + } + + async fn refresh_checks_for_open_cls_by_author(&self, username: &str) -> Result<(), MegaError> { + let open_cls = self + .storage + .cl_storage() + .get_open_cls() + .await? + .into_iter() + .filter(|cl| cl.username == username) + .collect::>(); + if open_cls.is_empty() { + return Ok(()); + } + + let check_reg = CheckerRegistry::new(self.storage.clone().into(), username.to_string()); + for cl in open_cls { + check_reg.run_checks(cl.into()).await?; + } + + Ok(()) + } +} diff --git a/ceres/src/application/api_service/mono/edit/entry.rs b/ceres/src/application/api_service/mono/edit/entry.rs new file mode 100644 index 000000000..15cd6aff6 --- /dev/null +++ b/ceres/src/application/api_service/mono/edit/entry.rs @@ -0,0 +1,656 @@ +//! Monorepo entry creation and file edit operations for [`MonoApiService`](super::service::MonoApiService). + +use std::{ + path::{Path, PathBuf}, + str::FromStr, +}; + +use callisto::mega_tree; +use common::utils::MEGA_BRANCH_NAME; +use git_internal::{ + errors::GitError, + hash::ObjectHash, + internal::{ + metadata::EntryMeta, + object::{ + blob::Blob, + commit::Commit, + tree::{Tree, TreeItem, TreeItemMode}, + }, + }, +}; +use jupiter::{ + storage::base_storage::StorageConnector, + utils::converter::{IntoMegaModel, generate_git_keep_with_timestamp}, +}; + +use crate::{ + api_service::{ + ApiHandler, + mono::{ + MonoApiService, + logic::{MonoServiceLogic, path_not_exist_re}, + types::{CreateEntryUpdate, TreeUpdateResult}, + }, + tree_ops, + }, + code_edit::{on_edit::OneditCodeEdit, utils as edit_utils}, + model::git::{CreateEntryInfo, CreateEntryResult, EditFilePayload, EditFileResult}, +}; + +impl MonoApiService { + pub(crate) async fn prepare_create_entry_update( + &self, + entry_info: &CreateEntryInfo, + ) -> Result { + let path = PathBuf::from(&entry_info.path); + let mut save_trees = vec![]; + let file_content = if entry_info.is_directory { + None + } else { + Some(entry_info.content.as_deref().ok_or_else(|| { + GitError::CustomError("content is required for file creation".to_string()) + })?) + }; + + // Try to get the update chain for the given path. + // If the path exists, return an empty missing_parts and prefix. + // If part of the path does not exist, extract the missing segments (missing_parts), + // determine the valid existing prefix, and rebuild the update_chain from that prefix. + let (missing_parts, prefix, mut update_chain) = + match self.search_tree_for_update(&path).await { + Ok(chain) => (Vec::new(), "", chain), + Err(err) => { + // If search_tree_for_update failed, try to extract the + // portion of the path that does not exist from the + // error message. The error message is expected to + // contain a substring like: Path '.../missing' not exist + // We capture that substring to determine which segments + // need to be created. + let err_str = err.to_string(); + let extracted = path_not_exist_re() + .captures(&err_str) + .map(|caps| caps[1].to_string()) + .ok_or_else(|| { + GitError::CustomError(format!("Path resolution failed: {err_str}")) + })?; + + // missing_parts: the trailing path segments after the + // first occurrence of the extracted non-existent path. + // Example: entry_info.path = "a/b/c/d" and extracted = "c/d" + // Then missing_parts = ["c", "d"] + let missing_parts = entry_info + .path + .find(&extracted) + .map(|pos| &entry_info.path[pos..]) + .map(|sub| sub.split('/').collect::>()) + .unwrap_or_default(); + + if missing_parts.is_empty() { + return Err(GitError::CustomError(format!( + "Missing path segments for '{}': {err_str}", + entry_info.path + ))); + } + + // prefix: the valid existing path before the missing parts. + // Using the same example above, prefix = "a/b/" + let prefix = entry_info + .path + .find(&extracted) + .map(|pos| &entry_info.path[..pos]) + .unwrap_or(""); + + // Rebuild the update chain starting from the valid prefix + // so subsequent operations only update from that known + // existing tree downward. + let chain = self.search_tree_for_update(Path::new(prefix)).await?; + (missing_parts, prefix, chain) + } + }; + + let target_items = update_chain + .pop() + .ok_or_else(|| GitError::CustomError("Empty update chain".to_string()))? + .tree_items + .clone(); + + // If there are no missing parts, we are inserting directly into an + // existing tree. This branch handles both creating a new file or + // creating a new directory in the target tree. + let (update_result, blob, entry_oid, repo_path) = if missing_parts.is_empty() { + let mut target_items = target_items; + + // Check for duplicate + let is_tree_mode = if entry_info.is_directory { + TreeItemMode::Tree + } else { + TreeItemMode::Blob + }; + if target_items + .iter() + .any(|x| x.mode == is_tree_mode && x.name == entry_info.name) + { + return Err(GitError::CustomError("Duplicate name".to_string())); + } + + // Create a new tree item based on whether it's a directory or file + let (new_item, blob, entry_oid) = if entry_info.is_directory { + // For a new directory, create a .gitkeep blob so the + // directory can be represented as a tree with at least + // one blob entry. The blob contains a timestamp so it's + // unique. + let blob = generate_git_keep_with_timestamp(); + let tree_item = TreeItem { + mode: TreeItemMode::Blob, + id: blob.id, + name: String::from(".gitkeep"), + }; + let new_dir_tree = Tree::from_tree_items(vec![tree_item]).unwrap(); + save_trees.push(new_dir_tree.clone()); + let entry_oid = new_dir_tree.id; + ( + TreeItem { + mode: TreeItemMode::Tree, + id: new_dir_tree.id, + name: entry_info.name.clone(), + }, + blob, + entry_oid, + ) + } else { + let content = file_content + .ok_or_else(|| GitError::CustomError("Missing file content".to_string()))?; + let blob = Blob::from_content(content); + let entry_oid = blob.id; + ( + TreeItem { + mode: TreeItemMode::Blob, + id: blob.id, + name: entry_info.name.clone(), + }, + blob, + entry_oid, + ) + }; + + target_items.push(new_item); + target_items.sort_by(|a, b| a.name.cmp(&b.name)); + let target_tree = Tree::from_tree_items(target_items).unwrap(); + save_trees.push(target_tree.clone()); + + // Build update instructions for parent trees and refs. + // build_result_by_chain walks the update_chain (parent trees) + // and prepares the list of updated trees and ref updates + // that must be applied to persist the change. + let update_result = MonoServiceLogic::build_result_by_chain( + if prefix.is_empty() { + path.clone() + } else { + PathBuf::from(prefix) + }, + update_chain, + target_tree.id, + )?; + let repo_path = if prefix.is_empty() { + path.clone() + } else { + PathBuf::from(prefix) + }; + (update_result, blob, entry_oid, repo_path) + } else { + // If missing_parts is not empty, we must create intermediate + // directories (trees) for each missing segment. This branch + // constructs the leaf tree first and then wraps it with + // additional trees for each missing path component up to the + // existing prefix. + // Create a new tree item based on whether it's a directory or file + let (leaf_item, blob, entry_oid) = if entry_info.is_directory { + // Create .gitkeep blob and an initial tree for the new + // directory leaf. This represents the directory's own + // tree object which will be nested under new parent trees. + let blob = generate_git_keep_with_timestamp(); + let tree_item = TreeItem { + mode: TreeItemMode::Blob, + id: blob.id, + name: String::from(".gitkeep"), + }; + let new_dir_tree = Tree::from_tree_items(vec![tree_item]).unwrap(); + save_trees.push(new_dir_tree.clone()); + let entry_oid = new_dir_tree.id; + ( + TreeItem { + mode: TreeItemMode::Tree, + id: new_dir_tree.id, + name: entry_info.name.clone(), + }, + blob, + entry_oid, + ) + } else { + let content = file_content + .ok_or_else(|| GitError::CustomError("Missing file content".to_string()))?; + let blob = Blob::from_content(content); + let entry_oid = blob.id; + ( + TreeItem { + mode: TreeItemMode::Blob, + id: blob.id, + name: entry_info.name.clone(), + }, + blob, + entry_oid, + ) + }; + + let mut current_tree = Tree::from_tree_items(vec![leaf_item]).unwrap(); + save_trees.push(current_tree.clone()); + + // Wrap the leaf tree with trees for each missing parent segment. + // We iterate the missing parts in reverse (from leaf's parent up + // to the topmost missing segment) and create a tree object for + // each level that points to the previously built child tree. + let missing_len = missing_parts.len(); + for part in missing_parts.iter().rev().take(missing_len - 1) { + let sub_item = TreeItem { + mode: TreeItemMode::Tree, + id: current_tree.id, + name: part.to_string(), + }; + + current_tree = Tree::from_tree_items(vec![sub_item]).unwrap(); + save_trees.push(current_tree.clone()); + } + + // top_part is the highest-level missing segment (closest to the + // existing prefix). We'll insert this as a child into the + // existing target_items collected from the update chain. + let top_part = missing_parts + .first() + .expect("missing_parts is non-empty by branch condition") + .to_string(); + let top_item = TreeItem { + mode: TreeItemMode::Tree, + id: current_tree.id, + name: top_part.clone(), + }; + + let mut target_items = target_items; + + // Check for duplicate + if target_items + .iter() + .any(|x| x.mode == TreeItemMode::Tree && x.name == top_part) + { + return Err(GitError::CustomError("Duplicate name".to_string())); + } + + target_items.push(top_item); + target_items.sort_by(|a, b| a.name.cmp(&b.name)); + let target_tree = Tree::from_tree_items(target_items).unwrap(); + save_trees.push(target_tree.clone()); + + // After constructing the nested trees, build update instructions + // and apply them to update the parent trees and refs so the + // new nested directory/file is persisted in the repository. + let update_result = MonoServiceLogic::build_result_by_chain( + PathBuf::from(prefix), + update_chain, + target_tree.id, + )?; + let repo_path = PathBuf::from(prefix); + (update_result, blob, entry_oid, repo_path) + }; + + Ok(CreateEntryUpdate { + update_result, + blob, + entry_oid, + repo_path, + save_trees, + }) + } + + pub(crate) fn build_entry_path(path: &str, name: &str) -> String { + let trimmed = path.trim_end_matches('/'); + if trimmed.is_empty() || trimmed == "/" { + format!("/{name}") + } else { + format!("{trimmed}/{name}") + } + } + + pub(crate) fn ref_update_tree_id_for_path( + result: &TreeUpdateResult, + repo_path: &str, + ) -> Option { + let normalized = MonoServiceLogic::clean_path_str(repo_path); + result + .ref_updates + .iter() + .find(|update| update.path == normalized) + .map(|update| update.tree_id) + } + pub(crate) async fn save_file_edit_impl( + &self, + payload: EditFilePayload, + ) -> Result { + let file_path = PathBuf::from("/").join(PathBuf::from(&payload.path)); + let parent_path = file_path + .parent() + .ok_or_else(|| GitError::CustomError("Invalid file path".to_string()))?; + let cl_root_path = MonoServiceLogic::subtree_ref_path(parent_path) + .map_err(|e| GitError::CustomError(e.to_string()))?; + let build_repo_path = match edit_utils::resolve_build_repo_root( + &self.storage, + &cl_root_path, + ) + .await + { + Ok(path) => path, + Err(e) => { + tracing::warn!( + repo_path = %cl_root_path, + "Failed to resolve build repo root for edit, fallback to CL subtree root: {}", + e + ); + cl_root_path.clone() + } + }; + + let parent_tree = tree_ops::search_tree_by_path(self, parent_path, None) + .await? + .ok_or(GitError::CustomError(format!( + "invalid repo_path {}, Parent tree not found", + cl_root_path + )))?; + + let file_name = file_path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| GitError::CustomError("Invalid file name".to_string()))?; + + let _current_item = parent_tree + .tree_items + .iter() + .find(|x| x.name == file_name && x.mode == TreeItemMode::Blob) + .ok_or_else(|| GitError::CustomError("[code:404] File not found".to_string()))?; + + // Create new blob and build update result up to root + let new_blob = Blob::from_content(&payload.content); + let new_tree = MonoServiceLogic::update_tree_hash( + parent_tree.into(), + file_path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| GitError::CustomError("Invalid path".into()))?, + new_blob.id, + )?; + + let mut update_chain = self.search_tree_for_update(parent_path).await?; + let _target_tree = update_chain + .pop() + .ok_or_else(|| GitError::CustomError("Empty update chain".to_string()))?; + let update_result = MonoServiceLogic::build_result_by_chain( + parent_path.to_path_buf(), + update_chain, + new_tree.id, + )?; + let target_tree_id = Self::ref_update_tree_id_for_path(&update_result, &build_repo_path) + .ok_or_else(|| { + GitError::CustomError(format!( + "Missing updated tree for build repo root {build_repo_path}" + )) + })?; + + let src_commit = + edit_utils::get_repo_main_latest_commit(&self.storage, &build_repo_path).await?; + let dst_commit = Commit::from_tree_id( + target_tree_id, + vec![ + ObjectHash::from_str(&src_commit.id.to_string()).map_err(|e| { + GitError::CustomError(format!("Invalid commit hash {}: {e}", src_commit.id)) + })?, + ], + &payload.commit_message, + ); + let new_commit_id = dst_commit.id.to_string(); + + let username = payload + .author_username + .clone() + .unwrap_or("Anonymous".to_string()); + + self.storage + .mono_service + .mono_storage + .save_mega_commits(vec![dst_commit], None) + .await?; + + let mut all_trees = vec![new_tree]; + all_trees.extend(update_result.updated_trees); + let save_trees: Vec = all_trees + .into_iter() + .map(|save_t| { + let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new()); + tree_model.commit_id.clone_from(&new_commit_id); + tree_model.into() + }) + .collect(); + + self.storage + .mono_service + .mono_storage + .batch_save_model(save_trees) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + let editor = OneditCodeEdit::from( + &build_repo_path, + MEGA_BRANCH_NAME + .strip_prefix("refs/heads/") + .unwrap_or(MEGA_BRANCH_NAME), + &src_commit.id.to_string(), + self, + self.storage.mono_storage(), + ); + let cl = editor + .find_or_create_cl_for_edit( + &self.storage, + &editor, + payload.mode, + &new_commit_id, + &username, + ) + .await?; + + self.storage + .mono_service + .save_blobs(&new_commit_id, vec![new_blob.clone()]) + .await?; + + if !payload.skip_build { + self.trigger_build_for_cl(&editor, &cl, &username).await?; + } + + Ok(EditFileResult { + commit_id: new_commit_id, + new_oid: new_blob.id.to_string(), + path: build_repo_path, + cl_link: Some(cl.link), + }) + } + + /// Creates a new file or directory in the monorepo based on the provided file information. + /// + /// # Arguments + /// + /// * `entry_info` - Information about the file or directory to create. + /// + /// # Returns + /// + /// Returns commit metadata on success, or a `GitError` on failure. + pub(crate) async fn create_monorepo_entry_impl( + &self, + entry_info: CreateEntryInfo, + ) -> Result { + let storage = self.storage.mono_storage(); + let CreateEntryUpdate { + update_result, + blob, + entry_oid, + repo_path, + mut save_trees, + } = self.prepare_create_entry_update(&entry_info).await?; + + let repo_path_str = MonoServiceLogic::subtree_ref_path(&repo_path) + .map_err(|e| GitError::CustomError(e.to_string()))?; + let build_repo_path = match edit_utils::resolve_build_repo_root( + &self.storage, + &repo_path_str, + ) + .await + { + Ok(path) => path, + Err(e) => { + tracing::warn!( + repo_path = %repo_path_str, + "Failed to resolve build repo root for create entry, fallback to CL subtree root: {}", + e + ); + repo_path_str.clone() + } + }; + + let src_commit = + edit_utils::get_repo_main_latest_commit(&self.storage, &build_repo_path).await?; + let base_commit = ObjectHash::from_str(&src_commit.id.to_string()).map_err(|e| { + GitError::CustomError(format!("Invalid commit hash {}: {e}", src_commit.id)) + })?; + let target_tree_id = Self::ref_update_tree_id_for_path(&update_result, &build_repo_path) + .ok_or_else(|| { + GitError::CustomError(format!( + "Missing updated tree for build repo root {build_repo_path}" + )) + })?; + let dst_commit = + Commit::from_tree_id(target_tree_id, vec![base_commit], &entry_info.commit_msg()); + let new_commit_id = dst_commit.id.to_string(); + + let username = entry_info + .author_username + .clone() + .unwrap_or("Anonymous".to_string()); + + let new_oid = entry_oid.to_string(); + + let mut all_trees = update_result.updated_trees; + all_trees.append(&mut save_trees); + let save_trees: Vec = all_trees + .into_iter() + .map(|save_t| { + let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new()); + tree_model.commit_id.clone_from(&new_commit_id); + tree_model.into() + }) + .collect(); + self.storage + .mono_service + .save_blobs(&new_commit_id, vec![blob]) + .await?; + + storage + .batch_save_model(save_trees) + .await + .map_err(|e| GitError::CustomError(e.to_string()))?; + + self.storage + .mono_service + .mono_storage + .save_mega_commits(vec![dst_commit], None) + .await?; + + let editor = OneditCodeEdit::from( + &build_repo_path, + MEGA_BRANCH_NAME + .strip_prefix("refs/heads/") + .unwrap_or(MEGA_BRANCH_NAME), + &src_commit.id.to_string(), + self, + self.storage.mono_storage(), + ); + let cl = editor + .find_or_create_cl_for_edit( + &self.storage, + &editor, + entry_info.mode.clone(), + &new_commit_id, + &username, + ) + .await?; + + if !entry_info.skip_build { + self.trigger_build_for_cl(&editor, &cl, &username).await?; + } + + let entry_path = Self::build_entry_path(&entry_info.path, &entry_info.name); + + Ok(CreateEntryResult { + commit_id: new_commit_id, + new_oid, + path: entry_path, + cl_link: Some(cl.link), + }) + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use git_internal::hash::ObjectHash; + + use crate::api_service::mono::{ + MonoApiService, + types::{RefUpdate, TreeUpdateResult}, + }; + + #[test] + fn test_save_file_edit_uses_build_repo_root_tree_from_ref_updates() { + let build_root_tree = + ObjectHash::from_str("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap(); + let nested_tree = ObjectHash::from_str("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb").unwrap(); + let result = TreeUpdateResult { + updated_trees: vec![], + ref_updates: vec![ + RefUpdate { + path: "/project/buck2_test/src".to_string(), + tree_id: nested_tree, + }, + RefUpdate { + path: "/project/buck2_test".to_string(), + tree_id: build_root_tree, + }, + ], + }; + + let selected = MonoApiService::ref_update_tree_id_for_path(&result, "/project/buck2_test"); + assert_eq!(selected, Some(build_root_tree)); + } + + #[test] + fn test_create_monorepo_entry_uses_normalized_build_repo_root_tree() { + let build_root_tree = + ObjectHash::from_str("cccccccccccccccccccccccccccccccccccccccc").unwrap(); + let result = TreeUpdateResult { + updated_trees: vec![], + ref_updates: vec![RefUpdate { + path: "/project/buck2_test".to_string(), + tree_id: build_root_tree, + }], + }; + + let selected = MonoApiService::ref_update_tree_id_for_path(&result, "/project/buck2_test/"); + assert_eq!(selected, Some(build_root_tree)); + } +} diff --git a/ceres/src/application/api_service/mono/edit/mod.rs b/ceres/src/application/api_service/mono/edit/mod.rs new file mode 100644 index 000000000..64affd1a0 --- /dev/null +++ b/ceres/src/application/api_service/mono/edit/mod.rs @@ -0,0 +1,3 @@ +//! Web edit / monorepo entry creation. + +pub mod entry; diff --git a/ceres/src/application/api_service/mono/logic/mod.rs b/ceres/src/application/api_service/mono/logic/mod.rs new file mode 100644 index 000000000..01c8a128d --- /dev/null +++ b/ceres/src/application/api_service/mono/logic/mod.rs @@ -0,0 +1,9 @@ +//! Stateless monorepo path/tree/ref helpers for [`MonoApiService`](super::service::MonoApiService). + +mod path; +mod tree; + +pub(crate) use path::path_not_exist_re; + +/// Stateless logic helpers for monorepo operations (easy to unit test). +pub struct MonoServiceLogic; diff --git a/ceres/src/application/api_service/mono/logic/path.rs b/ceres/src/application/api_service/mono/logic/path.rs new file mode 100644 index 000000000..0d9a4d558 --- /dev/null +++ b/ceres/src/application/api_service/mono/logic/path.rs @@ -0,0 +1,295 @@ +use std::{ + path::{Path, PathBuf}, + sync::LazyLock, +}; + +use bytes::Bytes; +use common::{ + errors::{BuckError, MegaError}, + utils::ZERO_ID, +}; +use regex::Regex; + +use super::MonoServiceLogic; + +static PATH_NOT_EXIST_RE: LazyLock = LazyLock::new(|| { + Regex::new(r"Path '([^']+)' not exist").expect("PATH_NOT_EXIST_RE must be valid") +}); + +pub(crate) fn path_not_exist_re() -> &'static Regex { + &PATH_NOT_EXIST_RE +} + +impl MonoServiceLogic { + pub fn clean_path_str(path: &str) -> String { + let s = path.trim_end_matches('/'); + if s.is_empty() { + "/".to_string() + } else { + s.to_string() + } + } + + /// Normalize and validate repository path. + /// + /// Rules: trim; reject empty or whitespace-only (validation error). Reject `..`, backslash, + /// Windows drive letters (e.g. `C:`), and paths starting with `:`. Strip trailing `/`; + /// input consisting only of slashes becomes `"/"`. Collapse middle repeated slashes and + /// remove `.` segments (e.g. `//project//foo` -> `/project/foo`, `project/./foo` -> `/project/foo`). + /// Paths that consist only of `.` and slashes (e.g. `"."`, `"./"`) are rejected so they do not + /// silently resolve to root. Non-empty result gets a leading `"/"` if missing. Result matches + /// mega_refs.path format. + pub fn normalize_repo_path(path: &str) -> Result { + let s = path.trim(); + if s.is_empty() { + return Err(MegaError::Buck(BuckError::ValidationError( + "Path cannot be empty".to_string(), + ))); + } + if s.contains("..") { + return Err(MegaError::Buck(BuckError::ValidationError(format!( + "Path traversal not allowed: {}", + s + )))); + } + if s.contains('\\') { + return Err(MegaError::Buck(BuckError::ValidationError(format!( + "Path must use '/' separator: {}", + s + )))); + } + if s.len() >= 2 { + let mut chars = s.chars(); + if let (Some(c1), Some(':')) = (chars.next(), chars.next()) + && c1.is_ascii_alphabetic() + { + return Err(MegaError::Buck(BuckError::ValidationError(format!( + "Absolute path not allowed (Windows drive letter detected): {}", + s + )))); + } + } + if s.starts_with(':') { + return Err(MegaError::Buck(BuckError::ValidationError( + "Path must not start with ':'".to_string(), + ))); + } + let s = s.trim_end_matches('/'); + if s.is_empty() { + return Ok("/".to_string()); + } + let parts: Vec<&str> = s + .split('/') + .filter(|p| !p.is_empty() && *p != ".") + .collect(); + let s = parts.join("/"); + if s.is_empty() { + return Err(MegaError::Buck(BuckError::ValidationError( + "Path cannot be empty or consist only of '.' segments".to_string(), + ))); + } + Ok(format!("/{}", s)) + } + + /// Validate a GitHub sync target path (`/third-party/...` or `/project/...` subdirectories). + pub fn validate_github_sync_path(path: &str) -> Result { + let normalized = Self::normalize_repo_path(path)?; + if normalized == "/third-party" || normalized == "/project" { + return Err(MegaError::Buck(BuckError::ValidationError( + "GitHub sync path must be a subdirectory under /third-party or /project" + .to_string(), + ))); + } + if normalized.starts_with("/third-party/") || normalized.starts_with("/project/") { + return Ok(normalized); + } + Err(MegaError::Buck(BuckError::ValidationError( + "GitHub sync path must start with /third-party/ or /project/".to_string(), + ))) + } + + /// Returns true when a receive-pack status report contains a failed ref update (`ng`). + pub fn receive_pack_report_failed(report: &Bytes) -> bool { + String::from_utf8_lossy(report).contains("ng refs/") + } + + /// True when a CL represents a brand-new monorepo subdirectory (no prior main ref). + pub fn is_new_directory_cl(from_hash: &str) -> bool { + from_hash == ZERO_ID + } + + /// Enumerate candidate repo roots from the deepest directory back to `/`. + pub fn repo_root_candidates(path: &Path) -> Vec { + let mut current = PathBuf::from("/").join(path); + let mut candidates = Vec::new(); + + loop { + candidates.push(Self::clean_path_str(¤t.to_string_lossy())); + if !current.pop() { + break; + } + } + + candidates + } + + pub fn subtree_ref_path(path: &Path) -> Result { + Self::normalize_repo_path(&path.display().to_string()) + } +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use common::errors::{BuckError, MegaError}; + + use super::MonoServiceLogic; + + #[test] + fn test_clean_path_str_edges() { + assert_eq!(MonoServiceLogic::clean_path_str(""), "/"); + assert_eq!(MonoServiceLogic::clean_path_str("/"), "/"); + assert_eq!(MonoServiceLogic::clean_path_str("abc/"), "abc"); + assert_eq!(MonoServiceLogic::clean_path_str("abc///"), "abc"); + } + + #[test] + fn test_normalize_repo_path() { + // Normalization: add leading slash, strip trailing + assert_eq!( + MonoServiceLogic::normalize_repo_path("project").unwrap(), + "/project" + ); + assert_eq!( + MonoServiceLogic::normalize_repo_path("/project").unwrap(), + "/project" + ); + assert_eq!( + MonoServiceLogic::normalize_repo_path("project/").unwrap(), + "/project" + ); + assert_eq!( + MonoServiceLogic::normalize_repo_path("/project/").unwrap(), + "/project" + ); + assert_eq!( + MonoServiceLogic::normalize_repo_path(" /project ").unwrap(), + "/project" + ); + assert_eq!(MonoServiceLogic::normalize_repo_path("/").unwrap(), "/"); + + // Empty / whitespace-only -> ValidationError + assert!(MonoServiceLogic::normalize_repo_path("").is_err()); + assert!(MonoServiceLogic::normalize_repo_path(" ").is_err()); + assert!(matches!( + MonoServiceLogic::normalize_repo_path(""), + Err(MegaError::Buck(BuckError::ValidationError(_))) + )); + + // Path traversal and invalid chars -> ValidationError + assert!(MonoServiceLogic::normalize_repo_path("project/../foo").is_err()); + assert!(MonoServiceLogic::normalize_repo_path("project\\foo").is_err()); + + // Middle slashes and "." segments are collapsed + assert_eq!( + MonoServiceLogic::normalize_repo_path("//project//foo//").unwrap(), + "/project/foo" + ); + assert_eq!( + MonoServiceLogic::normalize_repo_path("project/./foo").unwrap(), + "/project/foo" + ); + assert_eq!( + MonoServiceLogic::normalize_repo_path("/project/./foo").unwrap(), + "/project/foo" + ); + + // Dot-only paths are rejected (do not silently resolve to root) + assert!(matches!( + MonoServiceLogic::normalize_repo_path("."), + Err(MegaError::Buck(BuckError::ValidationError(_))) + )); + assert!(matches!( + MonoServiceLogic::normalize_repo_path("./"), + Err(MegaError::Buck(BuckError::ValidationError(_))) + )); + assert!(matches!( + MonoServiceLogic::normalize_repo_path("./."), + Err(MegaError::Buck(BuckError::ValidationError(_))) + )); + + // Leading colon is rejected + assert!(matches!( + MonoServiceLogic::normalize_repo_path(":/test"), + Err(MegaError::Buck(BuckError::ValidationError(_))) + )); + assert!(matches!( + MonoServiceLogic::normalize_repo_path(":"), + Err(MegaError::Buck(BuckError::ValidationError(_))) + )); + + // Windows drive letters are rejected + assert!(matches!( + MonoServiceLogic::normalize_repo_path("C:"), + Err(MegaError::Buck(BuckError::ValidationError(_))) + )); + assert!(matches!( + MonoServiceLogic::normalize_repo_path("D:/project"), + Err(MegaError::Buck(BuckError::ValidationError(_))) + )); + } + + #[test] + fn test_repo_root_candidates_walk_from_leaf_to_root() { + assert_eq!( + MonoServiceLogic::repo_root_candidates(Path::new("/project/buck2_test/src")), + vec![ + "/project/buck2_test/src".to_string(), + "/project/buck2_test".to_string(), + "/project".to_string(), + "/".to_string(), + ] + ); + } + + #[test] + fn test_repo_root_candidates_normalize_relative_paths() { + assert_eq!( + MonoServiceLogic::repo_root_candidates(Path::new("project/buck2_test/src")), + vec![ + "/project/buck2_test/src".to_string(), + "/project/buck2_test".to_string(), + "/project".to_string(), + "/".to_string(), + ] + ); + } + + #[test] + fn test_subtree_ref_path_keeps_parent_directory_for_file_edits() { + assert_eq!( + MonoServiceLogic::subtree_ref_path(Path::new("/project/buck2_test/src")).unwrap(), + "/project/buck2_test/src".to_string() + ); + } + + #[test] + fn test_subtree_ref_path_normalizes_relative_create_paths() { + assert_eq!( + MonoServiceLogic::subtree_ref_path(Path::new("project/buck2_test/src")).unwrap(), + "/project/buck2_test/src".to_string() + ); + } + + #[test] + fn test_path_traversal_with_pop() { + let mut full_path = PathBuf::from("/project/rust/mega"); + for _ in 0..3 { + let cloned_path = full_path.clone(); + let name = cloned_path.file_name().unwrap().to_str().unwrap(); + full_path.pop(); + println!("name: {name}, path: {full_path:?}"); + } + } +} diff --git a/ceres/src/application/api_service/mono/logic/tree.rs b/ceres/src/application/api_service/mono/logic/tree.rs new file mode 100644 index 000000000..d299165f0 --- /dev/null +++ b/ceres/src/application/api_service/mono/logic/tree.rs @@ -0,0 +1,380 @@ +use std::{collections::HashMap, path::PathBuf, str::FromStr, sync::Arc}; + +use callisto::mega_refs; +use common::utils::MEGA_BRANCH_NAME; +use git_internal::{ + errors::GitError, + hash::ObjectHash, + internal::object::{ + commit::Commit, + tree::{Tree, TreeItem}, + }, +}; +use jupiter::storage::mono_storage::RefUpdateData; + +use super::MonoServiceLogic; +use crate::api_service::mono::types::{RefUpdate, TreeUpdateResult}; + +impl MonoServiceLogic { + pub fn update_tree_hash( + tree: Arc, + name: &str, + target_hash: ObjectHash, + ) -> Result { + let index = tree + .tree_items + .iter() + .position(|item| item.name == name) + .ok_or_else(|| GitError::CustomError(format!("Tree item '{}' not found", name)))?; + let mut items = tree.tree_items.clone(); + items[index].id = target_hash; + Tree::from_tree_items(items).map_err(|_| GitError::CustomError("Invalid tree".to_string())) + } + + /// Walk an update chain from leaf to root, returning rebuilt trees and the new root tree id. + pub fn propagate_tree_chain( + mut path: PathBuf, + mut update_chain: Vec>, + mut updated_tree_hash: ObjectHash, + ) -> Result<(Vec, ObjectHash), GitError> { + let mut updated_trees = Vec::new(); + while let Some(tree) = update_chain.pop() { + let cloned_path = path.clone(); + let name = cloned_path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| GitError::CustomError("Invalid path".into()))?; + path.pop(); + + let new_tree = Self::update_tree_hash(tree, name, updated_tree_hash)?; + updated_tree_hash = new_tree.id; + updated_trees.push(new_tree); + } + Ok((updated_trees, updated_tree_hash)) + } + + /// Update parent trees along the given update chain with the new child tree hash. + /// This function prepares all updated trees and their associated ref updates. + /// Trees that do not depend on each other (e.g., sibling directories) can be updated in parallel. + /// No new commits are created; only tree objects and ref updates are produced. + pub fn build_result_by_chain( + mut path: PathBuf, + mut update_chain: Vec>, + mut updated_tree_hash: ObjectHash, + ) -> Result { + let mut updated_trees = Vec::new(); + let mut ref_updates = Vec::new(); + let mut path_str = path.to_string_lossy().to_string(); + + loop { + let clean_path = MonoServiceLogic::clean_path_str(&path_str); + let ref_path = if clean_path == "/" || clean_path.starts_with('/') { + clean_path + } else { + format!("/{clean_path}") + }; + + ref_updates.push(RefUpdate { + path: ref_path, + tree_id: updated_tree_hash, + }); + + if update_chain.is_empty() { + break; + } + + let cloned_path = path.clone(); + let name = cloned_path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| GitError::CustomError("Invalid path".into()))?; + path.pop(); + path_str = path.to_string_lossy().to_string(); + + let tree = update_chain + .pop() + .ok_or_else(|| GitError::CustomError("Empty update chain".into()))?; + + let new_tree = MonoServiceLogic::update_tree_hash(tree, name, updated_tree_hash)?; + updated_tree_hash = new_tree.id; + updated_trees.push(new_tree); + } + + Ok(TreeUpdateResult { + updated_trees, + ref_updates, + }) + } + + /// Processes all ref updates by creating new commits and updating refs accordingly. + /// + /// This method abstracts the entire loop logic for processing ref updates, + /// creating commits for each update and managing the refs that need to be updated. + pub fn process_ref_updates( + result: &TreeUpdateResult, + refs: &[mega_refs::Model], + commit_msg: &str, + commits: &mut Vec, + updates: &mut Vec, + new_commit_id: &mut String, + ) -> Result<(), GitError> { + for update in &result.ref_updates { + let path_refs: Vec<&mega_refs::Model> = + refs.iter().filter(|r| r.path == update.path).collect(); + let p_ref = path_refs + .iter() + .find(|r| r.ref_name.starts_with("refs/cl/")) + .copied() + .or_else(|| { + path_refs + .iter() + .find(|r| r.ref_name == MEGA_BRANCH_NAME) + .copied() + }); + let Some(p_ref) = p_ref else { + continue; + }; + let commit = Commit::from_tree_id( + update.tree_id, + vec![ObjectHash::from_str(&p_ref.ref_commit_hash).unwrap()], + commit_msg, + ); + let commit_id = commit.id.to_string(); + *new_commit_id = commit_id.clone(); + + commits.push(commit); + + let mut push_update = |ref_name: &str| { + updates.push(RefUpdateData { + path: p_ref.path.clone(), + ref_name: ref_name.to_string(), + commit_id: commit_id.to_string(), + tree_hash: update.tree_id.to_string(), + }); + }; + + push_update(&p_ref.ref_name); + if p_ref.ref_name.starts_with("refs/cl/") { + push_update(MEGA_BRANCH_NAME); + } + } + + Ok(()) + } + + /// Processes ref updates but only for CL refs; never touches main and supports chaining parents. + pub fn process_ref_updates_cl_only( + result: &TreeUpdateResult, + cl_ref: &mega_refs::Model, + commit_msg: &str, + parent_override: Option, + commits: &mut Vec, + updates: &mut Vec, + new_commit_id: &mut String, + ) -> Result<(), GitError> { + let mut prev_parent: Option = None; + + for update in &result.ref_updates { + let parent_ids = if let Some(prev) = prev_parent { + vec![prev] + } else if let Some(po) = parent_override { + vec![po] + } else { + vec![ObjectHash::from_str(&cl_ref.ref_commit_hash).map_err(|_| { + GitError::CustomError(format!( + "Invalid CL ref hash: {}", + cl_ref.ref_commit_hash + )) + })?] + }; + + let commit = Commit::from_tree_id(update.tree_id, parent_ids, commit_msg); + let commit_id = commit.id; + *new_commit_id = commit_id.to_string(); + + commits.push(commit.clone()); + prev_parent = Some(commit_id); + + updates.push(RefUpdateData { + path: cl_ref.path.clone(), + ref_name: cl_ref.ref_name.clone(), + commit_id: commit_id.to_string(), + tree_hash: update.tree_id.to_string(), + }); + } + + Ok(()) + } + + /// Maps each TreeItem in a Tree to its corresponding Commit, if available. + /// + /// # Arguments + /// + /// * `tree` - The tree containing the TreeItems to map. + /// * `item_to_commit_id` - Mapping from TreeItem id (as string) to commit id. + /// * `commit_map` - Mapping from commit id to Commit object. + /// + /// # Returns + /// + /// A HashMap where each TreeItem maps to an Option. If a commit cannot + /// be found, the value is None. + pub fn map_tree_items_to_commits( + tree: Tree, + item_to_commit_id: &HashMap, + commit_map: &HashMap, + ) -> HashMap> { + let mut result: HashMap> = HashMap::new(); + + for item in tree.tree_items { + if let Some(commit_id) = item_to_commit_id.get(&item.id.to_string()) { + let commit = commit_map.get(commit_id).cloned(); + if commit.is_none() { + tracing::warn!( + item_name = %item.name, + item_mode = ?item.mode, + commit_id = %commit_id, + "failed fetch from commit map" + ); + } + result.insert(item, commit); + } else { + result.insert(item, None); + } + } + result + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, path::PathBuf, str::FromStr, sync::Arc}; + + use git_internal::{ + hash::ObjectHash, + internal::object::{ + commit::Commit, + signature::{Signature, SignatureType}, + tree::{Tree, TreeItem, TreeItemMode}, + }, + }; + + use super::MonoServiceLogic; + + #[test] + fn test_update_tree_hash() { + let item = TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + "path".to_string(), + ); + + let tree = Tree::from_tree_items(vec![item]).expect("tree should build"); + let tree = Arc::new(tree); + + let new_hash = ObjectHash::from_str("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap(); + + let new_tree = MonoServiceLogic::update_tree_hash(tree, "path", new_hash) + .expect("update_tree_hash should succeed"); + + assert_eq!(new_tree.tree_items.len(), 1); + assert_eq!(new_tree.tree_items[0].id, new_hash); + } + + #[test] + fn test_build_result_by_chain_logic() { + let item = TreeItem::new( + TreeItemMode::Blob, + ObjectHash::from_str("1234567890123456789012345678901234567890").unwrap(), + "path".to_string(), + ); + + let tree = Tree::from_tree_items(vec![item]).expect("tree should build"); + let tree_id = tree.id; + + let update_chain = vec![Arc::new(tree)]; + let path = PathBuf::from("/test/path"); + + let result = MonoServiceLogic::build_result_by_chain(path, update_chain, tree_id) + .expect("build_result_by_chain should succeed"); + + assert_eq!(result.updated_trees.len(), 1); + assert_eq!(result.ref_updates.len(), 2); + + let paths: Vec<&str> = result.ref_updates.iter().map(|r| r.path.as_str()).collect(); + assert!(paths.contains(&"/test/path")); + assert!(paths.contains(&"/test")); + } + + #[test] + fn test_build_result_by_chain_normalizes_relative_paths_for_ref_updates() { + let old_hash = ObjectHash::from_str("1111111111111111111111111111111111111111").unwrap(); + let updated_child_hash = + ObjectHash::from_str("2222222222222222222222222222222222222222").unwrap(); + let item = TreeItem::new(TreeItemMode::Tree, old_hash, "src".to_string()); + let tree = Tree::from_tree_items(vec![item]).expect("tree should build"); + let update_chain = vec![Arc::new(tree)]; + + let result = MonoServiceLogic::build_result_by_chain( + PathBuf::from("project/buck2_test/src"), + update_chain, + updated_child_hash, + ) + .expect("build_result_by_chain should succeed"); + + let paths: Vec<&str> = result.ref_updates.iter().map(|r| r.path.as_str()).collect(); + assert!(paths.contains(&"/project/buck2_test/src")); + assert!(paths.contains(&"/project/buck2_test")); + } + + #[test] + fn test_map_tree_items_to_commits() { + let id1 = ObjectHash::Sha1([1u8; 20]); + let id2 = ObjectHash::Sha1([2u8; 20]); + let commit_hash = ObjectHash::Sha1([3u8; 20]); + + let item1 = TreeItem { + id: id1, + name: "file1.txt".into(), + mode: TreeItemMode::Blob, + }; + let item2 = TreeItem { + id: id2, + name: "file2.txt".into(), + mode: TreeItemMode::Blob, + }; + + let tree = Tree { + id: ObjectHash::Sha1([9u8; 20]), + tree_items: vec![item1.clone(), item2.clone()], + }; + + let mut item_to_commit_id = HashMap::new(); + item_to_commit_id.insert(id1.to_string(), commit_hash.to_string()); + + let fake_sig = Signature { + signature_type: SignatureType::Committer, + name: "tester".into(), + email: "tester@example.com".into(), + timestamp: 0, + timezone: "+0000".into(), + }; + + let commit_a = Commit { + id: commit_hash, + tree_id: ObjectHash::Sha1([8u8; 20]), + parent_commit_ids: vec![], + author: fake_sig.clone(), + committer: fake_sig.clone(), + message: "test commit".into(), + }; + + let mut commit_map = HashMap::new(); + commit_map.insert(commit_hash.to_string(), commit_a.clone()); + + let result = + MonoServiceLogic::map_tree_items_to_commits(tree, &item_to_commit_id, &commit_map); + + assert_eq!(result.get(&item1), Some(&Some(commit_a))); + assert_eq!(result.get(&item2), Some(&None)); + } +} diff --git a/ceres/src/application/api_service/mono/mod.rs b/ceres/src/application/api_service/mono/mod.rs new file mode 100644 index 000000000..927e0d215 --- /dev/null +++ b/ceres/src/application/api_service/mono/mod.rs @@ -0,0 +1,19 @@ +//! Monorepo API implementation (`MonoApiService` and domain modules). + +pub mod admin; +pub mod logic; +pub mod service; +pub mod types; + +pub mod buck; +pub mod cl; +pub mod cla; +pub mod edit; +pub mod sync; +pub mod tag; + +pub use admin::{ADMIN_FILE, EffectiveResourcePermission}; +pub use cl::merge_strategy as cl_merge; +pub use logic::MonoServiceLogic; +pub use service::MonoApiService; +pub use types::{RefUpdate, TreeUpdateResult}; diff --git a/ceres/src/application/api_service/mono/service.rs b/ceres/src/application/api_service/mono/service.rs new file mode 100644 index 000000000..915c87649 --- /dev/null +++ b/ceres/src/application/api_service/mono/service.rs @@ -0,0 +1,228 @@ +//! # Mono API Service +//! +//! Thin facade over monorepo operation modules. See sibling `mono_*_ops` modules for +//! CLA, buck upload, merge queue, sync, tags, entries, diffs, branch updates, and merges. + +use std::{ + collections::{HashMap, HashSet}, + path::{Path, PathBuf}, + sync::Arc, +}; + +use api_model::common::Pagination; +use async_trait::async_trait; +use common::errors::MegaError; +use git_internal::{ + errors::GitError, + internal::object::{ + commit::Commit, + tree::{Tree, TreeItem, TreeItemMode}, + }, +}; +use jupiter::{storage::Storage, utils::converter::FromMegaModel}; + +use super::logic::MonoServiceLogic; +use crate::{ + api_service::{ApiHandler, cache::GitObjectCache, tree_ops}, + model::git::{CreateEntryInfo, CreateEntryResult, EditFilePayload, EditFileResult}, + pack::{import_repo::ImportRepo, monorepo::MonoRepo}, +}; + +#[derive(Clone)] +pub struct MonoApiService { + pub storage: Storage, + pub git_object_cache: Arc, +} + +impl From<&MonoRepo> for MonoApiService { + fn from(mono_repo: &MonoRepo) -> Self { + MonoApiService { + storage: mono_repo.storage.clone(), + git_object_cache: mono_repo.git_object_cache.clone(), + } + } +} + +impl From<&ImportRepo> for MonoApiService { + fn from(import_repo: &ImportRepo) -> Self { + MonoApiService { + storage: import_repo.storage.clone(), + git_object_cache: import_repo.git_object_cache.clone(), + } + } +} + +#[async_trait] +impl ApiHandler for MonoApiService { + fn get_context(&self) -> Storage { + self.storage.clone() + } + + fn object_cache(&self) -> &GitObjectCache { + &self.git_object_cache + } + + async fn get_root_commit(&self) -> Result { + let storage = self.storage.mono_storage(); + let refs = storage.get_main_ref("/").await.unwrap().unwrap(); + self.get_commit_by_hash(&refs.ref_commit_hash).await + } + + async fn save_file_edit(&self, payload: EditFilePayload) -> Result { + self.save_file_edit_impl(payload).await + } + + async fn create_monorepo_entry( + &self, + entry_info: CreateEntryInfo, + ) -> Result { + self.create_monorepo_entry_impl(entry_info).await + } + + fn strip_relative(&self, path: &Path) -> Result { + Ok(path.to_path_buf()) + } + + async fn get_root_tree(&self, refs: Option<&str>) -> Result { + let refs = refs.unwrap_or("").trim(); + + if refs.is_empty() { + let storage = self.storage.mono_storage(); + let refs = storage.get_main_ref("/").await.unwrap().unwrap(); + return self.get_tree_by_hash(&refs.ref_tree_hash).await; + } + + if refs.len() == 40 && refs.chars().all(|c| c.is_ascii_hexdigit()) { + let commit = self.get_commit_by_hash(refs).await?; + return self.get_tree_by_hash(&commit.tree_id.to_string()).await; + } + + if let Ok(Some(tag)) = self.get_tag(None, refs.to_string()).await { + let commit = self.get_commit_by_hash(&tag.object_id).await?; + return self.get_tree_by_hash(&commit.tree_id.to_string()).await; + } + + Err(MegaError::Other(format!( + "Invalid refs: '{}' is not a valid commit hash or tag", + refs + ))) + } + + async fn get_tree_by_hash(&self, hash: &str) -> Result { + let model = self + .storage + .mono_storage() + .get_tree_by_hash(hash) + .await? + .ok_or_else(|| MegaError::NotFound(format!("tree not found: {}", hash)))?; + Ok(Tree::from_mega_model(model)) + } + + async fn get_commit_by_hash(&self, hash: &str) -> Result { + let model = self + .storage + .mono_storage() + .get_commit_by_hash(hash) + .await? + .ok_or_else(|| MegaError::NotFound(format!("commit not found: {}", hash)))?; + Ok(Commit::from_mega_model(model)) + } + + async fn item_to_commit_map( + &self, + path: PathBuf, + reference: Option<&str>, + ) -> Result>, GitError> { + match tree_ops::search_tree_by_path(self, &path, reference).await? { + Some(tree) => { + let mut item_to_commit = HashMap::new(); + + let storage = self.storage.mono_storage(); + let tree_hashes = tree + .tree_items + .iter() + .filter(|x| x.mode == TreeItemMode::Tree) + .map(|x| x.id.to_string()) + .collect(); + let trees = storage.get_trees_by_hashes(tree_hashes).await.unwrap(); + for tree in trees { + if !tree.commit_id.is_empty() { + item_to_commit.insert(tree.tree_id, tree.commit_id); + } + } + + let blob_hashes = tree + .tree_items + .iter() + .filter(|x| x.mode == TreeItemMode::Blob) + .map(|x| x.id.to_string()) + .collect(); + let blobs = storage.get_mega_blobs_by_hashes(blob_hashes).await.unwrap(); + for blob in blobs { + if !blob.commit_id.is_empty() { + item_to_commit.insert(blob.blob_id, blob.commit_id); + } + } + + let commit_ids: HashSet = item_to_commit.values().cloned().collect(); + let commits = self + .get_commits_by_hashes(commit_ids.into_iter().collect()) + .await + .unwrap(); + + let commit_map: HashMap = + commits.into_iter().map(|x| (x.id.to_string(), x)).collect(); + + Ok(MonoServiceLogic::map_tree_items_to_commits( + tree, + &item_to_commit, + &commit_map, + )) + } + None => Ok(HashMap::new()), + } + } + + async fn get_commits_by_hashes(&self, c_hashes: Vec) -> Result, GitError> { + let commits = self + .storage + .mono_storage() + .get_commits_by_hashes(&c_hashes) + .await + .unwrap(); + Ok(commits.into_iter().map(Commit::from_mega_model).collect()) + } + + async fn create_tag( + &self, + repo_path: Option, + name: String, + target: Option, + tagger_name: Option, + tagger_email: Option, + message: Option, + ) -> Result { + self.create_tag_impl(repo_path, name, target, tagger_name, tagger_email, message) + .await + } + + async fn list_tags( + &self, + repo_path: Option, + pagination: Pagination, + ) -> Result<(Vec, u64), GitError> { + self.list_tags_impl(repo_path, pagination).await + } + + async fn get_tag( + &self, + repo_path: Option, + name: String, + ) -> Result, GitError> { + self.get_tag_impl(repo_path, name).await + } + + async fn delete_tag(&self, repo_path: Option, name: String) -> Result<(), GitError> { + self.delete_tag_impl(repo_path, name).await + } +} diff --git a/ceres/src/application/api_service/mono/sync.rs b/ceres/src/application/api_service/mono/sync.rs new file mode 100644 index 000000000..8f41ebc4a --- /dev/null +++ b/ceres/src/application/api_service/mono/sync.rs @@ -0,0 +1,244 @@ +//! Third-party sync and tree traversal for [`MonoApiService`](super::service::MonoApiService). + +use std::{path::PathBuf, str::FromStr, sync::Arc}; + +use bytes::Bytes; +use common::{errors::MegaError, utils::ZERO_ID}; +use git_internal::{ + hash::ObjectHash, + internal::object::{commit::Commit, tree::Tree}, +}; +use jupiter::{redis::lock::RedLock, utils::converter::FromMegaModel}; + +use crate::{ + api_service::{ + mono::{MonoApiService, MonoServiceLogic}, + state::ProtocolApiState, + tree_ops, + }, + model::third_party::{ThirdPartyClient, ThirdPartyRepoTrait}, + pack::into_pack_byte_stream, + protocol::{PushUserInfo, ServiceType, SmartSession, TransportProtocol}, +}; + +impl MonoApiService { + /// Attach a new `/project/*` path into the monorepo root tree (placeholder `.gitkeep` dirs). + pub async fn attach_project_path_to_monorepo_root(&self, path: &str) -> Result<(), MegaError> { + const MAX_ATTACH_ATTEMPTS: u32 = 64; + const ROOT_LOCK_KEY: &str = "git:receive-pack:lock:monorepo-root"; + const ROOT_LOCK_TTL_MS: u64 = 30_000; + + let path_buf = PathBuf::from(path); + let storage = self.storage.mono_storage(); + let redlock = Arc::new(RedLock::new( + self.git_object_cache.connection.clone(), + ROOT_LOCK_KEY.to_string(), + ROOT_LOCK_TTL_MS, + )); + + for attempt in 0..MAX_ATTACH_ATTEMPTS { + let guard = redlock.clone().lock().await?; + let root_ref = storage + .get_main_ref("/") + .await? + .ok_or_else(|| MegaError::Other("root ref not found".to_string()))?; + let expected_commit = root_ref.ref_commit_hash.clone(); + let expected_tree = root_ref.ref_tree_hash.clone(); + let root_ref_id = root_ref.id; + + let save_trees = tree_ops::search_and_create_tree(self, &path_buf).await?; + let leaf_tree = save_trees + .back() + .ok_or_else(|| MegaError::Other("no tree generated".to_string()))?; + let commit_msg = format!("\nInitialize path {path} for project GitHub sync"); + let new_commit = Commit::from_tree_id( + leaf_tree.id, + vec![ + ObjectHash::from_str(&expected_commit) + .map_err(|e| MegaError::Other(format!("invalid root commit hash: {e}")))?, + ], + &commit_msg, + ); + + let txn = self.storage.begin_db_transaction().await?; + match storage + .attach_to_monorepo_parent_in_txn( + &txn, + root_ref_id, + &expected_commit, + &expected_tree, + new_commit, + save_trees.into(), + ) + .await + { + Ok(()) => { + txn.commit().await.map_err(MegaError::Db)?; + guard.unlock().await?; + crate::api_service::mono::cl_merge::sync_path_prefix_main_refs(self, path) + .await?; + return Ok(()); + } + Err(MegaError::StaleMonorepoRootRef) if attempt + 1 < MAX_ATTACH_ATTEMPTS => { + let _ = txn.rollback().await; + let _ = guard.unlock().await; + tracing::warn!( + attempt, + repo_path = %path, + "attach_project_path_to_monorepo_root: root ref moved, retrying" + ); + tokio::task::yield_now().await; + } + Err(e) => { + let _ = txn.rollback().await; + let _ = guard.unlock().await; + return Err(e); + } + } + } + + Err(MegaError::Other( + "attach_project_path_to_monorepo_root: exceeded retry limit".into(), + )) + } + + async fn resolve_sync_old_id( + &self, + repo_path_str: &str, + ref_name: &str, + ) -> Result { + let import_dir = self.storage.config().monorepo.import_dir.clone(); + if PathBuf::from(repo_path_str).starts_with(&import_dir) { + let storage = self.storage.git_db_storage(); + match storage.find_git_repo_exact_match(repo_path_str).await? { + Some(repo_model) => Ok(storage + .get_ref(repo_model.id) + .await? + .into_iter() + .find(|r| r.ref_name == ref_name) + .map(|r| r.ref_git_id) + .unwrap_or_else(|| ZERO_ID.to_string())), + None => Ok(ZERO_ID.to_string()), + } + } else { + let mono_storage = self.storage.mono_storage(); + if let Some(r) = mono_storage + .get_ref_at_path(repo_path_str, ref_name) + .await? + { + return Ok(r.ref_commit_hash); + } + // GitHub sync stores commits on CL refs (`refs/cl/*`), not `refs/heads/main`. + if let Some(cl_ref) = mono_storage + .get_all_refs(repo_path_str, false) + .await? + .into_iter() + .find(|r| r.is_cl) + { + return Ok(cl_ref.ref_commit_hash); + } + Ok(ZERO_ID.to_string()) + } + } + + pub async fn sync_third_party_repo( + &self, + owner: &str, + repo: &str, + mega_path: PathBuf, + username: &str, + ) -> Result { + let repo_path_str = mega_path + .to_str() + .ok_or_else(|| MegaError::Other("Invalid UTF-8 in mega_path".to_string()))?; + let repo_path_str = MonoServiceLogic::validate_github_sync_path(repo_path_str)?; + let mega_path = PathBuf::from(&repo_path_str); + + let url = format!("https://github.com/{owner}/{repo}.git"); + let remote_client = ThirdPartyClient::new(&url); + + let import_dir = self.storage.config().monorepo.import_dir.clone(); + let fetch_depth = if mega_path.starts_with(&import_dir) { + None + } else { + // MonoRepo receive-pack only accepts a single commit per push. + Some(1) + }; + + let (ref_name, ref_hash) = remote_client.fetch_refs().await?; + + let res = remote_client + .fetch_packs(std::slice::from_ref(&ref_hash), fetch_depth) + .await?; + let pack_data = remote_client + .process_pack_stream(res) + .await + .map_err(|e| MegaError::Other(format!("{e}")))?; + if pack_data.is_empty() { + return Err(MegaError::Other( + "GitHub sync failed: remote returned no pack data".to_string(), + )); + } + + let mut protocol = + SmartSession::new(mega_path, ServiceType::ReceivePack, TransportProtocol::Http); + protocol.auth.username = Some(username.to_string()); + protocol.auth.authenticated_user = Some(PushUserInfo { + username: username.to_string(), + }); + + let old_id = self.resolve_sync_old_id(&repo_path_str, &ref_name).await?; + + let commands = vec![crate::protocol::import_refs::RefCommand::new( + old_id, + ref_hash.clone(), + ref_name.clone(), + )]; + let state = ProtocolApiState::new(self.storage.clone(), self.git_object_cache.clone()); + let bytes = protocol + .git_receive_pack_stream( + &state, + commands, + into_pack_byte_stream(tokio_stream::once(Ok::( + Bytes::from(pack_data), + ))), + ) + .await + .map_err(|e| MegaError::Other(format!("{e}")))?; + + if MonoServiceLogic::receive_pack_report_failed(&bytes) { + return Err(MegaError::Other(format!( + "GitHub sync failed during receive-pack: {}", + String::from_utf8_lossy(&bytes) + ))); + } + + Ok(bytes) + } + + pub(crate) async fn traverse_tree( + &self, + root_tree: Tree, + ) -> Result, MegaError> { + let mut result = vec![]; + let mut stack = vec![(PathBuf::new(), root_tree)]; + + while let Some((base_path, tree)) = stack.pop() { + for item in tree.tree_items { + let path = base_path.join(&item.name); + if item.is_tree() { + let child = self + .storage + .mono_storage() + .get_tree_by_hash(&item.id.to_string()) + .await? + .unwrap(); + stack.push((path.clone(), Tree::from_mega_model(child))); + } else { + result.push((path, item.id)); + } + } + } + Ok(result) + } +} diff --git a/ceres/src/application/api_service/mono/tag.rs b/ceres/src/application/api_service/mono/tag.rs new file mode 100644 index 000000000..1a77a0b0e --- /dev/null +++ b/ceres/src/application/api_service/mono/tag.rs @@ -0,0 +1,346 @@ +//! Tag CRUD and helpers for [`MonoApiService`](super::service::MonoApiService). + +use api_model::common::Pagination; +use callisto::{mega_refs, mega_tag}; +use git_internal::errors::GitError; +use tracing; + +use crate::{ + api_service::{ + mono::MonoApiService, + tag_ops::{ + self, build_git_internal_tag, db_error, format_tagger_info, is_annotated_tag, + lightweight_commit_tag, merge_paginated_tags, tag_already_exists, tags_full_ref, + }, + }, + model::tag::TagInfo, +}; + +impl MonoApiService { + pub(crate) fn tag_model_to_info(&self, tag: mega_tag::Model) -> TagInfo { + TagInfo { + name: tag.tag_name, + tag_id: tag.tag_id, + object_id: tag.object_id, + object_type: tag.object_type, + tagger: tag.tagger, + message: tag.message, + created_at: tag.created_at.and_utc().to_rfc3339(), + } + } + + pub async fn create_tag_impl( + &self, + repo_path: Option, + name: String, + target: Option, + tagger_name: Option, + tagger_email: Option, + message: Option, + ) -> Result { + let mono_storage = self.storage.mono_storage(); + let tagger_info = format_tagger_info(tagger_name, tagger_email); + + self.validate_target_commit_mono(target.as_ref()).await?; + + let full_ref = tags_full_ref(&name); + + match mono_storage.get_tag_by_name(&name).await { + Ok(Some(_)) => return Err(tag_already_exists(&name)), + Ok(None) => {} + Err(e) => { + tracing::error!("DB error while checking tag existence: {}", e); + return Err(db_error()); + } + } + + if let Ok(Some(_)) = mono_storage.get_ref_by_name(&full_ref).await { + return Err(tag_already_exists(&name)); + } + + if is_annotated_tag(&message) { + return self + .create_annotated_tag_mono(repo_path, name, target, tagger_info, message, full_ref) + .await; + } + + self.create_lightweight_tag_mono(repo_path, name, target, tagger_info, full_ref) + .await + } + + pub async fn list_tags_impl( + &self, + repo_path: Option, + pagination: Pagination, + ) -> Result<(Vec, u64), GitError> { + let mono_storage = self.storage.mono_storage(); + let (annotated_page, annotated_total) = + match mono_storage.get_tags_by_page(pagination.clone()).await { + Ok(v) => v, + Err(e) => { + tracing::error!("DB error while listing tags: {}", e); + return Err(db_error()); + } + }; + + let annotated: Vec = annotated_page + .into_iter() + .map(|t| self.tag_model_to_info(t)) + .collect(); + + let repo_path = repo_path.as_deref().unwrap_or("/"); + let mut lightweight_refs = Vec::new(); + if let Ok(refs) = mono_storage.get_all_refs(repo_path, false).await { + for r in refs { + if r.ref_name.starts_with("refs/tags/") { + let tag_name = r.ref_name.trim_start_matches("refs/tags/").to_string(); + if annotated.iter().any(|t| t.name == tag_name) { + continue; + } + lightweight_refs.push(lightweight_commit_tag( + tag_name, + r.ref_commit_hash.clone(), + "", + r.created_at.and_utc().to_rfc3339(), + )); + } + } + } + + Ok(merge_paginated_tags( + annotated, + lightweight_refs, + annotated_total, + pagination.per_page, + )) + } + + pub async fn get_tag_impl( + &self, + _repo_path: Option, + name: String, + ) -> Result, GitError> { + let mono_storage = self.storage.mono_storage(); + match mono_storage.get_tag_by_name(&name).await { + Ok(Some(tag)) => return Ok(Some(self.tag_model_to_info(tag))), + Ok(None) => {} + Err(e) => { + tracing::error!("DB error while getting tag: {}", e); + return Err(db_error()); + } + } + + let full_ref = tags_full_ref(&name); + if let Ok(Some(r)) = mono_storage.get_ref_by_name(&full_ref).await { + return Ok(Some(lightweight_commit_tag( + name, + r.ref_commit_hash.clone(), + "", + r.created_at.and_utc().to_rfc3339(), + ))); + } + Ok(None) + } + + pub async fn delete_tag_impl( + &self, + _repo_path: Option, + name: String, + ) -> Result<(), GitError> { + let mono_storage = self.storage.mono_storage(); + match mono_storage.get_tag_by_name(&name).await { + Ok(Some(_tag)) => { + let full_ref = tags_full_ref(&name); + if let Ok(Some(r)) = mono_storage.get_ref_by_name(&full_ref).await { + mono_storage.remove_ref(r).await.map_err(|e| { + tracing::error!("Failed to remove ref while deleting annotated tag: {}", e); + GitError::CustomError("[code:500] Failed to remove ref".to_string()) + })?; + } + mono_storage.delete_tag_by_name(&name).await.map_err(|e| { + tracing::error!("DB delete error when deleting annotated tag: {}", e); + GitError::CustomError("[code:500] DB delete error".to_string()) + })?; + Ok(()) + } + Ok(None) => { + let full_ref = tags_full_ref(&name); + if let Ok(Some(r)) = mono_storage.get_ref_by_name(&full_ref).await { + mono_storage.remove_ref(r).await.map_err(|e| { + tracing::error!( + "Failed to remove ref while deleting lightweight tag: {}", + e + ); + GitError::CustomError("[code:500] Failed to remove ref".to_string()) + })?; + Ok(()) + } else { + Err(GitError::CustomError( + "[code:404] Tag not found".to_string(), + )) + } + } + Err(e) => { + tracing::error!("DB error while deleting tag: {}", e); + Err(db_error()) + } + } + } + + async fn create_annotated_tag_mono( + &self, + repo_path: Option, + name: String, + target: Option, + tagger_info: String, + message: Option, + full_ref: String, + ) -> Result { + let mono_storage = self.storage.mono_storage(); + + let (tag_id_hex, object_id) = + build_git_internal_tag(name.clone(), target, tagger_info.clone(), message.clone())?; + let tag_model = self.build_mega_tag_model( + tag_id_hex, + object_id.clone(), + name.clone(), + tagger_info, + message, + ); + + match mono_storage.insert_tag(tag_model).await { + Ok(saved_tag) => { + let path_str = repo_path.unwrap_or_else(|| "/".to_string()); + let tree_hash = self.resolve_tree_hash_for_commit(&object_id).await?; + let refs = mega_refs::Model::new(&path_str, full_ref, object_id, tree_hash, false); + + if let Err(e) = mono_storage.save_refs(refs, None).await { + if let Err(del_e) = mono_storage.delete_tag_by_name(&name).await { + tracing::error!( + "Failed to rollback tag DB record after ref write failure: {}", + del_e + ); + } + tracing::error!("Failed to write ref after DB insert: {}", e); + return Err(GitError::CustomError( + "[code:500] Failed to write ref".to_string(), + )); + } + Ok(self.tag_model_to_info(saved_tag)) + } + Err(e) => { + tracing::error!("DB insert error when creating annotated tag: {}", e); + Err(GitError::CustomError( + "[code:500] DB insert error".to_string(), + )) + } + } + } + + async fn create_lightweight_tag_mono( + &self, + repo_path: Option, + name: String, + target: Option, + tagger_info: String, + full_ref: String, + ) -> Result { + let mono_storage = self.storage.mono_storage(); + + let path_str = repo_path.unwrap_or_else(|| "/".to_string()); + let object_id = target.unwrap_or_default(); + if object_id.is_empty() { + return Err(GitError::CustomError( + "[code:400] Missing target commit for lightweight tag".to_string(), + )); + } + let tree_hash = self.resolve_tree_hash_for_commit(&object_id).await?; + + let refs = mega_refs::Model::new( + &path_str, + full_ref.clone(), + object_id.clone(), + tree_hash, + false, + ); + mono_storage.save_refs(refs, None).await.map_err(|e| { + tracing::error!("Failed to write lightweight tag ref: {}", e); + GitError::CustomError("[code:500] Failed to write lightweight tag ref".to_string()) + })?; + + let saved_ref = mono_storage + .get_ref_by_name(&full_ref) + .await + .map_err(|e| GitError::CustomError(e.to_string()))? + .ok_or_else(|| GitError::CustomError("Ref not found after creation".to_string()))?; + + Ok(lightweight_commit_tag( + name, + object_id, + tagger_info, + saved_ref.created_at.and_utc().to_rfc3339(), + )) + } + + async fn resolve_tree_hash_for_commit(&self, commit_id: &str) -> Result { + let mono_storage = self.storage.mono_storage(); + match mono_storage.get_commit_by_hash(commit_id).await { + Ok(Some(commit_model)) => Ok(commit_model.tree.clone()), + Ok(None) => { + tracing::error!( + "Target commit '{}' not found while resolving tree hash", + commit_id + ); + Err(tag_ops::commit_not_found(commit_id)) + } + Err(e) => { + tracing::error!( + "DB error fetching commit '{}' for tree hash resolution: {}", + commit_id, + e + ); + Err(db_error()) + } + } + } + + async fn validate_target_commit_mono(&self, target: Option<&String>) -> Result<(), GitError> { + let mono_storage = self.storage.mono_storage(); + if let Some(t) = target { + match mono_storage.get_commit_by_hash(t).await { + Ok(commit_opt) => { + if commit_opt.is_none() { + return Err(tag_ops::commit_not_found(t)); + } + } + Err(e) => { + tracing::error!("DB error while fetching commit by hash: {}", e); + return Err(db_error()); + } + } + } + Ok(()) + } + + fn build_mega_tag_model( + &self, + tag_id_hex: String, + object_id: String, + name: String, + tagger_info: String, + message: Option, + ) -> mega_tag::Model { + mega_tag::Model { + id: common::utils::generate_id(), + tag_id: tag_id_hex, + object_id, + object_type: "commit".to_string(), + tag_name: name, + tagger: tagger_info, + message: message.unwrap_or_default(), + pack_id: String::new(), + pack_offset: 0, + created_at: chrono::Utc::now().naive_utc(), + } + } +} diff --git a/ceres/src/application/api_service/mono/types.rs b/ceres/src/application/api_service/mono/types.rs new file mode 100644 index 000000000..fb6517137 --- /dev/null +++ b/ceres/src/application/api_service/mono/types.rs @@ -0,0 +1,32 @@ +use std::{collections::HashMap, path::PathBuf}; + +use git_internal::{ + hash::ObjectHash, + internal::object::{blob::Blob, tree::Tree}, +}; + +pub struct TreeUpdateResult { + pub updated_trees: Vec, + pub ref_updates: Vec, +} + +pub struct RefUpdate { + pub path: String, + pub tree_id: ObjectHash, +} + +pub(crate) struct CreateEntryUpdate { + pub update_result: TreeUpdateResult, + pub blob: Blob, + pub entry_oid: ObjectHash, + pub repo_path: PathBuf, + pub save_trees: Vec, +} + +pub(crate) struct ApplyChangeContext<'a> { + pub components: &'a [String], + pub chain_paths: &'a [PathBuf], + pub chain_trees: &'a [Tree], + pub tree_cache: &'a mut HashMap, + pub new_trees: &'a mut HashMap, +} diff --git a/ceres/src/application/api_service/state.rs b/ceres/src/application/api_service/state.rs new file mode 100644 index 000000000..ced05063f --- /dev/null +++ b/ceres/src/application/api_service/state.rs @@ -0,0 +1,2 @@ +//! Backward-compatible re-export; prefer [`crate::transport::ProtocolApiState`] or [`crate::bus::TransportRuntime`]. +pub use crate::transport::ProtocolApiState; diff --git a/ceres/src/application/api_service/tag_ops.rs b/ceres/src/application/api_service/tag_ops.rs new file mode 100644 index 000000000..c2cc9c46d --- /dev/null +++ b/ceres/src/application/api_service/tag_ops.rs @@ -0,0 +1,138 @@ +//! Shared tag helpers used by mono and import API services. + +use std::str::FromStr; + +use git_internal::{ + errors::GitError, + hash::ObjectHash, + internal::object::{ + signature::{Signature, SignatureType}, + tag::Tag, + types::ObjectType, + }, +}; + +use crate::model::tag::TagInfo; + +pub fn format_tagger_info(tagger_name: Option, tagger_email: Option) -> String { + match (tagger_name, tagger_email) { + (Some(n), Some(e)) => format!("{n} <{e}>"), + (Some(n), None) => n, + (None, Some(e)) => e, + (None, None) => "unknown".to_string(), + } +} + +pub fn is_annotated_tag(message: &Option) -> bool { + message.as_ref().is_some_and(|s| !s.is_empty()) +} + +pub fn tags_full_ref(name: &str) -> String { + format!("refs/tags/{name}") +} + +pub fn tag_already_exists(name: &str) -> GitError { + GitError::CustomError(format!("[code:400] Tag '{name}' already exists")) +} + +pub fn commit_not_found(commit_id: &str) -> GitError { + GitError::CustomError(format!("[code:404] Target commit '{commit_id}' not found")) +} + +pub fn db_error() -> GitError { + GitError::CustomError("[code:500] DB error".to_string()) +} + +/// Build git-internal tag id and resolved object id from create-tag inputs. +pub fn build_git_internal_tag( + name: String, + target: Option, + tagger_info: String, + message: Option, +) -> Result<(String, String), GitError> { + let tag_target = target + .as_ref() + .ok_or(GitError::InvalidCommitObject) + .and_then(|t| ObjectHash::from_str(t).map_err(|_| GitError::InvalidCommitObject))?; + let tagger_sig = Signature::new(SignatureType::Tagger, tagger_info, String::new()); + let git_internal_tag = Tag::new( + tag_target, + ObjectType::Commit, + name, + tagger_sig, + message.unwrap_or_default(), + ); + Ok(( + git_internal_tag.id.to_string(), + target.unwrap_or_else(|| "HEAD".to_string()), + )) +} + +pub fn lightweight_commit_tag( + name: impl Into, + object_id: impl Into, + tagger_info: impl Into, + created_at: impl Into, +) -> TagInfo { + let name = name.into(); + let object_id = object_id.into(); + TagInfo { + tag_id: object_id.clone(), + object_id, + name, + object_type: "commit".to_string(), + tagger: tagger_info.into(), + message: String::new(), + created_at: created_at.into(), + } +} + +/// Merge annotated tag page with lightweight refs for list_tags pagination. +pub fn merge_paginated_tags( + mut annotated: Vec, + lightweight_refs: Vec, + annotated_total: u64, + per_page: u64, +) -> (Vec, u64) { + let per_page = if per_page == 0 { 20 } else { per_page as usize }; + let total = annotated_total + lightweight_refs.len() as u64; + if annotated.len() < per_page { + let need = per_page - annotated.len(); + for item in lightweight_refs.into_iter().take(need) { + annotated.push(item); + } + } + (annotated, total) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn format_tagger_info_formats_email() { + assert_eq!( + format_tagger_info(Some("Alice".into()), Some("a@b.c".into())), + "Alice " + ); + } + + #[test] + fn is_annotated_tag_requires_non_empty_message() { + assert!(!is_annotated_tag(&None)); + assert!(!is_annotated_tag(&Some(String::new()))); + assert!(is_annotated_tag(&Some("release".into()))); + } + + #[test] + fn merge_paginated_tags_fills_from_lightweight() { + let annotated = vec![lightweight_commit_tag("a", "1", "", "t")]; + let lightweight = vec![ + lightweight_commit_tag("b", "2", "", "t"), + lightweight_commit_tag("c", "3", "", "t"), + ]; + let (page, total) = merge_paginated_tags(annotated, lightweight, 1, 2); + assert_eq!(page.len(), 2); + assert_eq!(total, 3); + } +} diff --git a/ceres/src/api_service/tree_ops.rs b/ceres/src/application/api_service/tree_ops.rs similarity index 100% rename from ceres/src/api_service/tree_ops.rs rename to ceres/src/application/api_service/tree_ops.rs diff --git a/ceres/src/application/artifact/mod.rs b/ceres/src/application/artifact/mod.rs new file mode 100644 index 000000000..f139f5b66 --- /dev/null +++ b/ceres/src/application/artifact/mod.rs @@ -0,0 +1,37 @@ +//! Artifact protocol application facade. + +use std::{ops::Deref, time::Duration}; + +use common::errors::MegaError; +use jupiter::{ + service::artifact_service::{ArtifactObjectGcStats, ArtifactService}, + storage::Storage, +}; + +/// Ceres-facing artifact orchestration service. +#[derive(Clone)] +pub struct ArtifactApplicationService(ArtifactService); + +impl ArtifactApplicationService { + pub fn from_storage(storage: &Storage) -> Self { + Self(storage.artifact_service.clone()) + } + + pub async fn gc_unreferenced_once( + &self, + grace: Duration, + batch_limit: u64, + ) -> Result { + self.0 + .gc_unreferenced_artifact_objects_once(grace, batch_limit) + .await + } +} + +impl Deref for ArtifactApplicationService { + type Target = ArtifactService; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/ceres/src/application/buck/mod.rs b/ceres/src/application/buck/mod.rs new file mode 100644 index 000000000..f5f58cd88 --- /dev/null +++ b/ceres/src/application/buck/mod.rs @@ -0,0 +1,3 @@ +//! Buck upload application facade (re-export from mono API module during migration). + +pub use crate::application::api_service::mono::buck::*; diff --git a/ceres/src/build_trigger/buck_upload_handler.rs b/ceres/src/application/build_trigger/buck_upload_handler.rs similarity index 86% rename from ceres/src/build_trigger/buck_upload_handler.rs rename to ceres/src/application/build_trigger/buck_upload_handler.rs index 4520b0609..ce6942faf 100644 --- a/ceres/src/build_trigger/buck_upload_handler.rs +++ b/ceres/src/application/build_trigger/buck_upload_handler.rs @@ -6,9 +6,9 @@ use chrono::Utc; use common::errors::MegaError; use jupiter::storage::Storage; -use super::changes_calculator::ChangesCalculator; +use super::changes_calculator::MonoChangesCalculator; use crate::{ - api_service::cache::GitObjectCache, + api_service::{cache::GitObjectCache, mono::MonoApiService}, build_trigger::{ BuckFileUploadPayload, BuildTrigger, BuildTriggerPayload, BuildTriggerType, TriggerContext, TriggerHandler, @@ -17,13 +17,16 @@ use crate::{ /// Handler for Buck file upload triggers. pub struct BuckFileUploadHandler { - changes_calculator: ChangesCalculator, + changes_calculator: MonoChangesCalculator, } impl BuckFileUploadHandler { pub fn new(storage: Storage, git_object_cache: Arc) -> Self { Self { - changes_calculator: ChangesCalculator::new(storage, git_object_cache), + changes_calculator: MonoChangesCalculator::new(MonoApiService { + storage, + git_object_cache, + }), } } } diff --git a/ceres/src/build_trigger/changes_calculator.rs b/ceres/src/application/build_trigger/changes_calculator.rs similarity index 93% rename from ceres/src/build_trigger/changes_calculator.rs rename to ceres/src/application/build_trigger/changes_calculator.rs index 4ca8a2446..3a216f90a 100644 --- a/ceres/src/build_trigger/changes_calculator.rs +++ b/ceres/src/application/build_trigger/changes_calculator.rs @@ -1,16 +1,12 @@ -use std::{ - path::{Component, Path, PathBuf}, - sync::Arc, -}; +use std::path::{Component, Path, PathBuf}; pub use api_model::buck2::{status::Status, types::ProjectRelativePath}; use common::errors::MegaError; use git_internal::hash::ObjectHash; -use jupiter::storage::Storage; +use super::changes_port::ChangesPort; use crate::{ - api_service::{cache::GitObjectCache, mono_api_service::MonoApiService}, - build_trigger::TriggerContext, + api_service::mono::MonoApiService, build_trigger::TriggerContext, model::change_list::ClDiffFile, }; @@ -187,17 +183,16 @@ fn build_changes_for_repo( Ok(counter_changes) } -pub struct ChangesCalculator { - storage: Storage, - git_object_cache: Arc, +pub struct ChangesCalculator { + port: P, } -impl ChangesCalculator { - pub fn new(storage: Storage, git_object_cache: Arc) -> Self { - Self { - storage, - git_object_cache, - } +/// Changes calculator backed by [`MonoApiService`]. +pub type MonoChangesCalculator = ChangesCalculator; + +impl ChangesCalculator

{ + pub fn new(port: P) -> Self { + Self { port } } pub async fn get_builds_for_commit( @@ -217,11 +212,7 @@ impl ChangesCalculator { &self, commit_hash: &str, ) -> Result, MegaError> { - let api_service = MonoApiService { - storage: self.storage.clone(), - git_object_cache: self.git_object_cache.clone(), - }; - api_service.get_commit_blobs(commit_hash).await + self.port.get_commit_blobs(commit_hash).await } async fn cl_files_list( @@ -229,11 +220,7 @@ impl ChangesCalculator { old_files: Vec<(PathBuf, ObjectHash)>, new_files: Vec<(PathBuf, ObjectHash)>, ) -> Result, MegaError> { - let api_service = MonoApiService { - storage: self.storage.clone(), - git_object_cache: self.git_object_cache.clone(), - }; - api_service.cl_files_list(old_files, new_files).await + self.port.cl_files_list(old_files, new_files).await } } diff --git a/ceres/src/application/build_trigger/changes_port.rs b/ceres/src/application/build_trigger/changes_port.rs new file mode 100644 index 000000000..53100e8e3 --- /dev/null +++ b/ceres/src/application/build_trigger/changes_port.rs @@ -0,0 +1,40 @@ +use std::path::PathBuf; + +use async_trait::async_trait; +use common::errors::MegaError; +use git_internal::hash::ObjectHash; + +use crate::{api_service::mono::MonoApiService, model::change_list::ClDiffFile}; + +/// Port for computing CL file diffs used by build trigger handlers. +#[async_trait] +pub trait ChangesPort: Send + Sync { + async fn get_commit_blobs( + &self, + commit_hash: &str, + ) -> Result, MegaError>; + + async fn cl_files_list( + &self, + old_files: Vec<(PathBuf, ObjectHash)>, + new_files: Vec<(PathBuf, ObjectHash)>, + ) -> Result, MegaError>; +} + +#[async_trait] +impl ChangesPort for MonoApiService { + async fn get_commit_blobs( + &self, + commit_hash: &str, + ) -> Result, MegaError> { + MonoApiService::get_commit_blobs(self, commit_hash).await + } + + async fn cl_files_list( + &self, + old_files: Vec<(PathBuf, ObjectHash)>, + new_files: Vec<(PathBuf, ObjectHash)>, + ) -> Result, MegaError> { + MonoApiService::cl_files_list(self, old_files, new_files).await + } +} diff --git a/ceres/src/build_trigger/dispatcher.rs b/ceres/src/application/build_trigger/dispatcher.rs similarity index 100% rename from ceres/src/build_trigger/dispatcher.rs rename to ceres/src/application/build_trigger/dispatcher.rs diff --git a/ceres/src/build_trigger/git_push_handler.rs b/ceres/src/application/build_trigger/git_push_handler.rs similarity index 85% rename from ceres/src/build_trigger/git_push_handler.rs rename to ceres/src/application/build_trigger/git_push_handler.rs index d08a426d3..194bb7d95 100644 --- a/ceres/src/build_trigger/git_push_handler.rs +++ b/ceres/src/application/build_trigger/git_push_handler.rs @@ -5,9 +5,9 @@ use chrono::Utc; use common::errors::MegaError; use jupiter::storage::Storage; -use super::changes_calculator::ChangesCalculator; +use super::changes_calculator::MonoChangesCalculator; use crate::{ - api_service::cache::GitObjectCache, + api_service::{cache::GitObjectCache, mono::MonoApiService}, build_trigger::{ BuildTrigger, BuildTriggerPayload, BuildTriggerType, GitPushPayload, TriggerContext, TriggerHandler, @@ -16,13 +16,16 @@ use crate::{ /// Handler for Git push triggers. pub struct GitPushHandler { - changes_calculator: ChangesCalculator, + changes_calculator: MonoChangesCalculator, } impl GitPushHandler { pub fn new(storage: Storage, git_object_cache: Arc) -> Self { Self { - changes_calculator: ChangesCalculator::new(storage, git_object_cache), + changes_calculator: MonoChangesCalculator::new(MonoApiService { + storage, + git_object_cache, + }), } } } diff --git a/ceres/src/build_trigger/manual_handler.rs b/ceres/src/application/build_trigger/manual_handler.rs similarity index 90% rename from ceres/src/build_trigger/manual_handler.rs rename to ceres/src/application/build_trigger/manual_handler.rs index e8be35a23..2e72f74ba 100644 --- a/ceres/src/build_trigger/manual_handler.rs +++ b/ceres/src/application/build_trigger/manual_handler.rs @@ -5,9 +5,9 @@ use chrono::Utc; use common::errors::MegaError; use jupiter::storage::Storage; -use super::changes_calculator::ChangesCalculator; +use super::changes_calculator::MonoChangesCalculator; use crate::{ - api_service::cache::GitObjectCache, + api_service::{cache::GitObjectCache, mono::MonoApiService}, build_trigger::{ BuildTrigger, BuildTriggerPayload, BuildTriggerType, ManualPayload, TriggerContext, TriggerHandler, @@ -17,14 +17,17 @@ use crate::{ /// Handler for manual build triggers. pub struct ManualHandler { storage: Storage, - changes_calculator: ChangesCalculator, + changes_calculator: MonoChangesCalculator, } impl ManualHandler { pub fn new(storage: Storage, git_object_cache: Arc) -> Self { Self { storage: storage.clone(), - changes_calculator: ChangesCalculator::new(storage, git_object_cache), + changes_calculator: MonoChangesCalculator::new(MonoApiService { + storage: storage.clone(), + git_object_cache, + }), } } diff --git a/ceres/src/build_trigger/mod.rs b/ceres/src/application/build_trigger/mod.rs similarity index 99% rename from ceres/src/build_trigger/mod.rs rename to ceres/src/application/build_trigger/mod.rs index 868845506..7ceedfb9b 100644 --- a/ceres/src/build_trigger/mod.rs +++ b/ceres/src/application/build_trigger/mod.rs @@ -9,6 +9,7 @@ use crate::api_service::cache::GitObjectCache; mod buck_upload_handler; mod changes_calculator; +mod changes_port; mod dispatcher; mod git_push_handler; mod manual_handler; diff --git a/ceres/src/build_trigger/model.rs b/ceres/src/application/build_trigger/model.rs similarity index 100% rename from ceres/src/build_trigger/model.rs rename to ceres/src/application/build_trigger/model.rs diff --git a/ceres/src/build_trigger/ref_resolver.rs b/ceres/src/application/build_trigger/ref_resolver.rs similarity index 100% rename from ceres/src/build_trigger/ref_resolver.rs rename to ceres/src/application/build_trigger/ref_resolver.rs diff --git a/ceres/src/build_trigger/retry_handler.rs b/ceres/src/application/build_trigger/retry_handler.rs similarity index 88% rename from ceres/src/build_trigger/retry_handler.rs rename to ceres/src/application/build_trigger/retry_handler.rs index 28ecfbcbd..eef421dfc 100644 --- a/ceres/src/build_trigger/retry_handler.rs +++ b/ceres/src/application/build_trigger/retry_handler.rs @@ -5,9 +5,9 @@ use chrono::Utc; use common::errors::MegaError; use jupiter::storage::Storage; -use super::changes_calculator::ChangesCalculator; +use super::changes_calculator::MonoChangesCalculator; use crate::{ - api_service::cache::GitObjectCache, + api_service::{cache::GitObjectCache, mono::MonoApiService}, build_trigger::{ BuildTrigger, BuildTriggerPayload, BuildTriggerType, RetryPayload, TriggerContext, TriggerHandler, @@ -16,13 +16,16 @@ use crate::{ /// Handler for retry build triggers. pub struct RetryHandler { - changes_calculator: ChangesCalculator, + changes_calculator: MonoChangesCalculator, } impl RetryHandler { pub fn new(storage: Storage, git_object_cache: Arc) -> Self { Self { - changes_calculator: ChangesCalculator::new(storage, git_object_cache), + changes_calculator: MonoChangesCalculator::new(MonoApiService { + storage, + git_object_cache, + }), } } } diff --git a/ceres/src/build_trigger/service.rs b/ceres/src/application/build_trigger/service.rs similarity index 97% rename from ceres/src/build_trigger/service.rs rename to ceres/src/application/build_trigger/service.rs index ec50ea244..3b22f2090 100644 --- a/ceres/src/build_trigger/service.rs +++ b/ceres/src/application/build_trigger/service.rs @@ -32,15 +32,12 @@ use common::errors::MegaError; use jupiter::storage::Storage; use orion_client::OrionBuildClient; +use super::model::{ + BuildParams, GitPushEvent, ListTriggersParams, TriggerContext, TriggerRecord, TriggerResponse, +}; use crate::{ api_service::cache::GitObjectCache, - build_trigger::{ - RefResolver, TriggerRegistry, - model::{ - BuildParams, GitPushEvent, ListTriggersParams, TriggerContext, TriggerRecord, - TriggerResponse, - }, - }, + build_trigger::{RefResolver, TriggerRegistry}, code_edit::utils as edit_utils, }; @@ -289,7 +286,7 @@ mod tests { use tempfile::tempdir; use super::*; - use crate::build_trigger::model::BuildTriggerType; + use crate::build_trigger::BuildTriggerType; #[tokio::test] async fn test_context_from_cl_resolves_repo_root_from_registered_repo_path() { diff --git a/ceres/src/build_trigger/web_edit_handler.rs b/ceres/src/application/build_trigger/web_edit_handler.rs similarity index 93% rename from ceres/src/build_trigger/web_edit_handler.rs rename to ceres/src/application/build_trigger/web_edit_handler.rs index e4f8a0d91..ae8041bb5 100644 --- a/ceres/src/build_trigger/web_edit_handler.rs +++ b/ceres/src/application/build_trigger/web_edit_handler.rs @@ -5,9 +5,9 @@ use chrono::Utc; use common::errors::MegaError; use jupiter::storage::Storage; -use super::changes_calculator::ChangesCalculator; +use super::changes_calculator::MonoChangesCalculator; use crate::{ - api_service::cache::GitObjectCache, + api_service::{cache::GitObjectCache, mono::MonoApiService}, build_trigger::{ BuildTrigger, BuildTriggerPayload, BuildTriggerType, TriggerContext, TriggerHandler, WebEditPayload, @@ -37,13 +37,16 @@ fn serialize_builds( /// Handler for web edit trigger pub struct WebEditHandler { - changes_calculator: ChangesCalculator, + changes_calculator: MonoChangesCalculator, } impl WebEditHandler { pub fn new(storage: Storage, git_object_cache: Arc) -> Self { Self { - changes_calculator: ChangesCalculator::new(storage, git_object_cache), + changes_calculator: MonoChangesCalculator::new(MonoApiService { + storage, + git_object_cache, + }), } } } diff --git a/ceres/src/code_edit/mod.rs b/ceres/src/application/code_edit/mod.rs similarity index 74% rename from ceres/src/code_edit/mod.rs rename to ceres/src/application/code_edit/mod.rs index 3718abfe0..d2de2ca70 100644 --- a/ceres/src/code_edit/mod.rs +++ b/ceres/src/application/code_edit/mod.rs @@ -1,4 +1,5 @@ pub mod model; pub mod on_edit; pub mod on_push; +pub mod post_receive; pub mod utils; diff --git a/ceres/src/code_edit/model.rs b/ceres/src/application/code_edit/model.rs similarity index 98% rename from ceres/src/code_edit/model.rs rename to ceres/src/application/code_edit/model.rs index 2aa9f5892..4647a6279 100644 --- a/ceres/src/code_edit/model.rs +++ b/ceres/src/application/code_edit/model.rs @@ -4,7 +4,7 @@ use callisto::{entity_ext::generate_link, mega_cl, mega_refs, sea_orm_active_enu use common::errors::MegaError; use git_internal::internal::object::commit::Commit; use jupiter::{ - service::{reviewer_service::ReviewerService, webhook_service::WebhookEvent}, + service::reviewer_service::ReviewerService, storage::{Storage, mono_storage::MonoStorage}, utils::converter::FromMegaModel, }; @@ -12,6 +12,7 @@ use orion_client::OrionBuildClient; use crate::{ api_service::{ApiHandler, cache::GitObjectCache}, + application::webhook::{WebhookEvent, dispatch_cl_webhook}, build_trigger::{BuildTriggerService, TriggerContext}, code_edit::{model, utils as edit_utils}, merge_checker::CheckerRegistry, @@ -323,9 +324,7 @@ impl< ConvTypeEnum::Comment, ) .await?; - storage - .webhook_service - .dispatch(WebhookEvent::ClCreated, &cl); + dispatch_cl_webhook(storage, WebhookEvent::ClCreated, &cl); Ok(cl) } diff --git a/ceres/src/code_edit/on_edit.rs b/ceres/src/application/code_edit/on_edit.rs similarity index 98% rename from ceres/src/code_edit/on_edit.rs rename to ceres/src/application/code_edit/on_edit.rs index d4722b53c..b0f8c1c43 100644 --- a/ceres/src/code_edit/on_edit.rs +++ b/ceres/src/application/code_edit/on_edit.rs @@ -7,7 +7,7 @@ use git_internal::errors::GitError; use jupiter::storage::{Storage, mono_storage::MonoStorage}; use crate::{ - api_service::{cache::GitObjectCache, mono_api_service::MonoApiService}, + api_service::{cache::GitObjectCache, mono::MonoApiService}, build_trigger::{BuildTriggerService, TriggerContext}, code_edit::{model, utils as edit_utils}, model::git::EditCLMode, diff --git a/ceres/src/code_edit/on_push.rs b/ceres/src/application/code_edit/on_push.rs similarity index 94% rename from ceres/src/code_edit/on_push.rs rename to ceres/src/application/code_edit/on_push.rs index 56da28edc..ff823c656 100644 --- a/ceres/src/code_edit/on_push.rs +++ b/ceres/src/application/code_edit/on_push.rs @@ -6,7 +6,7 @@ use jupiter::storage::Storage; use orion_client::OrionBuildClient; use crate::{ - api_service::{cache::GitObjectCache, mono_api_service::MonoApiService}, + api_service::{cache::GitObjectCache, mono::MonoApiService}, build_trigger::{BuildTriggerService, TriggerContext}, code_edit::{ model::{self, CLRefUpdateVisitor}, @@ -25,7 +25,10 @@ impl model::CLRefUpdateVisitor for OnpushVisitor { _: &str, _: &str, ) -> Result { - panic!("visit not implemented"); + Err(MegaError::Other( + "OnpushVisitor::visit is unused; OnpushAcceptor does not delegate to the visitor" + .to_string(), + )) } } diff --git a/ceres/src/application/code_edit/post_receive/import.rs b/ceres/src/application/code_edit/post_receive/import.rs new file mode 100644 index 000000000..13f72c5c9 --- /dev/null +++ b/ceres/src/application/code_edit/post_receive/import.rs @@ -0,0 +1,189 @@ +//! Import repo attach-to-monorepo handler. + +use std::{ + path::PathBuf, + str::FromStr, + sync::{Arc, Mutex}, + time::Instant, +}; + +use callisto::sea_orm_active_enums::RefTypeEnum; +use common::errors::MegaError; +use git_internal::{hash::ObjectHash, internal::object::commit::Commit}; +use jupiter::{redis::lock::RedLock, storage::Storage, utils::converter::FromGitModel}; + +use crate::{ + api_service::{cache::GitObjectCache, mono::MonoApiService, tree_ops}, + protocol::import_refs::{CommandType, RefCommand}, +}; + +pub async fn dispatch_import_receive_pack_finalized( + storage: Storage, + git_object_cache: Arc, + repo_path: PathBuf, + repo_id: i64, + commands: Vec, + unpack_redlock: Arc, + extra_timings: Arc>>, +) -> Result<(), MegaError> { + let commit_id = match commands.iter().find(|c| c.ref_type == RefTypeEnum::Branch) { + Some(cmd) => cmd.new_id.clone(), + None => return Ok(()), + }; + + let mono_api_service = MonoApiService { + storage: storage.clone(), + git_object_cache, + }; + let mono_storage = storage.mono_storage(); + + let latest_commit: Commit = Commit::from_git_model( + storage + .git_db_storage() + .get_commit_by_hash(repo_id, &commit_id) + .await? + .ok_or_else(|| MegaError::Other(format!("commit {commit_id} not found")))?, + ); + let commit_msg = latest_commit.format_message(); + + const MAX_ATTACH_ATTEMPTS: u32 = 64; + let mut root_lock_wait_max_ms: u128 = 0; + let mut root_lock_wait_sum_ms: u128 = 0; + + for attempt in 0..MAX_ATTACH_ATTEMPTS { + let t_lock = Instant::now(); + let guard = unpack_redlock.clone().lock().await?; + let lock_wait_ms = t_lock.elapsed().as_millis(); + root_lock_wait_max_ms = root_lock_wait_max_ms.max(lock_wait_ms); + root_lock_wait_sum_ms += lock_wait_ms; + + let root_ref = mono_storage + .get_main_ref("/") + .await? + .ok_or_else(|| MegaError::Other("root ref not found".to_string()))?; + let expected_commit = root_ref.ref_commit_hash.clone(); + let expected_tree = root_ref.ref_tree_hash.clone(); + let root_ref_id = root_ref.id; + + let save_trees = tree_ops::search_and_create_tree(&mono_api_service, &repo_path).await?; + + let new_commit = Commit::from_tree_id( + save_trees + .back() + .ok_or_else(|| MegaError::Other("no tree generated".to_string()))? + .id, + vec![ObjectHash::from_str(&expected_commit).unwrap()], + &format!("\n{commit_msg}"), + ); + + let txn = storage.begin_db_transaction().await?; + let git_db = storage.git_db_storage(); + for cmd in &commands { + if cmd.ref_type != RefTypeEnum::Branch { + continue; + } + match cmd.command_type { + CommandType::Create => { + git_db + .save_ref_in_txn(repo_id, cmd.clone().into(), &txn) + .await?; + } + CommandType::Delete => { + git_db + .remove_ref_in_txn(repo_id, &cmd.ref_name, &txn) + .await?; + } + CommandType::Update => { + git_db + .update_ref_in_txn(repo_id, &cmd.ref_name, &cmd.new_id, &txn) + .await?; + } + } + } + + let t_attach_txn = Instant::now(); + match mono_storage + .attach_to_monorepo_parent_in_txn( + &txn, + root_ref_id, + &expected_commit, + &expected_tree, + new_commit, + save_trees.into(), + ) + .await + { + Ok(()) => { + txn.commit().await.map_err(MegaError::Db)?; + let t_unlock = Instant::now(); + guard.unlock().await?; + extra_timings + .lock() + .expect("import extra_timings lock poisoned") + .extend([ + ( + "import_attach_attempts_count".to_string(), + (attempt + 1) as u128, + ), + ( + "import_root_lock_wait_sum_ms".to_string(), + root_lock_wait_sum_ms, + ), + ( + "import_root_lock_wait_max_ms".to_string(), + root_lock_wait_max_ms, + ), + ( + "import_attach_txn_ms".to_string(), + t_attach_txn.elapsed().as_millis(), + ), + ( + "import_root_lock_unlock_ms".to_string(), + t_unlock.elapsed().as_millis(), + ), + ]); + return Ok(()); + } + Err(MegaError::StaleMonorepoRootRef) if attempt + 1 < MAX_ATTACH_ATTEMPTS => { + let _ = txn.rollback().await; + let _ = guard.unlock().await; + tracing::warn!( + attempt = attempt, + repo_path = %repo_path.display(), + "attach_to_monorepo_parent: root ref moved, retrying" + ); + tokio::task::yield_now().await; + } + Err(e) => { + let _ = txn.rollback().await; + let _ = guard.unlock().await; + extra_timings + .lock() + .expect("import extra_timings lock poisoned") + .extend([ + ( + "import_attach_attempts_count".to_string(), + (attempt + 1) as u128, + ), + ( + "import_root_lock_wait_sum_ms".to_string(), + root_lock_wait_sum_ms, + ), + ( + "import_root_lock_wait_max_ms".to_string(), + root_lock_wait_max_ms, + ), + ( + "import_attach_txn_ms".to_string(), + t_attach_txn.elapsed().as_millis(), + ), + ]); + return Err(e); + } + } + } + + Err(MegaError::Other( + "attach_to_monorepo_parent: exceeded retry limit for concurrent root updates".into(), + )) +} diff --git a/ceres/src/application/code_edit/post_receive/mod.rs b/ceres/src/application/code_edit/post_receive/mod.rs new file mode 100644 index 000000000..3b064e481 --- /dev/null +++ b/ceres/src/application/code_edit/post_receive/mod.rs @@ -0,0 +1,4 @@ +mod import; +mod mono; + +pub use mono::RuntimeApplicationHandler; diff --git a/ceres/src/application/code_edit/post_receive/mono.rs b/ceres/src/application/code_edit/post_receive/mono.rs new file mode 100644 index 000000000..971bcc791 --- /dev/null +++ b/ceres/src/application/code_edit/post_receive/mono.rs @@ -0,0 +1,256 @@ +use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + sync::Arc, +}; + +use api_model::common::Pagination; +use async_trait::async_trait; +use callisto::{mega_cl, mega_code_review_anchor}; +use common::{errors::MegaError, utils::ZERO_ID}; +use futures::{StreamExt, stream}; +use jupiter::storage::Storage; +use orion_client::OrionBuildClient; + +use crate::{ + api_service::{ + ApiHandler, + cache::GitObjectCache, + mono::{MonoApiService, cl_merge}, + }, + bus::{ApplicationEventHandler, TransportEvent}, + code_edit::{on_push::OnpushCodeEdit, utils::get_changed_files}, +}; + +/// Handles CL creation, bootstrap, build triggers, and code-review reanchoring after mono push. +pub async fn dispatch_mono_receive_pack_finalized( + storage: Storage, + git_object_cache: Arc, + repo_path: PathBuf, + base_branch: String, + from_hash: String, + to_hash: String, + username: Option, +) -> Result<(), MegaError> { + let username = username.unwrap_or_else(|| String::from("Anonymous")); + let mono_api_service = MonoApiService { + storage: storage.clone(), + git_object_cache: git_object_cache.clone(), + }; + let repo_path_str = repo_path + .to_str() + .ok_or_else(|| MegaError::Other("invalid repo path".to_string()))?; + + let editor = OnpushCodeEdit::from(repo_path_str, &base_branch, &from_hash, &mono_api_service); + let cl = editor + .update_or_create_cl(&storage, &from_hash, &to_hash, &username) + .await?; + + if from_hash == ZERO_ID && repo_path_str.starts_with("/project/") { + cl_merge::bootstrap_monorepo_path(&mono_api_service, repo_path_str, Some(&cl)).await?; + } + + let orion_client = Arc::new(OrionBuildClient::new(storage.config().build.clone())); + if orion_client.enable_build() { + editor + .trigger_build_and_check( + storage.clone(), + git_object_cache.clone(), + orion_client, + &cl, + &username, + ) + .await?; + } + + reanchor_code_review_threads(&storage, &mono_api_service, &cl, &to_hash).await +} + +async fn reanchor_code_review_threads( + storage: &Storage, + mono_api_service: &MonoApiService, + cl: &mega_cl::Model, + to_hash: &str, +) -> Result<(), MegaError> { + let cl_link = cl.link.clone(); + let changed_files = get_changed_files(mono_api_service, cl).await?; + let files_with_threads = storage + .code_review_thread_storage() + .get_files_with_threads_by_link(&cl_link) + .await?; + + let files_with_threads_set: HashSet<&String> = files_with_threads.iter().collect(); + + let affected_files: Vec = changed_files + .into_iter() + .filter(|file| files_with_threads_set.contains(file)) + .collect(); + + tracing::info!( + "Reanchor code review thread in cl_link: {}, affected files: {:?}", + cl_link, + affected_files + ); + + let pending_reanchor_threads = storage + .code_review_thread_storage() + .find_threads_by_file_paths(affected_files) + .await?; + + let pending_reanchor_thread_ids: Vec = pending_reanchor_threads + .iter() + .map(|thread| thread.id) + .collect(); + + storage + .code_review_thread_storage() + .mark_positions_status_by_thread_ids( + &pending_reanchor_thread_ids, + callisto::sea_orm_active_enums::PositionStatusEnum::PendingReanchor, + ) + .await?; + + let anchors = storage + .code_review_thread_storage() + .get_anchors_by_thread_ids(&pending_reanchor_thread_ids) + .await?; + + let mono_api_service = Arc::new(mono_api_service.clone()); + let mut anchors_map: HashMap> = HashMap::new(); + for anchor in anchors { + anchors_map + .entry(anchor.thread_id) + .or_default() + .push(anchor); + } + + let reanchor_tasks: Vec<_> = pending_reanchor_threads + .into_iter() + .map(|thread| { + let cl_link = cl_link.clone(); + let mono_api_service = Arc::clone(&mono_api_service); + let anchors_map = anchors_map.clone(); + let to_hash = to_hash.to_string(); + let storage = storage.clone(); + + async move { + let thread_id = thread.id; + + let thread_anchors = match anchors_map.get(&thread_id) { + Some(anchors) => anchors, + None => { + tracing::warn!("Thread {} has no anchors", thread_id); + return Err(MegaError::Other(format!( + "Thread {} has no anchors", + thread_id + ))); + } + }; + + let (diff_content, _) = mono_api_service + .paged_content_diff(&cl_link, Pagination::default()) + .await?; + + let mut blob_cache: HashMap = HashMap::new(); + + for anchor in thread_anchors { + let file_path = anchor.file_path.clone(); + + let latest_blob = if let Some(blob) = blob_cache.get(&file_path) { + blob.clone() + } else { + let blob = mono_api_service + .get_blob_as_string(PathBuf::from(&file_path), Some(&to_hash)) + .await? + .expect("latest blob must exist"); + + blob_cache.insert(file_path.clone(), blob.clone()); + blob + }; + + if let Err(e) = storage + .code_review_service + .reanchor_thread(anchor, Some(latest_blob), diff_content.clone(), &to_hash) + .await + { + tracing::error!("Reanchor failed for anchor {}: {:?}", anchor.id, e); + } + } + + Ok(()) + } + }) + .collect::>(); + + let results: Vec> = stream::iter(reanchor_tasks) + .buffer_unordered(storage.get_recommended_batch_concurrency()) + .collect() + .await; + + for res in results { + if let Err(e) = res { + tracing::error!("Reanchor task failed: {:?}", e); + } + } + + Ok(()) +} + +/// Application handler that dispatches transport events using storage + cache context. +pub struct RuntimeApplicationHandler { + storage: Storage, + git_object_cache: Arc, +} + +impl RuntimeApplicationHandler { + pub fn new(storage: Storage, git_object_cache: Arc) -> Self { + Self { + storage, + git_object_cache, + } + } +} + +#[async_trait] +impl ApplicationEventHandler for RuntimeApplicationHandler { + async fn handle(&self, event: TransportEvent) -> Result<(), MegaError> { + match event { + TransportEvent::MonoReceivePackFinalized { + repo_path, + base_branch, + from_hash, + to_hash, + username, + } => { + dispatch_mono_receive_pack_finalized( + self.storage.clone(), + self.git_object_cache.clone(), + repo_path, + base_branch, + from_hash, + to_hash, + username, + ) + .await + } + TransportEvent::ImportReceivePackFinalized { + repo_path, + repo_id, + commands, + unpack_redlock, + extra_timings, + } => { + super::import::dispatch_import_receive_pack_finalized( + self.storage.clone(), + self.git_object_cache.clone(), + repo_path, + repo_id, + commands, + unpack_redlock, + extra_timings, + ) + .await + } + } + } +} diff --git a/ceres/src/code_edit/utils.rs b/ceres/src/application/code_edit/utils.rs similarity index 98% rename from ceres/src/code_edit/utils.rs rename to ceres/src/application/code_edit/utils.rs index 5780674d5..c838acc41 100644 --- a/ceres/src/code_edit/utils.rs +++ b/ceres/src/application/code_edit/utils.rs @@ -16,7 +16,7 @@ use git_internal::{ use jupiter::{storage::Storage, utils::converter::FromMegaModel}; use crate::{ - api_service::{ApiHandler, commit_ops, mono_api_service::MonoServiceLogic}, + api_service::{ApiHandler, commit_ops, mono::MonoServiceLogic}, model::change_list::ClDiffFile, }; @@ -146,9 +146,13 @@ pub async fn get_changed_files( handler: &T, cl: &mega_cl::Model, ) -> Result, MegaError> { - let from_commit = handler.get_commit_by_hash(&cl.from_hash).await?; let to_commit = handler.get_commit_by_hash(&cl.to_hash).await?; - let old_files = commit_ops::collect_commit_blobs(handler, &from_commit).await?; + let old_files = if cl.from_hash == ZERO_ID { + Vec::new() + } else { + let from_commit = handler.get_commit_by_hash(&cl.from_hash).await?; + commit_ops::collect_commit_blobs(handler, &from_commit).await? + }; let new_files = commit_ops::collect_commit_blobs(handler, &to_commit).await?; let changed = cl_files_list(old_files, new_files).await?; diff --git a/ceres/src/application/mod.rs b/ceres/src/application/mod.rs new file mode 100644 index 000000000..fccadc884 --- /dev/null +++ b/ceres/src/application/mod.rs @@ -0,0 +1,6 @@ +pub mod api_service; +pub mod artifact; +pub mod buck; +pub mod build_trigger; +pub mod code_edit; +pub mod webhook; diff --git a/ceres/src/application/webhook/mod.rs b/ceres/src/application/webhook/mod.rs new file mode 100644 index 000000000..86c00d2cb --- /dev/null +++ b/ceres/src/application/webhook/mod.rs @@ -0,0 +1,30 @@ +//! Webhook delivery facade (orchestration entry point for CL lifecycle events). + +use callisto::mega_cl; +use common::errors::MegaError; +pub use jupiter::service::webhook_service::{ + ClPayload, RepositoryPayload, WebhookEvent, WebhookPayload, +}; +use jupiter::{service::webhook_service::WebhookService, storage::Storage}; + +/// Dispatches CL webhook events asynchronously. +#[derive(Clone)] +pub struct WebhookDispatcher { + service: WebhookService, +} + +impl WebhookDispatcher { + pub fn from_storage(storage: &Storage) -> Result { + Ok(Self { + service: storage.webhook_service.clone(), + }) + } + + pub fn dispatch(&self, event_type: WebhookEvent, cl_model: &mega_cl::Model) { + self.service.dispatch(event_type, cl_model); + } +} + +pub fn dispatch_cl_webhook(storage: &Storage, event_type: WebhookEvent, cl_model: &mega_cl::Model) { + storage.webhook_service.dispatch(event_type, cl_model); +} diff --git a/ceres/src/bus/event.rs b/ceres/src/bus/event.rs new file mode 100644 index 000000000..1324f8c90 --- /dev/null +++ b/ceres/src/bus/event.rs @@ -0,0 +1,26 @@ +use std::{ + path::PathBuf, + sync::{Arc, Mutex}, +}; + +use jupiter::redis::lock::RedLock; + +use crate::transport::protocol::import_refs::RefCommand; + +#[derive(Clone)] +pub enum TransportEvent { + MonoReceivePackFinalized { + repo_path: PathBuf, + base_branch: String, + from_hash: String, + to_hash: String, + username: Option, + }, + ImportReceivePackFinalized { + repo_path: PathBuf, + repo_id: i64, + commands: Vec, + unpack_redlock: Arc, + extra_timings: Arc>>, + }, +} diff --git a/ceres/src/bus/handler.rs b/ceres/src/bus/handler.rs new file mode 100644 index 000000000..d45a9e787 --- /dev/null +++ b/ceres/src/bus/handler.rs @@ -0,0 +1,9 @@ +use async_trait::async_trait; +use common::errors::MegaError; + +use super::event::TransportEvent; + +#[async_trait] +pub trait ApplicationEventHandler: Send + Sync { + async fn handle(&self, event: TransportEvent) -> Result<(), MegaError>; +} diff --git a/ceres/src/bus/mod.rs b/ceres/src/bus/mod.rs new file mode 100644 index 000000000..c22a1246c --- /dev/null +++ b/ceres/src/bus/mod.rs @@ -0,0 +1,7 @@ +pub mod event; +pub mod handler; +pub mod runtime; + +pub use event::TransportEvent; +pub use handler::ApplicationEventHandler; +pub use runtime::TransportRuntime; diff --git a/ceres/src/bus/runtime.rs b/ceres/src/bus/runtime.rs new file mode 100644 index 000000000..2176ef964 --- /dev/null +++ b/ceres/src/bus/runtime.rs @@ -0,0 +1,26 @@ +use std::sync::Arc; + +use jupiter::storage::Storage; + +use super::handler::ApplicationEventHandler; +use crate::{code_edit::post_receive::RuntimeApplicationHandler, infra::cache::GitObjectCache}; + +#[derive(Clone)] +pub struct TransportRuntime { + pub storage: Storage, + pub git_object_cache: Arc, + pub application: Arc, +} + +impl TransportRuntime { + pub fn new(storage: Storage, git_object_cache: Arc) -> Self { + let application: Arc = Arc::new( + RuntimeApplicationHandler::new(storage.clone(), git_object_cache.clone()), + ); + Self { + storage, + git_object_cache, + application, + } + } +} diff --git a/ceres/src/api_service/cache.rs b/ceres/src/infra/cache.rs similarity index 100% rename from ceres/src/api_service/cache.rs rename to ceres/src/infra/cache.rs diff --git a/ceres/src/infra/mod.rs b/ceres/src/infra/mod.rs new file mode 100644 index 000000000..a5c08fdb0 --- /dev/null +++ b/ceres/src/infra/mod.rs @@ -0,0 +1 @@ +pub mod cache; diff --git a/ceres/src/lib.rs b/ceres/src/lib.rs index 6e5822140..0abdf1fd8 100644 --- a/ceres/src/lib.rs +++ b/ceres/src/lib.rs @@ -1,9 +1,34 @@ -pub mod api_service; -pub mod build_trigger; -pub mod code_edit; +//! Ceres: monorepo domain library (transport, application, shared models). + +pub mod application; +pub mod bus; pub mod diff; +pub mod infra; pub mod lfs; pub mod merge_checker; pub mod model; -pub mod pack; -pub mod protocol; +pub mod transport; + +// Legacy module paths (internal + `mono` crate compatibility). +pub mod api_service { + pub use crate::application::api_service::*; +} +pub mod build_trigger { + pub use crate::application::build_trigger::*; +} +pub mod code_edit { + pub use crate::application::code_edit::*; +} +pub mod pack { + pub use crate::transport::pack::*; +} +pub mod protocol { + pub use crate::transport::protocol::*; +} + +pub use application::api_service::{ + ADMIN_FILE, EffectiveResourcePermission, MonoApiService, MonoServiceLogic, RefUpdate, + TreeUpdateResult, cl_merge, +}; +pub use bus::{ApplicationEventHandler, TransportEvent, TransportRuntime}; +pub use transport::ProtocolApiState; diff --git a/ceres/src/merge_checker/cl_sync_checker.rs b/ceres/src/merge_checker/cl_sync_checker.rs index eaea72dd6..6a478cad7 100644 --- a/ceres/src/merge_checker/cl_sync_checker.rs +++ b/ceres/src/merge_checker/cl_sync_checker.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use async_trait::async_trait; -use common::errors::MegaError; +use common::{errors::MegaError, utils::ZERO_ID}; use jupiter::{model::cl_dto::ClInfoDto, storage::Storage}; use serde::Deserialize; @@ -42,15 +42,24 @@ impl Checker for ClSyncChecker { } async fn build_params(&self, cl_info: &ClInfoDto) -> Result { - let refs = self + let current = match self .storage .mono_storage() .get_main_ref(&cl_info.path) .await? - .expect("Err: CL Related Refs Not Found"); + { + Some(refs) => refs.ref_commit_hash, + None if cl_info.from_hash == ZERO_ID => ZERO_ID.to_string(), + None => { + return Err(MegaError::Other(format!( + "Main ref not found for CL path {}", + cl_info.path + ))); + } + }; Ok(serde_json::json!({ "cl_from": cl_info.from_hash, - "current": refs.ref_commit_hash + "current": current })) } } diff --git a/ceres/src/model/cl.rs b/ceres/src/model/cl.rs deleted file mode 100644 index 8b1378917..000000000 --- a/ceres/src/model/cl.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ceres/src/model/mod.rs b/ceres/src/model/mod.rs index e7152643c..70960b94c 100644 --- a/ceres/src/model/mod.rs +++ b/ceres/src/model/mod.rs @@ -2,7 +2,6 @@ pub mod blame; pub mod bots; pub mod buck; pub mod change_list; -pub mod cl; pub mod code_review; pub mod commit; pub mod conversation; diff --git a/ceres/src/model/third_party.rs b/ceres/src/model/third_party.rs index 88e8fa5dd..d0f44a621 100644 --- a/ceres/src/model/third_party.rs +++ b/ceres/src/model/third_party.rs @@ -33,6 +33,7 @@ pub trait ThirdPartyRepoTrait { async fn fetch_packs( &self, want: &[String], + depth: Option, ) -> Result> + Send>>, MegaError>; } @@ -156,9 +157,10 @@ impl ThirdPartyRepoTrait for ThirdPartyClient { async fn fetch_packs( &self, want: &[String], + depth: Option, ) -> Result> + Send>>, MegaError> { let request_url = format!("{}/git-upload-pack", self.url); - let body = self.generate_upload_pack_content(want); + let body = self.generate_upload_pack_content(want, depth); tracing::debug!("fetch_objects with body {:?}", body); let res = self @@ -175,11 +177,19 @@ impl ThirdPartyRepoTrait for ThirdPartyClient { } impl ThirdPartyClient { - fn generate_upload_pack_content(&self, want: &[String]) -> Bytes { + fn generate_upload_pack_content(&self, want: &[String], depth: Option) -> Bytes { let mut buf = BytesMut::new(); - let mut write_first_line = false; + if let Some(depth) = depth { + self.add_pkt_line_string(&mut buf, format!("deepen {depth}\n")); + } - let capability = ["side-band-64k", "ofs-delta", "multi_ack_detailed"].join(" "); + let mut write_first_line = false; + // Shallow fetches only need a single tip commit; skip multi_ack_detailed negotiation. + let capability = if depth.is_some() { + ["side-band-64k", "ofs-delta", "no-progress"].join(" ") + } else { + ["side-band-64k", "ofs-delta", "multi_ack_detailed"].join(" ") + }; for w in want { if !write_first_line { self.add_pkt_line_string( @@ -220,22 +230,30 @@ impl ThirdPartyClient { Err(_) => break, }; + // A flush (0000) separates negotiation lines (NAK/shallow) from the pack stream. if len == 0 { - break; + continue; + } + + if !reach_pack && data.len() >= 4 && &data[0..4] == b"PACK" { + reach_pack = true; + pack_data.extend_from_slice(&data); + tracing::debug!("Receiving raw PACK data..."); + continue; } if data.len() >= 5 && &data[1..5] == b"PACK" { reach_pack = true; - tracing::debug!("Receiving PACK data..."); + tracing::debug!("Receiving side-band PACK data..."); } if reach_pack { let code = data[0]; - let data = &data[1..]; + let payload = &data[1..]; match code { - 1 => pack_data.extend_from_slice(data), - 2 => tracing::info!("{}", String::from_utf8_lossy(data)), - 3 => tracing::warn!("{}", String::from_utf8_lossy(data)), + 1 => pack_data.extend_from_slice(payload), + 2 => tracing::info!("{}", String::from_utf8_lossy(payload)), + 3 => tracing::warn!("{}", String::from_utf8_lossy(payload)), _ => tracing::warn!("unknown side-band-64k code: {code}"), } } else if &data != b"NAK\n" { @@ -271,3 +289,17 @@ impl ThirdPartyClient { Ok((len as usize, data)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generate_upload_pack_content_includes_deepen_for_shallow_fetch() { + let client = ThirdPartyClient::new("https://github.com/foo/bar.git"); + let body = client.generate_upload_pack_content(&["abc123".to_string()], Some(1)); + let text = String::from_utf8_lossy(&body); + assert!(text.contains("deepen 1")); + assert!(text.contains("want abc123")); + } +} diff --git a/ceres/src/transport/mod.rs b/ceres/src/transport/mod.rs new file mode 100644 index 000000000..150b5ef77 --- /dev/null +++ b/ceres/src/transport/mod.rs @@ -0,0 +1,5 @@ +pub mod pack; +pub mod protocol; +pub mod state; + +pub use state::{ProtocolApiState, TransportRuntime}; diff --git a/ceres/src/pack/import_repo.rs b/ceres/src/transport/pack/import_repo.rs similarity index 69% rename from ceres/src/pack/import_repo.rs rename to ceres/src/transport/pack/import_repo.rs index 8a2b0a865..69b5f9821 100644 --- a/ceres/src/pack/import_repo.rs +++ b/ceres/src/transport/pack/import_repo.rs @@ -34,7 +34,8 @@ use tokio::sync::mpsc::{self, Sender}; use tokio_stream::wrappers::ReceiverStream; use crate::{ - api_service::{cache::GitObjectCache, mono_api_service::MonoApiService, tree_ops}, + api_service::cache::GitObjectCache, + bus::{ApplicationEventHandler, TransportEvent}, pack::RepoHandler, protocol::{ import_refs::{CommandType, RefCommand, Refs}, @@ -48,6 +49,7 @@ pub struct ImportRepo { pub command_list: Mutex>, pub unpack_redlock: Arc, pub git_object_cache: Arc, + pub application: Arc, pub receive_pack_extra_timings_ms: Mutex>, } @@ -102,7 +104,30 @@ impl RepoHandler for ImportRepo { )); let t_attach = Instant::now(); - self.attach_to_monorepo_parent().await?; + let commands = self + .command_list + .lock() + .expect("command_list lock poisoned") + .clone(); + let handler_timings = Arc::new(Mutex::new(Vec::new())); + self.application + .handle(TransportEvent::ImportReceivePackFinalized { + repo_path: PathBuf::from(self.repo.repo_path.clone()), + repo_id: self.repo.repo_id, + commands, + unpack_redlock: self.unpack_redlock.clone(), + extra_timings: Arc::clone(&handler_timings), + }) + .await?; + self.receive_pack_extra_timings_ms + .lock() + .expect("receive_pack_extra_timings_ms lock poisoned") + .extend( + handler_timings + .lock() + .expect("handler_timings lock poisoned") + .drain(..), + ); self.receive_pack_extra_timings_ms .lock() .expect("receive_pack_extra_timings_ms lock poisoned") @@ -438,186 +463,6 @@ impl ImportRepo { Ok(()) } - - // attach import repo to monorepo parent tree - pub(crate) async fn attach_to_monorepo_parent(&self) -> Result<(), MegaError> { - // Snapshot commands without holding the mutex across await (Send + avoids deadlocks). - let commands_snapshot: Vec = self - .command_list - .lock() - .expect("command_list lock poisoned") - .clone(); - let commit_id = match commands_snapshot - .iter() - .find(|c| c.ref_type == RefTypeEnum::Branch) - { - Some(cmd) => cmd.new_id.clone(), - None => return Ok(()), - }; - - let path = PathBuf::from(self.repo.repo_path.clone()); - let mono_api_service: MonoApiService = self.into(); - let storage = self.storage.mono_storage(); - - // Import tip commit + message do not depend on monorepo root; load once so retries - // under the root lock do not repeat git_db reads. - let latest_commit: Commit = Commit::from_git_model( - self.storage - .git_db_storage() - .get_commit_by_hash(self.repo.repo_id, &commit_id) - .await? - .ok_or_else(|| MegaError::Other(format!("commit {commit_id} not found")))?, - ); - let commit_msg = latest_commit.format_message(); - - // Concurrent attaches need CAS on root mega_refs; retry when head moved. - // Redis lock reduces retry storms; DB still enforces correctness via StaleMonorepoRootRef. - // - // `search_and_create_tree` walks the live root tree via the API (`get_root_tree`); it must - // run against the same root snapshot as `get_main_ref` for this attempt, so it stays - // inside the locked section. Further reduction would require threading an explicit root - // tree/commit into `tree_ops` so tree building can run without holding the global lock. - const MAX_ATTACH_ATTEMPTS: u32 = 64; - let mut root_lock_wait_max_ms: u128 = 0; - let mut root_lock_wait_sum_ms: u128 = 0; - - for attempt in 0..MAX_ATTACH_ATTEMPTS { - let t_lock = Instant::now(); - let guard = self.unpack_redlock.clone().lock().await?; - let lock_wait_ms = t_lock.elapsed().as_millis(); - root_lock_wait_max_ms = root_lock_wait_max_ms.max(lock_wait_ms); - root_lock_wait_sum_ms += lock_wait_ms; - - let root_ref = storage - .get_main_ref("/") - .await? - .ok_or_else(|| MegaError::Other("root ref not found".to_string()))?; - let expected_commit = root_ref.ref_commit_hash.clone(); - let expected_tree = root_ref.ref_tree_hash.clone(); - let root_ref_id = root_ref.id; - - let save_trees = tree_ops::search_and_create_tree(&mono_api_service, &path).await?; - - let new_commit = Commit::from_tree_id( - save_trees - .back() - .ok_or_else(|| MegaError::Other("no tree generated".to_string()))? - .id, - vec![ObjectHash::from_str(&expected_commit).unwrap()], - &format!("\n{commit_msg}"), - ); - - let txn = self.storage.begin_db_transaction().await?; - let git_db = self.storage.git_db_storage(); - for cmd in &commands_snapshot { - if cmd.ref_type != RefTypeEnum::Branch { - continue; - } - match cmd.command_type { - CommandType::Create => { - git_db - .save_ref_in_txn(self.repo.repo_id, cmd.clone().into(), &txn) - .await?; - } - CommandType::Delete => { - git_db - .remove_ref_in_txn(self.repo.repo_id, &cmd.ref_name, &txn) - .await?; - } - CommandType::Update => { - git_db - .update_ref_in_txn(self.repo.repo_id, &cmd.ref_name, &cmd.new_id, &txn) - .await?; - } - } - } - - let t_attach_txn = Instant::now(); - match storage - .attach_to_monorepo_parent_in_txn( - &txn, - root_ref_id, - &expected_commit, - &expected_tree, - new_commit, - save_trees.into(), - ) - .await - { - Ok(()) => { - txn.commit().await.map_err(MegaError::Db)?; - let t_unlock = Instant::now(); - guard.unlock().await?; - self.receive_pack_extra_timings_ms - .lock() - .expect("receive_pack_extra_timings_ms lock poisoned") - .extend([ - ( - "import_attach_attempts_count".to_string(), - (attempt + 1) as u128, - ), - ( - "import_root_lock_wait_sum_ms".to_string(), - root_lock_wait_sum_ms, - ), - ( - "import_root_lock_wait_max_ms".to_string(), - root_lock_wait_max_ms, - ), - ( - "import_attach_txn_ms".to_string(), - t_attach_txn.elapsed().as_millis(), - ), - ( - "import_root_lock_unlock_ms".to_string(), - t_unlock.elapsed().as_millis(), - ), - ]); - return Ok(()); - } - Err(MegaError::StaleMonorepoRootRef) if attempt + 1 < MAX_ATTACH_ATTEMPTS => { - let _ = txn.rollback().await; - let _ = guard.unlock().await; - tracing::warn!( - attempt = attempt, - repo_path = %self.repo.repo_path, - "attach_to_monorepo_parent: root ref moved, retrying" - ); - tokio::task::yield_now().await; - } - Err(e) => { - let _ = txn.rollback().await; - let _ = guard.unlock().await; - self.receive_pack_extra_timings_ms - .lock() - .expect("receive_pack_extra_timings_ms lock poisoned") - .extend([ - ( - "import_attach_attempts_count".to_string(), - (attempt + 1) as u128, - ), - ( - "import_root_lock_wait_sum_ms".to_string(), - root_lock_wait_sum_ms, - ), - ( - "import_root_lock_wait_max_ms".to_string(), - root_lock_wait_max_ms, - ), - ( - "import_attach_txn_ms".to_string(), - t_attach_txn.elapsed().as_millis(), - ), - ]); - return Err(e); - } - } - } - - Err(MegaError::Other( - "attach_to_monorepo_parent: exceeded retry limit for concurrent root updates".into(), - )) - } } async fn process_objects( diff --git a/ceres/src/pack/mod.rs b/ceres/src/transport/pack/mod.rs similarity index 94% rename from ceres/src/pack/mod.rs rename to ceres/src/transport/pack/mod.rs index f0939367d..b1433e276 100644 --- a/ceres/src/pack/mod.rs +++ b/ceres/src/transport/pack/mod.rs @@ -33,11 +33,23 @@ use sysinfo::System; use tokio::sync::{Semaphore, mpsc::UnboundedReceiver}; use tokio_stream::wrappers::ReceiverStream; -use crate::protocol::import_refs::{RefCommand, Refs}; +use crate::transport::protocol::import_refs::{RefCommand, Refs}; pub mod import_repo; pub mod monorepo; +/// Framework-neutral receive-pack body stream (decoupled from axum). +pub type PackStreamError = Box; +pub type PackByteStream = Pin> + Send>>; + +pub fn into_pack_byte_stream(stream: S) -> PackByteStream +where + S: Stream> + Send + 'static, + E: std::error::Error + Send + Sync + 'static, +{ + Box::pin(stream.map_err(|e| Box::new(e) as PackStreamError)) +} + #[async_trait] pub trait RepoHandler: Send + Sync + 'static { fn is_monorepo(&self) -> bool; @@ -249,7 +261,7 @@ pub trait RepoHandler: Send + Sync + 'static { async fn unpack_stream( &self, pack_config: &PackConfig, - stream: Pin> + Send>>, + stream: PackByteStream, ) -> Result< ( UnboundedReceiver>, @@ -277,7 +289,10 @@ pub trait RepoHandler: Send + Sync + 'static { Some(pack_config.pack_decode_cache_path.clone()), pack_config.clean_cache_after_decode, ); - p.decode_stream(stream, sender, Some(pack_id_sender)).await; + let decode_stream = + stream.map_err(|e| axum::Error::new(std::io::Error::other(e.to_string()))); + p.decode_stream(decode_stream, sender, Some(pack_id_sender)) + .await; Ok((receiver, pack_id_receiver)) } diff --git a/ceres/src/pack/monorepo.rs b/ceres/src/transport/pack/monorepo.rs similarity index 74% rename from ceres/src/pack/monorepo.rs rename to ceres/src/transport/pack/monorepo.rs index 89016a9a4..8ddf82c70 100644 --- a/ceres/src/pack/monorepo.rs +++ b/ceres/src/transport/pack/monorepo.rs @@ -9,19 +9,15 @@ use std::{ vec, }; -use api_model::common::Pagination; use async_recursion::async_recursion; use async_trait::async_trait; use callisto::{ - entity_ext::generate_link, - mega_cl, mega_code_review_anchor, mega_commit, mega_refs, - sea_orm_active_enums::{PositionStatusEnum, RefTypeEnum}, + entity_ext::generate_link, mega_commit, mega_refs, sea_orm_active_enums::RefTypeEnum, }; use common::{ errors::MegaError, utils::{self, ZERO_ID}, }; -use futures::{StreamExt, stream}; use git_internal::{ errors::GitError, hash::ObjectHash, @@ -35,13 +31,12 @@ use git_internal::{ }; use io_orbit::object_storage::MultiObjectByteStream; use jupiter::{sea_orm::DatabaseTransaction, storage::Storage, utils::converter::FromMegaModel}; -use orion_client::OrionBuildClient; use tokio::sync::{RwLock, mpsc}; use tokio_stream::wrappers::ReceiverStream; use crate::{ - api_service::{ApiHandler, cache::GitObjectCache, mono_api_service::MonoApiService}, - code_edit::{on_push::OnpushCodeEdit, utils::get_changed_files}, + api_service::{cache::GitObjectCache, mono::MonoApiService}, + bus::{ApplicationEventHandler, TransportEvent}, model::change_list::ClDiffFile, pack::RepoHandler, protocol::import_refs::{RefCommand, Refs}, @@ -58,7 +53,7 @@ pub struct MonoRepo { // When only a branch is updated and the pack file is empty, this value will be None. pub current_commit: Arc>>, pub cl_link: Arc>>, - pub orion_client: Arc, + pub application: Arc, pub username: Option, /// Ref commands for this push (same role as on [`ImportRepo`](crate::pack::import_repo::ImportRepo)). pub command_list: Mutex>, @@ -167,7 +162,16 @@ impl RepoHandler for MonoRepo { async fn finalize_receive_pack(&self) -> Result<(), MegaError> { self.persist_mono_branch_cl_mega_refs_transaction().await?; - self.run_mono_post_push_pipeline().await + self.traverses_tree_and_update_filepath().await?; + self.application + .handle(TransportEvent::MonoReceivePackFinalized { + repo_path: self.path.clone(), + base_branch: self.base_branch.clone(), + from_hash: self.from_hash.clone(), + to_hash: self.to_hash.clone(), + username: self.username.clone(), + }) + .await } async fn save_entry( @@ -515,35 +519,9 @@ impl MonoRepo { } Ok(()) } +} - /// CL / conversations / build / code-review hooks after branch `mega_refs` are committed. - async fn run_mono_post_push_pipeline(&self) -> Result<(), MegaError> { - let username = self.username(); - let mono_api_service = self.into(); - let editor = OnpushCodeEdit::from( - self.path.to_str().unwrap(), - &self.base_branch, - &self.from_hash, - &mono_api_service, - ); - let cl = editor - .update_or_create_cl(&self.storage, &self.from_hash, &self.to_hash, &username) - .await?; - self.traverses_tree_and_update_filepath().await?; - if self.orion_client.enable_build() { - editor - .trigger_build_and_check( - self.storage.clone(), - self.git_object_cache.clone(), - self.orion_client.clone(), - &cl, - &username, - ) - .await?; - } - self.reanchor_code_review_threads(&cl).await - } - +impl MonoRepo { #[async_recursion] async fn traverses_and_update_filepath( &self, @@ -628,14 +606,7 @@ impl MonoRepo { .await? { Some(cl) => cl.link.clone(), - None => { - if self.from_hash == "0".repeat(40) { - return Err(MegaError::Other( - "Can not init directory under monorepo directory!".to_string(), - )); - } - generate_link() - } + None => generate_link(), }; let mut lock = self.cl_link.write().await; *lock = Some(cl_link.clone()); @@ -662,148 +633,4 @@ impl MonoRepo { let api_service: MonoApiService = self.into(); api_service.cl_files_list(old_files, new_files).await } - - // Mark code review threads whose anchors may be affected by this change as outdated. - // These threads will require reanchoring to restore accurate code positions. - pub async fn reanchor_code_review_threads(&self, cl: &mega_cl::Model) -> Result<(), MegaError> { - let mono_api_service: MonoApiService = self.into(); - let cl_link = cl.link.clone(); - - // Marks code review threads as outdated if their file paths - // are affected by the latest change list. - let changed_files = get_changed_files(&mono_api_service, cl).await?; - let files_with_threads = self - .storage - .code_review_thread_storage() - .get_files_with_threads_by_link(&cl_link) - .await?; - - let files_with_threads_set: HashSet<&String> = files_with_threads.iter().collect(); - - // Intersection: files that are changed AND have threads - let affected_files: Vec = changed_files - .into_iter() - .filter(|file| files_with_threads_set.contains(file)) - .collect(); - - tracing::info!( - "Reanchor code review thread in cl_link: {}, affected files: {:?}", - cl_link, - affected_files - ); - - let pending_reanchor_threads = self - .storage - .code_review_thread_storage() - .find_threads_by_file_paths(affected_files) - .await?; - - let pending_reanchor_thread_ids: Vec = pending_reanchor_threads - .iter() - .map(|thread| thread.id) - .collect(); - - // Mark as PendingReanchor - self.storage - .code_review_thread_storage() - .mark_positions_status_by_thread_ids( - &pending_reanchor_thread_ids, - PositionStatusEnum::PendingReanchor, - ) - .await?; - - // Start reanchor - let anchors = self - .storage - .code_review_thread_storage() - .get_anchors_by_thread_ids(&pending_reanchor_thread_ids) - .await?; - - let mono_api_service = Arc::new(mono_api_service); - let mut anchors_map: HashMap> = HashMap::new(); - for anchor in anchors { - anchors_map - .entry(anchor.thread_id) - .or_default() - .push(anchor); - } - - let reanchor_tasks: Vec<_> = pending_reanchor_threads - .into_iter() - .map(|thread| { - let cl_link = cl_link.clone(); - let mono_api_service = Arc::clone(&mono_api_service); - let anchors_map = anchors_map.clone(); - let to_hash = self.to_hash.clone(); - - async move { - let thread_id = thread.id; - - let thread_anchors = match anchors_map.get(&thread_id) { - Some(anchors) => anchors, - None => { - tracing::warn!("Thread {} has no anchors", thread_id); - return Err(MegaError::Other(format!( - "Thread {} has no anchors", - thread_id - ))); - } - }; - - let (diff_content, _) = mono_api_service - .paged_content_diff(&cl_link, Pagination::default()) - .await?; - - let mut blob_cache: HashMap = HashMap::new(); - - for anchor in thread_anchors { - let file_path = anchor.file_path.clone(); - - // Fetch blob once per file - let latest_blob = if let Some(blob) = blob_cache.get(&file_path) { - blob.clone() - } else { - let blob = mono_api_service - .get_blob_as_string(PathBuf::from(&file_path), Some(&to_hash)) - .await? - .expect("latest blob must exist"); - - blob_cache.insert(file_path.clone(), blob.clone()); - blob - }; - - // Reanchor - if let Err(e) = self - .storage - .code_review_service - .reanchor_thread( - anchor, - Some(latest_blob), - diff_content.clone(), - &self.to_hash, - ) - .await - { - tracing::error!("Reanchor failed for anchor {}: {:?}", anchor.id, e); - } - } - - Ok(()) - } - }) - .collect::>(); - - let results: Vec> = stream::iter(reanchor_tasks) - .buffer_unordered(self.storage.get_recommended_batch_concurrency()) - .collect() - .await; - - for res in results { - if let Err(e) = res { - tracing::error!("Reanchor task failed: {:?}", e); - } - } - - Ok(()) - } } diff --git a/ceres/src/protocol/import_refs.rs b/ceres/src/transport/protocol/import_refs.rs similarity index 100% rename from ceres/src/protocol/import_refs.rs rename to ceres/src/transport/protocol/import_refs.rs diff --git a/ceres/src/protocol/mod.rs b/ceres/src/transport/protocol/mod.rs similarity index 96% rename from ceres/src/protocol/mod.rs rename to ceres/src/transport/protocol/mod.rs index 53fdf30c1..06d0bc028 100644 --- a/ceres/src/protocol/mod.rs +++ b/ceres/src/transport/protocol/mod.rs @@ -7,18 +7,14 @@ use std::{ }; use callisto::sea_orm_active_enums::RefTypeEnum; -use common::{ - errors::{MegaError, ProtocolError}, - utils::ZERO_ID, -}; +use common::errors::{MegaError, ProtocolError}; use import_refs::RefCommand; use jupiter::redis::lock::RedLock; -use orion_client::OrionBuildClient; use repo::Repo; use tokio::sync::RwLock; use crate::{ - api_service::state::ProtocolApiState, + bus::TransportRuntime, pack::{RepoHandler, import_repo::ImportRepo, monorepo::MonoRepo}, }; @@ -26,6 +22,8 @@ pub mod import_refs; pub mod repo; pub mod smart; +pub use common::utils::ZERO_ID; + #[derive(Clone, Debug)] pub struct PushUserInfo { pub username: String, @@ -160,7 +158,7 @@ impl SmartSession { pub async fn repo_handler_with_commands( &self, - state: &ProtocolApiState, + state: &TransportRuntime, commands: Vec, ) -> Result, ProtocolError> { let config = state.storage.config(); @@ -198,6 +196,7 @@ impl SmartSession { repo, command_list: Mutex::new(commands), unpack_redlock, + application: state.application.clone(), receive_pack_extra_timings_ms: Mutex::new(Vec::new()), }) as Arc) } else { @@ -210,7 +209,7 @@ impl SmartSession { to_hash: String::new(), current_commit: Arc::new(RwLock::new(None)), cl_link: Arc::new(RwLock::new(None)), - orion_client: Arc::new(OrionBuildClient::new(config.build.clone())), + application: state.application.clone(), username: self.auth.username.clone(), command_list: Mutex::new(commands.clone()), }; diff --git a/ceres/src/protocol/repo.rs b/ceres/src/transport/protocol/repo.rs similarity index 100% rename from ceres/src/protocol/repo.rs rename to ceres/src/transport/protocol/repo.rs diff --git a/ceres/src/protocol/smart.rs b/ceres/src/transport/protocol/smart.rs similarity index 98% rename from ceres/src/protocol/smart.rs rename to ceres/src/transport/protocol/smart.rs index 51c25e43d..7267ec324 100644 --- a/ceres/src/protocol/smart.rs +++ b/ceres/src/transport/protocol/smart.rs @@ -1,6 +1,5 @@ use std::{ collections::{BTreeMap, HashSet}, - pin::Pin, time::Instant, }; @@ -8,14 +7,16 @@ use anyhow::Result; use bytes::{Buf, BufMut, Bytes, BytesMut}; use callisto::sea_orm_active_enums::RefTypeEnum; use common::errors::ProtocolError; -use futures::Stream; use tokio_stream::wrappers::ReceiverStream; use crate::{ - api_service::state::ProtocolApiState, - protocol::{ - Capability, ServiceType, SideBind, SmartSession, TransportProtocol, ZERO_ID, - import_refs::RefCommand, + bus::TransportRuntime, + transport::{ + pack::PackByteStream, + protocol::{ + Capability, ServiceType, SideBind, SmartSession, TransportProtocol, ZERO_ID, + import_refs::RefCommand, + }, }, }; @@ -64,7 +65,7 @@ impl SmartSession { /// Tracing information is logged regarding the response packet line stream. /// /// Finally, the constructed packet line stream is returned. - pub async fn git_info_refs(&self, state: &ProtocolApiState) -> Result { + pub async fn git_info_refs(&self, state: &TransportRuntime) -> Result { let repo_handler = self.repo_handler_with_commands(state, Vec::new()).await?; let service_type = self.service_type; @@ -93,7 +94,7 @@ impl SmartSession { pub async fn git_upload_pack( &mut self, - state: &ProtocolApiState, + state: &TransportRuntime, upload_request: &mut Bytes, ) -> Result<(ReceiverStream>, BytesMut), ProtocolError> { let repo_handler = self.repo_handler_with_commands(state, Vec::new()).await?; @@ -218,9 +219,9 @@ impl SmartSession { pub async fn git_receive_pack_stream( &mut self, - state: &ProtocolApiState, + state: &TransportRuntime, commands: Vec, - data_stream: Pin> + Send>>, + data_stream: PackByteStream, ) -> Result { let t0 = Instant::now(); let mut timings_ms: BTreeMap = BTreeMap::new(); @@ -429,7 +430,7 @@ impl SmartSession { } /// Process commit bindings for successfully pushed commits - async fn process_commit_bindings(&self, state: &ProtocolApiState, commands: &[RefCommand]) { + async fn process_commit_bindings(&self, state: &TransportRuntime, commands: &[RefCommand]) { for command in commands { // Only process successful branch updates (not tags or failed commands) if command.ref_type == RefTypeEnum::Branch @@ -446,7 +447,7 @@ impl SmartSession { /// Bind a single commit to a user based on authenticated user only (username-only model) async fn bind_commit_to_user( &self, - state: &ProtocolApiState, + state: &TransportRuntime, commit_sha: &str, ) -> Result<(), Box> { let commit_binding_storage = state.storage.commit_binding_storage(); diff --git a/ceres/src/transport/state.rs b/ceres/src/transport/state.rs new file mode 100644 index 000000000..c43b7cfc4 --- /dev/null +++ b/ceres/src/transport/state.rs @@ -0,0 +1,6 @@ +//! Transport-layer shared state (storage, cache, application event handler). + +pub use crate::bus::TransportRuntime; + +/// Backward-compatible alias for [`TransportRuntime`]. +pub type ProtocolApiState = TransportRuntime; diff --git a/jupiter/src/service/cl_service.rs b/jupiter/src/service/cl_service.rs index f8fe696e7..16044b5d0 100644 --- a/jupiter/src/service/cl_service.rs +++ b/jupiter/src/service/cl_service.rs @@ -35,6 +35,7 @@ impl CLService { } } + #[deprecated(note = "use ceres::MonoApiService::get_cl_details instead")] pub async fn get_cl_details( &self, link: &str, diff --git a/jupiter/src/storage/mono_storage.rs b/jupiter/src/storage/mono_storage.rs index eabaa81d4..9478f6dcb 100644 --- a/jupiter/src/storage/mono_storage.rs +++ b/jupiter/src/storage/mono_storage.rs @@ -135,6 +135,19 @@ impl MonoStorage { Ok(result) } + pub async fn get_ref_at_path( + &self, + path: &str, + ref_name: &str, + ) -> Result, MegaError> { + let result = mega_refs::Entity::find() + .filter(mega_refs::Column::Path.eq(path)) + .filter(mega_refs::Column::RefName.eq(ref_name)) + .one(self.get_connection()) + .await?; + Ok(result) + } + pub async fn get_ref_by_commit( &self, path: &str, @@ -307,6 +320,16 @@ impl MonoStorage { active.update(&conn).await?; Ok::<(), MegaError>(()) }); + } else { + let is_cl = update.ref_name.starts_with("refs/cl/"); + let model = mega_refs::Model::new( + &update.path, + update.ref_name.clone(), + update.commit_id.clone(), + update.tree_hash.clone(), + is_cl, + ); + self.save_refs(model, None).await?; } } @@ -317,6 +340,55 @@ impl MonoStorage { Ok(()) } + pub async fn batch_upsert_ref_updates_in_txn( + &self, + updates: Vec, + txn: &DatabaseTransaction, + ) -> Result<(), MegaError> { + if updates.is_empty() { + return Ok(()); + } + + let mut condition = Condition::any(); + for update in &updates { + condition = condition.add( + Condition::all() + .add(mega_refs::Column::Path.eq(update.path.clone())) + .add(mega_refs::Column::RefName.eq(update.ref_name.clone())), + ); + } + + let existing_refs: Vec = + mega_refs::Entity::find().filter(condition).all(txn).await?; + + let ref_map: HashMap<(String, String), mega_refs::Model> = existing_refs + .into_iter() + .map(|r| ((r.path.clone(), r.ref_name.clone()), r)) + .collect(); + + for update in updates { + if let Some(ref_data) = ref_map.get(&(update.path.clone(), update.ref_name.clone())) { + let mut active: mega_refs::ActiveModel = ref_data.clone().into(); + active.ref_commit_hash = Set(update.commit_id); + active.ref_tree_hash = Set(update.tree_hash); + active.updated_at = Set(chrono::Utc::now().naive_utc()); + active.update(txn).await?; + } else { + let is_cl = update.ref_name.starts_with("refs/cl/"); + let model = mega_refs::Model::new( + &update.path, + update.ref_name.clone(), + update.commit_id.clone(), + update.tree_hash.clone(), + is_cl, + ); + self.save_refs(model, Some(txn)).await?; + } + } + + Ok(()) + } + pub async fn update_blob_filepath( &self, blob_id: &str, diff --git a/jupiter/src/utils/converter.rs b/jupiter/src/utils/converter.rs deleted file mode 100644 index 955d4fdef..000000000 --- a/jupiter/src/utils/converter.rs +++ /dev/null @@ -1,945 +0,0 @@ -use std::{cell::RefCell, collections::HashMap, str::FromStr}; - -use callisto::{ - git_blob, git_commit, git_tag, git_tree, mega_blob, mega_commit, mega_refs, mega_tag, mega_tree, -}; -use common::{ - config::MonoConfig, - utils::{MEGA_BRANCH_NAME, generate_id}, -}; -use git_internal::{ - hash::ObjectHash, - internal::{ - metadata::EntryMeta, - object::{ - ObjectTrait, - blob::Blob, - commit::Commit, - signature::Signature, - tag::Tag, - tree::{Tree, TreeItem, TreeItemMode}, - types::ObjectType, - }, - pack::entry::Entry, - }, -}; - -/// Helper function to convert commit model data to Commit object -fn commit_from_model( - commit_id: &str, - tree: &str, - parents_id: &serde_json::Value, - author: Option, - committer: Option, - content: Option, -) -> Commit { - // Parse parents_id JSON array into Vec - let parent_commit_ids: Vec = - match serde_json::from_value::>(parents_id.clone()) { - Ok(parents_array) => parents_array - .into_iter() - .filter(|s: &String| !s.is_empty()) - .map(|s: String| ObjectHash::from_str(&s).unwrap()) - .collect(), - Err(_) => Vec::new(), - }; - - Commit { - id: ObjectHash::from_str(commit_id).unwrap(), - tree_id: ObjectHash::from_str(tree).unwrap(), - parent_commit_ids, - author: Signature::from_data(author.unwrap().into_bytes()).unwrap(), - committer: Signature::from_data(committer.unwrap().into_bytes()).unwrap(), - message: content.unwrap(), - } -} - -pub trait IntoMegaModel { - type MegaTarget; - fn into_mega_model(self, ext_meta: EntryMeta) -> Self::MegaTarget; -} - -pub trait IntoGitModel { - type GitTarget; - fn into_git_model(self, ext_meta: EntryMeta) -> Self::GitTarget; -} - -pub trait FromMegaModel { - type MegaSource; - fn from_mega_model(model: Self::MegaSource) -> Self; -} - -pub trait FromGitModel { - type GitSource; - fn from_git_model(model: Self::GitSource) -> Self; -} - -#[derive(PartialEq, Debug, Clone)] -pub enum GitObject { - Commit(Commit), - Tree(Tree), - Blob(Blob), - Tag(Tag), -} - -#[derive(PartialEq, Debug, Clone)] -pub enum GitObjectModel { - Commit(git_commit::Model), - Tree(git_tree::Model), - Blob(git_blob::Model, Vec), - Tag(git_tag::Model), -} - -pub enum MegaObjectModel { - Commit(mega_commit::Model), - Tree(mega_tree::Model), - Blob(mega_blob::Model, Vec), - Tag(mega_tag::Model), -} - -impl GitObject { - pub fn convert_to_mega_model(self, meta: EntryMeta) -> MegaObjectModel { - match self { - GitObject::Commit(commit) => MegaObjectModel::Commit(commit.into_mega_model(meta)), - GitObject::Tree(tree) => MegaObjectModel::Tree(tree.into_mega_model(meta)), - GitObject::Blob(blob) => { - let blob_data = blob.data.clone(); - let mega_model = blob.into_mega_model(meta); - MegaObjectModel::Blob(mega_model, blob_data) - } - GitObject::Tag(tag) => MegaObjectModel::Tag(tag.into_mega_model(meta)), - } - } - - pub fn convert_to_git_model(self, meta: EntryMeta) -> GitObjectModel { - match self { - GitObject::Commit(commit) => GitObjectModel::Commit(commit.into_git_model(meta)), - GitObject::Tree(tree) => GitObjectModel::Tree(tree.into_git_model(meta)), - GitObject::Blob(blob) => { - let blob_data = blob.data.clone(); - let git_model = blob.into_git_model(meta); - GitObjectModel::Blob(git_model, blob_data) - } - GitObject::Tag(tag) => GitObjectModel::Tag(tag.into_git_model(meta)), - } - } -} - -pub fn process_entry(entry: Entry) -> GitObject { - match entry.obj_type { - ObjectType::Commit => { - GitObject::Commit(Commit::from_bytes(&entry.data, entry.hash).unwrap()) - } - ObjectType::Tree => GitObject::Tree(Tree::from_bytes(&entry.data, entry.hash).unwrap()), - ObjectType::Blob => GitObject::Blob(Blob::from_bytes(&entry.data, entry.hash).unwrap()), - ObjectType::Tag => GitObject::Tag(Tag::from_bytes(&entry.data, entry.hash).unwrap()), - _ => unreachable!("can not parse delta!"), - } -} - -impl IntoMegaModel for Blob { - type MegaTarget = mega_blob::Model; - - /// Converts a Blob object to a mega_blob::Model - /// - /// This function creates a new mega_blob::Model from a Blob object. - /// The resulting model will have a newly generated ID, the blob's ID as string, - /// and default values for size, commit_id, and name. - /// - /// # Returns - /// - /// A new mega_blob::Model instance populated with data from the blob - fn into_mega_model(self, meta: EntryMeta) -> Self::MegaTarget { - mega_blob::Model { - id: generate_id(), - blob_id: self.id.to_string(), - size: 0, - commit_id: String::new(), - name: String::new(), - pack_id: meta.pack_id.unwrap_or_default(), - file_path: meta.file_path.unwrap_or_default(), - pack_offset: meta.pack_offset.unwrap_or(0) as i64, - is_delta_in_pack: meta.is_delta.unwrap_or(false), - created_at: chrono::Utc::now().naive_utc(), - } - } -} - -impl IntoMegaModel for Commit { - type MegaTarget = mega_commit::Model; - - /// Converts a Commit object to a mega_commit::Model - /// - /// This function transforms a Commit object into a mega_commit::Model for database storage. - /// It preserves all relevant commit metadata including tree reference, parent commit IDs, - /// author information, committer information, and commit message. - /// - /// # Returns - /// - /// A new mega_commit::Model instance populated with data from the commit - /// - /// # Panics - /// - /// This function will panic if author or committer signature data cannot be converted to bytes - fn into_mega_model(self, meta: EntryMeta) -> Self::MegaTarget { - mega_commit::Model { - id: generate_id(), - commit_id: self.id.to_string(), - tree: self.tree_id.to_string(), - parents_id: self - .parent_commit_ids - .iter() - .map(|x| x.to_string()) - .collect(), - author: Some(String::from_utf8_lossy(&self.author.to_data().unwrap()).to_string()), - committer: Some( - String::from_utf8_lossy(&self.committer.to_data().unwrap()).to_string(), - ), - content: Some(self.message.clone()), - pack_id: meta.pack_id.unwrap_or_default(), - pack_offset: meta.pack_offset.unwrap_or(0) as i64, - created_at: chrono::Utc::now().naive_utc(), - } - } -} - -impl IntoMegaModel for Tag { - type MegaTarget = mega_tag::Model; - - /// Converts a Tag object to a mega_tag::Model - /// - /// This function transforms a Tag object into a mega_tag::Model for database storage. - /// It preserves all tag metadata including the referenced object hash, object type, - /// tag name, tagger information, and tag message. - /// - /// # Returns - /// - /// A new mega_tag::Model instance populated with data from the tag - /// - /// # Panics - /// - /// This function will panic if tagger signature data cannot be converted to bytes - fn into_mega_model(self, meta: EntryMeta) -> Self::MegaTarget { - mega_tag::Model { - id: generate_id(), - tag_id: self.id.to_string(), - object_id: self.object_hash.to_string(), - object_type: self.object_type.to_string(), - tag_name: self.tag_name, - tagger: String::from_utf8_lossy(&self.tagger.to_data().unwrap()).to_string(), - message: self.message, - pack_id: meta.pack_id.unwrap_or_default(), - pack_offset: meta.pack_offset.unwrap_or(0) as i64, - created_at: chrono::Utc::now().naive_utc(), - } - } -} - -impl IntoMegaModel for Tree { - type MegaTarget = mega_tree::Model; - - /// Converts a Tree object to a mega_tree::Model - /// - /// This function transforms a Tree object into a mega_tree::Model for database storage. - /// It serializes the tree structure into binary data and stores essential metadata - /// like the tree ID. Size is set to 0 and commit_id to an empty string by default. - /// - /// # Returns - /// - /// A new mega_tree::Model instance populated with data from the tree - /// - /// # Panics - /// - /// This function will panic if the tree's data cannot be serialized - fn into_mega_model(self, meta: EntryMeta) -> Self::MegaTarget { - mega_tree::Model { - id: generate_id(), - tree_id: self.id.to_string(), - sub_trees: self.to_data().unwrap(), - size: 0, - commit_id: String::new(), - pack_id: meta.pack_id.unwrap_or_default(), - pack_offset: meta.pack_offset.unwrap_or(0) as i64, - created_at: chrono::Utc::now().naive_utc(), - } - } -} - -impl IntoGitModel for Blob { - type GitTarget = git_blob::Model; - - /// Converts a Blob object to a git_blob::Model - /// - /// This function creates a new git_blob::Model from a Blob object. - /// The resulting model will have a newly generated ID, the blob's ID as string, - /// repository ID set to 0, and default values for size and name. - /// - /// # Returns - /// - /// A new git_blob::Model instance populated with data from the blob - fn into_git_model(self, meta: EntryMeta) -> Self::GitTarget { - git_blob::Model { - id: generate_id(), - repo_id: 0, - blob_id: self.id.to_string(), - size: 0, - name: None, - pack_id: meta.pack_id.unwrap_or_default(), - pack_offset: meta.pack_offset.unwrap_or(0) as i64, - file_path: meta.file_path.unwrap_or_default(), - is_delta_in_pack: meta.is_delta.unwrap_or(false), - created_at: chrono::Utc::now().naive_utc(), - } - } -} - -impl IntoGitModel for Commit { - type GitTarget = git_commit::Model; - - /// Converts a Commit object to a git_commit::Model - /// - /// This function transforms a Commit object into a git_commit::Model for Git repository - /// database storage. It preserves all relevant commit metadata including tree reference, - /// parent commit IDs, author information, committer information, and commit message. - /// The repository ID is set to 0 by default. - /// - /// # Returns - /// - /// A new git_commit::Model instance populated with data from the commit - /// - /// # Panics - /// - /// This function will panic if author or committer signature data cannot be converted to bytes - fn into_git_model(self, meta: EntryMeta) -> Self::GitTarget { - git_commit::Model { - id: generate_id(), - repo_id: 0, - commit_id: self.id.to_string(), - tree: self.tree_id.to_string(), - parents_id: self - .parent_commit_ids - .iter() - .map(|x| x.to_string()) - .collect(), - author: Some(String::from_utf8_lossy(&self.author.to_data().unwrap()).to_string()), - committer: Some( - String::from_utf8_lossy(&self.committer.to_data().unwrap()).to_string(), - ), - content: Some(self.message.clone()), - pack_id: meta.pack_id.unwrap_or_default(), - pack_offset: meta.pack_offset.unwrap_or(0) as i64, - created_at: chrono::Utc::now().naive_utc(), - } - } -} - -impl IntoGitModel for Tag { - type GitTarget = git_tag::Model; - - /// Converts a Tag object to a git_tag::Model - /// - /// This function transforms a Tag object into a git_tag::Model for Git repository - /// database storage. It preserves all tag metadata including the referenced object hash, - /// object type, tag name, tagger information, and tag message. - /// The repository ID is set to 0 by default. - /// - /// # Returns - /// - /// A new git_tag::Model instance populated with data from the tag - /// - /// # Panics - /// - /// This function will panic if tagger signature data cannot be converted to bytes - fn into_git_model(self, meta: EntryMeta) -> Self::GitTarget { - git_tag::Model { - id: generate_id(), - repo_id: 0, - tag_id: self.id.to_string(), - object_id: self.object_hash.to_string(), - object_type: self.object_type.to_string(), - tag_name: self.tag_name, - tagger: String::from_utf8_lossy(&self.tagger.to_data().unwrap()).to_string(), - message: self.message, - pack_id: meta.pack_id.unwrap_or_default(), - pack_offset: meta.pack_offset.unwrap_or(0) as i64, - created_at: chrono::Utc::now().naive_utc(), - } - } -} - -impl IntoGitModel for Tree { - type GitTarget = git_tree::Model; - - /// Converts a Tree object to a git_tree::Model - /// - /// This function transforms a Tree object into a git_tree::Model for Git repository - /// database storage. It serializes the tree structure into binary data and stores - /// essential metadata like the tree ID. The repository ID is set to 0 and size - /// is set to 0 by default. - /// - /// # Returns - /// - /// A new git_tree::Model instance populated with data from the tree - /// - /// # Panics - /// - /// This function will panic if the tree's data cannot be serialized - fn into_git_model(self, meta: EntryMeta) -> Self::GitTarget { - git_tree::Model { - id: generate_id(), - repo_id: 0, - tree_id: self.id.to_string(), - sub_trees: self.to_data().unwrap(), - size: 0, - pack_id: meta.pack_id.unwrap_or_default(), - pack_offset: meta.pack_offset.unwrap_or(0) as i64, - created_at: chrono::Utc::now().naive_utc(), - } - } -} - -pub fn generate_git_keep() -> Blob { - let git_keep_content = String::from("This file was used to maintain the git tree"); - Blob::from_content(&git_keep_content) -} - -pub fn generate_git_keep_with_timestamp() -> Blob { - let git_keep_content = format!( - "This file was used to maintain the git tree, generate at:{}", - chrono::Utc::now().naive_utc() - ); - Blob::from_content(&git_keep_content) -} - -pub fn init_trees( - mono_config: &MonoConfig, -) -> (HashMap, HashMap, Tree) { - let mut root_items = Vec::new(); - let mut trees = Vec::new(); - let mut blobs = Vec::new(); - - // Create unique .gitkeep for each root directory to ensure different tree hashes - for dir in mono_config.root_dirs.clone() { - let gitkeep_content = format!("Placeholder file for /{} directory", dir); - let gitkeep_blob = Blob::from_content(&gitkeep_content); - blobs.push(gitkeep_blob.clone()); - - let tree_item = TreeItem { - mode: TreeItemMode::Blob, - id: gitkeep_blob.id, - name: String::from(".gitkeep"), - }; - let tree = Tree::from_tree_items(vec![tree_item]).unwrap(); - root_items.push(TreeItem { - mode: TreeItemMode::Tree, - id: tree.id, - name: dir, - }); - trees.push(tree); - } - - // Create global .mega_cedar.json in root directory - let entity_str = saturn::entitystore::generate_entity(&mono_config.admin, "/").unwrap(); - let cedar_blob = Blob::from_content(&entity_str); - root_items.push(TreeItem { - mode: TreeItemMode::Blob, - id: cedar_blob.id, - name: String::from(".mega_cedar.json"), - }); - blobs.push(cedar_blob); - - inject_cedar_policy_dir(&mut root_items, &mut trees, &mut blobs, &mono_config.admin); - - inject_root_buck_files(&mut root_items, &mut blobs); - - // Ensure the `toolchains` cell has a BUCK file at repo initialization time. - if let Some(toolchains_root_idx) = root_items.iter().position(|item| { - item.mode == TreeItemMode::Tree - && item - .name - .trim_start_matches('/') - .trim_start_matches("./") - .trim_end_matches('/') - == "toolchains" - }) { - let toolchains_tree_id = root_items[toolchains_root_idx].id; - if let Some(toolchains_tree_idx) = trees.iter().position(|t| t.id == toolchains_tree_id) { - let mut toolchains_items = trees[toolchains_tree_idx].tree_items.clone(); - inject_toolchains_buck_file(&mut toolchains_items, &mut blobs); - let toolchains_tree = Tree::from_tree_items(toolchains_items).unwrap(); - trees[toolchains_tree_idx] = toolchains_tree.clone(); - root_items[toolchains_root_idx].id = toolchains_tree.id; - } - } - - let root = Tree::from_tree_items(root_items).unwrap(); - ( - trees.into_iter().map(|x| (x.id, x)).collect(), - blobs.into_iter().map(|x| (x.id, x)).collect(), - root, - ) -} - -/// Injects Buck configuration files (.buckroot and .buckconfig) into the root directory. -fn inject_root_buck_files(root_items: &mut Vec, blobs: &mut Vec) { - // .buckroot - let buckroot_content = generate_buckroot_content(); - let buckroot_blob = Blob::from_content(&buckroot_content); - root_items.push(TreeItem { - mode: TreeItemMode::Blob, - id: buckroot_blob.id, - name: String::from(".buckroot"), - }); - blobs.push(buckroot_blob); - - // .buckconfig - let buckconfig_content = generate_buckconfig_content(); - let buckconfig_blob = Blob::from_content(&buckconfig_content); - root_items.push(TreeItem { - mode: TreeItemMode::Blob, - id: buckconfig_blob.id, - name: String::from(".buckconfig"), - }); - blobs.push(buckconfig_blob); -} - -/// Injects a BUCK file into the toolchains directory. -fn inject_toolchains_buck_file(toolchains_items: &mut Vec, blobs: &mut Vec) { - let toolchains_content = generate_toolchains_buck_content(); - let toolchains_blob = Blob::from_content(&toolchains_content); - toolchains_items.push(TreeItem { - mode: TreeItemMode::Blob, - id: toolchains_blob.id, - name: String::from("BUCK"), - }); - blobs.push(toolchains_blob); -} - -fn generate_toolchains_buck_content() -> String { - r#"load("@prelude//toolchains:demo.bzl", "system_demo_toolchains") - -# All the default toolchains, suitable for a quick demo or early prototyping. -# Most real projects should copy/paste the implementation to configure them. -system_demo_toolchains() -"# - .to_string() -} - -fn generate_buckroot_content() -> String { - // The .buckroot file is usually empty or contains a simple identifier. - String::new() -} - -/// Generates Cedar policy content that sets admins as default reviewers. -/// -/// Creates a policy rule that requires all admin users to review changes -/// across the entire repository (empty path pattern matches all paths). -fn generate_cedar_policy_content(admin_users: &[String]) -> String { - if admin_users.is_empty() { - return String::new(); - } - - // Format reviewer list: ["user1", "user2", ...] - let reviewers_formatted = admin_users - .iter() - .map(|u| format!(r#""{}""#, u)) - .collect::>() - .join(", "); - - generate_cedar_policy_template().replace("{}", &reviewers_formatted) -} - -/// Returns the default Cedar policy template for the repository root. -fn generate_cedar_policy_template() -> &'static str { - r#"permit(action == "code:review", principal, resource) - when { resource.path.startsWith("") } - to [{}]; -"# -} - -fn inject_cedar_policy_dir( - root_items: &mut Vec, - trees: &mut Vec, - blobs: &mut Vec, - admin_users: &[String], -) { - let policy_content = generate_cedar_policy_content(admin_users); - let policy_blob = Blob::from_content(&policy_content); - blobs.push(policy_blob.clone()); - - // Create .cedar directory with policies.cedar file - let cedar_tree_item = TreeItem { - mode: TreeItemMode::Blob, - id: policy_blob.id, - name: String::from("policies.cedar"), - }; - let cedar_tree = Tree::from_tree_items(vec![cedar_tree_item]).unwrap(); - trees.push(cedar_tree.clone()); - - // Add .cedar directory to root - root_items.push(TreeItem { - mode: TreeItemMode::Tree, - id: cedar_tree.id, - name: String::from(".cedar"), - }); -} - -fn generate_buckconfig_content() -> String { - let cells = [ - " root = .", - " prelude = prelude", - " toolchains = toolchains", - " buckal = toolchains/buckal-bundles", - " none = none", - ] - .join("\n"); - - format!( - r#"[cells] -{cells} - -[cell_aliases] - config = prelude - ovr_config = prelude - fbcode = none - fbsource = none - fbcode_macros = none - buck = none - -# Uses a copy of the prelude bundled with the buck2 binary. You can alternatively delete this -# section and vendor a copy of the prelude to the `prelude` directory of your project. -[external_cells] - prelude = bundled - -[parser] - target_platform_detector_spec = target:root//...->prelude//platforms:default \ - target:prelude//...->prelude//platforms:default \ - target:toolchains//...->prelude//platforms:default - -[build] - execution_platforms = prelude//platforms:default - default_target_platforms = prelude//platforms:default -"# - ) -} - -pub struct MegaModelConverter { - pub commit: Commit, - pub root_tree: Tree, - pub tree_maps: HashMap, - pub blob_maps: HashMap, - pub mega_trees: RefCell>, - pub mega_blobs: RefCell>, - pub raw_blobs: RefCell>, - pub refs: mega_refs::ActiveModel, -} - -impl MegaModelConverter { - fn traverse_from_root(&self) { - let root_tree = &self.root_tree; - let mut mega_tree: mega_tree::Model = root_tree.clone().into_mega_model(EntryMeta::new()); - mega_tree.commit_id = self.commit.id.to_string(); - self.mega_trees - .borrow_mut() - .insert(root_tree.id, mega_tree.clone().into()); - self.traverse_for_update(root_tree); - } - - fn traverse_for_update(&self, tree: &Tree) { - for item in &tree.tree_items { - if item.mode == TreeItemMode::Tree { - let child_tree = self.tree_maps.get(&item.id).unwrap(); - let mut mega_tree: mega_tree::Model = - child_tree.clone().into_mega_model(EntryMeta::new()); - mega_tree.commit_id = self.commit.id.to_string(); - self.mega_trees - .borrow_mut() - .insert(child_tree.id, mega_tree.clone().into()); - self.traverse_for_update(child_tree); - } else { - let blob = self.blob_maps.get(&item.id).unwrap(); - let mut mega_blob: mega_blob::Model = - blob.clone().into_mega_model(EntryMeta::new()); - mega_blob.commit_id = self.commit.id.to_string(); - self.mega_blobs - .borrow_mut() - .insert(blob.id, mega_blob.clone().into()); - - self.raw_blobs.borrow_mut().push(blob.clone()); - } - } - } - - pub fn init(mono_config: &MonoConfig) -> Self { - let (tree_maps, blob_maps, root_tree) = init_trees(mono_config); - let commit = Commit::from_tree_id(root_tree.id, vec![], "\nInit Mega Directory"); - - let mega_ref = mega_refs::Model { - id: generate_id(), - path: "/".to_owned(), - ref_name: MEGA_BRANCH_NAME.to_owned(), - ref_commit_hash: commit.id.to_string(), - ref_tree_hash: commit.tree_id.to_string(), - created_at: chrono::Utc::now().naive_utc(), - updated_at: chrono::Utc::now().naive_utc(), - is_cl: false, - }; - - let converter = MegaModelConverter { - commit, - root_tree, - tree_maps, - blob_maps, - mega_trees: RefCell::new(HashMap::new()), - mega_blobs: RefCell::new(HashMap::new()), - raw_blobs: RefCell::new(Vec::new()), - refs: mega_ref.into(), - }; - converter.traverse_from_root(); - converter - } -} - -// Reverse conversion implementations -impl FromMegaModel for Tag { - type MegaSource = mega_tag::Model; - - /// Converts a mega_tag::Model to a Tag object - /// - /// This function reconstructs a Tag object from a mega_tag::Model retrieved from the database. - /// It parses the stored strings back into their original types, such as converting - /// string IDs back to ObjectHash hashes and string data back to a Signature. - /// - /// # Arguments - /// - /// * `model` - The mega_tag::Model to convert - /// - /// # Returns - /// - /// A new Tag instance populated with data from the model - /// - /// # Panics - /// - /// This function will panic if: - /// - The tag_id string cannot be parsed into a valid ObjectHash - /// - The object_id string cannot be parsed into a valid ObjectHash - /// - The object_type string is not a recognized ObjectType - /// - The tagger string cannot be converted into a valid Signature - fn from_mega_model(model: Self::MegaSource) -> Self { - Tag { - id: ObjectHash::from_str(&model.tag_id).expect("Invalid tag_id in database"), - object_hash: ObjectHash::from_str(&model.object_id).unwrap(), - object_type: ObjectType::from_string(&model.object_type).unwrap(), - tag_name: model.tag_name, - tagger: Signature::from_data(model.tagger.into_bytes()).unwrap(), - message: model.message, - } - } -} - -impl FromGitModel for Tag { - type GitSource = git_tag::Model; - - /// Converts a git_tag::Model to a Tag object - /// - /// This function reconstructs a Tag object from a git_tag::Model retrieved from the database. - /// It parses the stored strings back into their original types, such as converting - /// string IDs back to ObjectHash hashes and string data back to a Signature. - /// - /// # Arguments - /// - /// * `model` - The git_tag::Model to convert - /// - /// # Returns - /// - /// A new Tag instance populated with data from the model - /// - /// # Panics - /// - /// This function will panic if: - /// - The tag_id string cannot be parsed into a valid ObjectHash - /// - The object_id string cannot be parsed into a valid ObjectHash - /// - The object_type string is not a recognized ObjectType - /// - The tagger string cannot be converted into a valid Signature - fn from_git_model(model: Self::GitSource) -> Self { - Tag { - id: ObjectHash::from_str(&model.tag_id).unwrap(), - object_hash: ObjectHash::from_str(&model.object_id).unwrap(), - object_type: ObjectType::from_string(&model.object_type).unwrap(), - tag_name: model.tag_name, - tagger: Signature::from_data(model.tagger.into_bytes()).unwrap(), - message: model.message, - } - } -} - -impl FromMegaModel for Tree { - type MegaSource = mega_tree::Model; - - /// Converts a mega_tree::Model to a Tree object - /// - /// This function reconstructs a Tree object from a mega_tree::Model retrieved from the database. - /// It parses the binary sub_trees data back into a structured Tree object and - /// uses the tree_id string to recreate the ObjectHash identifier. - /// - /// # Arguments - /// - /// * `model` - The mega_tree::Model to convert - /// - /// # Returns - /// - /// A new Tree instance reconstructed from the model data - /// - /// # Panics - /// - /// This function will panic if: - /// - The tree_id string cannot be parsed into a valid ObjectHash - /// - The binary sub_trees data cannot be parsed into a valid Tree structure - fn from_mega_model(model: Self::MegaSource) -> Self { - Tree::from_bytes( - &model.sub_trees, - ObjectHash::from_str(&model.tree_id).unwrap(), - ) - .unwrap() - } -} - -impl FromGitModel for Tree { - type GitSource = git_tree::Model; - - /// Converts a git_tree::Model to a Tree object - /// - /// This function reconstructs a Tree object from a git_tree::Model retrieved from the database. - /// It parses the binary sub_trees data back into a structured Tree object and - /// uses the tree_id string to recreate the ObjectHash hash identifier. - /// - /// # Arguments - /// - /// * `model` - The git_tree::Model to convert - /// - /// # Returns - /// - /// A new Tree instance reconstructed from the model data - /// - /// # Panics - /// - /// This function will panic if: - /// - The tree_id string cannot be parsed into a valid ObjectHash - /// - The binary sub_trees data cannot be parsed into a valid Tree structure - fn from_git_model(model: Self::GitSource) -> Self { - Tree::from_bytes( - &model.sub_trees, - ObjectHash::from_str(&model.tree_id).unwrap(), - ) - .unwrap() - } -} - -impl FromMegaModel for Commit { - type MegaSource = mega_commit::Model; - - /// Converts a mega_commit::Model to a Commit object - /// - /// This function reconstructs a Commit object from a mega_commit::Model retrieved from the database. - /// It parses the stored strings back into their original types, such as converting - /// string IDs back to ObjectHash hashes and string data back to Signature objects. - /// - /// # Arguments - /// - /// * `model` - The mega_commit::Model to convert - /// - /// # Returns - /// - /// A new Commit instance populated with data from the model - /// - /// # Panics - /// - /// This function will panic if: - /// - The commit_id string cannot be parsed into a valid ObjectHash - /// - The tree string cannot be parsed into a valid ObjectHash - /// - Any parent ID in parents_id cannot be parsed into a valid ObjectHash - /// - The author or committer strings cannot be converted into valid Signatures - fn from_mega_model(model: Self::MegaSource) -> Self { - commit_from_model( - &model.commit_id, - &model.tree, - &model.parents_id, - model.author, - model.committer, - model.content, - ) - } -} - -impl FromGitModel for Commit { - type GitSource = git_commit::Model; - - /// Converts a git_commit::Model to a Commit object - /// - /// This function reconstructs a Commit object from a git_commit::Model retrieved from the database. - /// It parses the stored strings back into their original types, such as converting - /// string IDs back to ObjectHash hashes and string data back to Signature objects. - /// - /// # Arguments - /// - /// * `model` - The git_commit::Model to convert - /// - /// # Returns - /// - /// A new Commit instance populated with data from the model - /// - /// # Panics - /// - /// This function will panic if: - /// - The commit_id string cannot be parsed into a valid ObjectHash hash - /// - The tree string cannot be parsed into a valid ObjectHash hash - /// - Any parent ID in parents_id cannot be parsed into a valid ObjectHash hash - /// - The author or committer strings cannot be converted into valid Signatures - fn from_git_model(model: Self::GitSource) -> Self { - commit_from_model( - &model.commit_id, - &model.tree, - &model.parents_id, - model.author, - model.committer, - model.content, - ) - } -} - -#[cfg(test)] -mod test { - - use std::str::FromStr; - - use common::config::MonoConfig; - use git_internal::{hash::ObjectHash, internal::object::commit::Commit}; - - use crate::utils::converter::MegaModelConverter; - - #[test] - pub fn test_init_mega_dir() { - let mut mono_config = MonoConfig::default(); - if !mono_config.root_dirs.iter().any(|d| d == "toolchains") { - mono_config.root_dirs.push("toolchains".to_string()); - } - let converter = MegaModelConverter::init(&mono_config); - let mega_trees = converter.mega_trees.borrow().clone(); - let mega_blobs = converter.mega_blobs.borrow().clone(); - let dir_nums = mono_config.root_dirs.len(); - // Trees: dir_nums (sub-directories) + 1 (root) + 1 (.cedar directory) - assert_eq!(mega_trees.len(), dir_nums + 2); - // Blobs: dir_nums (.gitkeep) + 1 (.mega_cedar.json) + 2 (.buckroot + .buckconfig) + 1 (toolchains/BUCK) + 1 (policies.cedar) - assert_eq!(mega_blobs.len(), dir_nums + 5); - } - - #[test] - pub fn test_init_commit() { - let commit = Commit::from_tree_id( - ObjectHash::from_str("bd4a28f2d8b2efc371f557c3b80d320466ed83f3").unwrap(), - vec![], - "\nInit Mega Directory", - ); - println!("{commit}"); - } -} diff --git a/jupiter/src/utils/converter/from_db.rs b/jupiter/src/utils/converter/from_db.rs new file mode 100644 index 000000000..29b3bc3d2 --- /dev/null +++ b/jupiter/src/utils/converter/from_db.rs @@ -0,0 +1,247 @@ +use std::str::FromStr; + +use callisto::{git_commit, git_tag, git_tree, mega_commit, mega_tag, mega_tree}; +use git_internal::{ + hash::ObjectHash, + internal::object::{ + ObjectTrait, commit::Commit, signature::Signature, tag::Tag, tree::Tree, types::ObjectType, + }, +}; + +use super::traits::{FromGitModel, FromMegaModel}; + +/// Helper function to convert commit model data to Commit object +fn commit_from_model( + commit_id: &str, + tree: &str, + parents_id: &serde_json::Value, + author: Option, + committer: Option, + content: Option, +) -> Commit { + // Parse parents_id JSON array into Vec + let parent_commit_ids: Vec = + match serde_json::from_value::>(parents_id.clone()) { + Ok(parents_array) => parents_array + .into_iter() + .filter(|s: &String| !s.is_empty()) + .map(|s: String| ObjectHash::from_str(&s).unwrap()) + .collect(), + Err(_) => Vec::new(), + }; + + Commit { + id: ObjectHash::from_str(commit_id).unwrap(), + tree_id: ObjectHash::from_str(tree).unwrap(), + parent_commit_ids, + author: Signature::from_data(author.unwrap().into_bytes()).unwrap(), + committer: Signature::from_data(committer.unwrap().into_bytes()).unwrap(), + message: content.unwrap(), + } +} +// Reverse conversion implementations +impl FromMegaModel for Tag { + type MegaSource = mega_tag::Model; + + /// Converts a mega_tag::Model to a Tag object + /// + /// This function reconstructs a Tag object from a mega_tag::Model retrieved from the database. + /// It parses the stored strings back into their original types, such as converting + /// string IDs back to ObjectHash hashes and string data back to a Signature. + /// + /// # Arguments + /// + /// * `model` - The mega_tag::Model to convert + /// + /// # Returns + /// + /// A new Tag instance populated with data from the model + /// + /// # Panics + /// + /// This function will panic if: + /// - The tag_id string cannot be parsed into a valid ObjectHash + /// - The object_id string cannot be parsed into a valid ObjectHash + /// - The object_type string is not a recognized ObjectType + /// - The tagger string cannot be converted into a valid Signature + fn from_mega_model(model: Self::MegaSource) -> Self { + Tag { + id: ObjectHash::from_str(&model.tag_id).expect("Invalid tag_id in database"), + object_hash: ObjectHash::from_str(&model.object_id).unwrap(), + object_type: ObjectType::from_string(&model.object_type).unwrap(), + tag_name: model.tag_name, + tagger: Signature::from_data(model.tagger.into_bytes()).unwrap(), + message: model.message, + } + } +} + +impl FromGitModel for Tag { + type GitSource = git_tag::Model; + + /// Converts a git_tag::Model to a Tag object + /// + /// This function reconstructs a Tag object from a git_tag::Model retrieved from the database. + /// It parses the stored strings back into their original types, such as converting + /// string IDs back to ObjectHash hashes and string data back to a Signature. + /// + /// # Arguments + /// + /// * `model` - The git_tag::Model to convert + /// + /// # Returns + /// + /// A new Tag instance populated with data from the model + /// + /// # Panics + /// + /// This function will panic if: + /// - The tag_id string cannot be parsed into a valid ObjectHash + /// - The object_id string cannot be parsed into a valid ObjectHash + /// - The object_type string is not a recognized ObjectType + /// - The tagger string cannot be converted into a valid Signature + fn from_git_model(model: Self::GitSource) -> Self { + Tag { + id: ObjectHash::from_str(&model.tag_id).unwrap(), + object_hash: ObjectHash::from_str(&model.object_id).unwrap(), + object_type: ObjectType::from_string(&model.object_type).unwrap(), + tag_name: model.tag_name, + tagger: Signature::from_data(model.tagger.into_bytes()).unwrap(), + message: model.message, + } + } +} + +impl FromMegaModel for Tree { + type MegaSource = mega_tree::Model; + + /// Converts a mega_tree::Model to a Tree object + /// + /// This function reconstructs a Tree object from a mega_tree::Model retrieved from the database. + /// It parses the binary sub_trees data back into a structured Tree object and + /// uses the tree_id string to recreate the ObjectHash identifier. + /// + /// # Arguments + /// + /// * `model` - The mega_tree::Model to convert + /// + /// # Returns + /// + /// A new Tree instance reconstructed from the model data + /// + /// # Panics + /// + /// This function will panic if: + /// - The tree_id string cannot be parsed into a valid ObjectHash + /// - The binary sub_trees data cannot be parsed into a valid Tree structure + fn from_mega_model(model: Self::MegaSource) -> Self { + Tree::from_bytes( + &model.sub_trees, + ObjectHash::from_str(&model.tree_id).unwrap(), + ) + .unwrap() + } +} + +impl FromGitModel for Tree { + type GitSource = git_tree::Model; + + /// Converts a git_tree::Model to a Tree object + /// + /// This function reconstructs a Tree object from a git_tree::Model retrieved from the database. + /// It parses the binary sub_trees data back into a structured Tree object and + /// uses the tree_id string to recreate the ObjectHash hash identifier. + /// + /// # Arguments + /// + /// * `model` - The git_tree::Model to convert + /// + /// # Returns + /// + /// A new Tree instance reconstructed from the model data + /// + /// # Panics + /// + /// This function will panic if: + /// - The tree_id string cannot be parsed into a valid ObjectHash + /// - The binary sub_trees data cannot be parsed into a valid Tree structure + fn from_git_model(model: Self::GitSource) -> Self { + Tree::from_bytes( + &model.sub_trees, + ObjectHash::from_str(&model.tree_id).unwrap(), + ) + .unwrap() + } +} + +impl FromMegaModel for Commit { + type MegaSource = mega_commit::Model; + + /// Converts a mega_commit::Model to a Commit object + /// + /// This function reconstructs a Commit object from a mega_commit::Model retrieved from the database. + /// It parses the stored strings back into their original types, such as converting + /// string IDs back to ObjectHash hashes and string data back to Signature objects. + /// + /// # Arguments + /// + /// * `model` - The mega_commit::Model to convert + /// + /// # Returns + /// + /// A new Commit instance populated with data from the model + /// + /// # Panics + /// + /// This function will panic if: + /// - The commit_id string cannot be parsed into a valid ObjectHash + /// - The tree string cannot be parsed into a valid ObjectHash + /// - Any parent ID in parents_id cannot be parsed into a valid ObjectHash + /// - The author or committer strings cannot be converted into valid Signatures + fn from_mega_model(model: Self::MegaSource) -> Self { + commit_from_model( + &model.commit_id, + &model.tree, + &model.parents_id, + model.author, + model.committer, + model.content, + ) + } +} + +impl FromGitModel for Commit { + type GitSource = git_commit::Model; + + /// Converts a git_commit::Model to a Commit object + /// + /// This function reconstructs a Commit object from a git_commit::Model retrieved from the database. + /// It parses the stored strings back into their original types, such as converting + /// string IDs back to ObjectHash hashes and string data back to Signature objects. + /// + /// # Arguments + /// + /// * `model` - The git_commit::Model to convert + /// + /// # Returns + /// + /// A new Commit instance populated with data from the model + /// + /// # Panics + /// + /// This function will panic if: + /// - The commit_id string cannot be parsed into a valid ObjectHash hash + /// - The tree string cannot be parsed into a valid ObjectHash hash + /// - Any parent ID in parents_id cannot be parsed into a valid ObjectHash hash + /// - The author or committer strings cannot be converted into valid Signatures + fn from_git_model(model: Self::GitSource) -> Self { + commit_from_model( + &model.commit_id, + &model.tree, + &model.parents_id, + model.author, + model.committer, + model.content, + ) + } +} diff --git a/jupiter/src/utils/converter/init_monorepo.rs b/jupiter/src/utils/converter/init_monorepo.rs new file mode 100644 index 000000000..282da005f --- /dev/null +++ b/jupiter/src/utils/converter/init_monorepo.rs @@ -0,0 +1,322 @@ +use std::{cell::RefCell, collections::HashMap}; + +use callisto::{mega_blob, mega_refs, mega_tree}; +use common::{ + config::MonoConfig, + utils::{MEGA_BRANCH_NAME, generate_id}, +}; +use git_internal::{ + hash::ObjectHash, + internal::{ + metadata::EntryMeta, + object::{ + blob::Blob, + commit::Commit, + tree::{Tree, TreeItem, TreeItemMode}, + }, + }, +}; + +use super::traits::IntoMegaModel; + +pub fn generate_git_keep() -> Blob { + let git_keep_content = String::from("This file was used to maintain the git tree"); + Blob::from_content(&git_keep_content) +} + +pub fn generate_git_keep_with_timestamp() -> Blob { + let git_keep_content = format!( + "This file was used to maintain the git tree, generate at:{}", + chrono::Utc::now().naive_utc() + ); + Blob::from_content(&git_keep_content) +} + +pub fn init_trees( + mono_config: &MonoConfig, +) -> (HashMap, HashMap, Tree) { + let mut root_items = Vec::new(); + let mut trees = Vec::new(); + let mut blobs = Vec::new(); + + // Create unique .gitkeep for each root directory to ensure different tree hashes + for dir in mono_config.root_dirs.clone() { + let gitkeep_content = format!("Placeholder file for /{} directory", dir); + let gitkeep_blob = Blob::from_content(&gitkeep_content); + blobs.push(gitkeep_blob.clone()); + + let tree_item = TreeItem { + mode: TreeItemMode::Blob, + id: gitkeep_blob.id, + name: String::from(".gitkeep"), + }; + let tree = Tree::from_tree_items(vec![tree_item]).unwrap(); + root_items.push(TreeItem { + mode: TreeItemMode::Tree, + id: tree.id, + name: dir, + }); + trees.push(tree); + } + + // Create global .mega_cedar.json in root directory + let entity_str = saturn::entitystore::generate_entity(&mono_config.admin, "/").unwrap(); + let cedar_blob = Blob::from_content(&entity_str); + root_items.push(TreeItem { + mode: TreeItemMode::Blob, + id: cedar_blob.id, + name: String::from(".mega_cedar.json"), + }); + blobs.push(cedar_blob); + + inject_cedar_policy_dir(&mut root_items, &mut trees, &mut blobs, &mono_config.admin); + + inject_root_buck_files(&mut root_items, &mut blobs); + + // Ensure the `toolchains` cell has a BUCK file at repo initialization time. + if let Some(toolchains_root_idx) = root_items.iter().position(|item| { + item.mode == TreeItemMode::Tree + && item + .name + .trim_start_matches('/') + .trim_start_matches("./") + .trim_end_matches('/') + == "toolchains" + }) { + let toolchains_tree_id = root_items[toolchains_root_idx].id; + if let Some(toolchains_tree_idx) = trees.iter().position(|t| t.id == toolchains_tree_id) { + let mut toolchains_items = trees[toolchains_tree_idx].tree_items.clone(); + inject_toolchains_buck_file(&mut toolchains_items, &mut blobs); + let toolchains_tree = Tree::from_tree_items(toolchains_items).unwrap(); + trees[toolchains_tree_idx] = toolchains_tree.clone(); + root_items[toolchains_root_idx].id = toolchains_tree.id; + } + } + + let root = Tree::from_tree_items(root_items).unwrap(); + ( + trees.into_iter().map(|x| (x.id, x)).collect(), + blobs.into_iter().map(|x| (x.id, x)).collect(), + root, + ) +} + +/// Injects Buck configuration files (.buckroot and .buckconfig) into the root directory. +fn inject_root_buck_files(root_items: &mut Vec, blobs: &mut Vec) { + // .buckroot + let buckroot_content = generate_buckroot_content(); + let buckroot_blob = Blob::from_content(&buckroot_content); + root_items.push(TreeItem { + mode: TreeItemMode::Blob, + id: buckroot_blob.id, + name: String::from(".buckroot"), + }); + blobs.push(buckroot_blob); + + // .buckconfig + let buckconfig_content = generate_buckconfig_content(); + let buckconfig_blob = Blob::from_content(&buckconfig_content); + root_items.push(TreeItem { + mode: TreeItemMode::Blob, + id: buckconfig_blob.id, + name: String::from(".buckconfig"), + }); + blobs.push(buckconfig_blob); +} + +/// Injects a BUCK file into the toolchains directory. +fn inject_toolchains_buck_file(toolchains_items: &mut Vec, blobs: &mut Vec) { + let toolchains_content = generate_toolchains_buck_content(); + let toolchains_blob = Blob::from_content(&toolchains_content); + toolchains_items.push(TreeItem { + mode: TreeItemMode::Blob, + id: toolchains_blob.id, + name: String::from("BUCK"), + }); + blobs.push(toolchains_blob); +} + +fn generate_toolchains_buck_content() -> String { + r#"load("@prelude//toolchains:demo.bzl", "system_demo_toolchains") + +# All the default toolchains, suitable for a quick demo or early prototyping. +# Most real projects should copy/paste the implementation to configure them. +system_demo_toolchains() +"# + .to_string() +} + +fn generate_buckroot_content() -> String { + // The .buckroot file is usually empty or contains a simple identifier. + String::new() +} + +/// Generates Cedar policy content that sets admins as default reviewers. +/// +/// Creates a policy rule that requires all admin users to review changes +/// across the entire repository (empty path pattern matches all paths). +fn generate_cedar_policy_content(admin_users: &[String]) -> String { + if admin_users.is_empty() { + return String::new(); + } + + // Format reviewer list: ["user1", "user2", ...] + let reviewers_formatted = admin_users + .iter() + .map(|u| format!(r#""{}""#, u)) + .collect::>() + .join(", "); + + generate_cedar_policy_template().replace("{}", &reviewers_formatted) +} + +/// Returns the default Cedar policy template for the repository root. +fn generate_cedar_policy_template() -> &'static str { + r#"permit(action == "code:review", principal, resource) + when { resource.path.startsWith("") } + to [{}]; +"# +} + +fn inject_cedar_policy_dir( + root_items: &mut Vec, + trees: &mut Vec, + blobs: &mut Vec, + admin_users: &[String], +) { + let policy_content = generate_cedar_policy_content(admin_users); + let policy_blob = Blob::from_content(&policy_content); + blobs.push(policy_blob.clone()); + + // Create .cedar directory with policies.cedar file + let cedar_tree_item = TreeItem { + mode: TreeItemMode::Blob, + id: policy_blob.id, + name: String::from("policies.cedar"), + }; + let cedar_tree = Tree::from_tree_items(vec![cedar_tree_item]).unwrap(); + trees.push(cedar_tree.clone()); + + // Add .cedar directory to root + root_items.push(TreeItem { + mode: TreeItemMode::Tree, + id: cedar_tree.id, + name: String::from(".cedar"), + }); +} + +fn generate_buckconfig_content() -> String { + let cells = [ + " root = .", + " prelude = prelude", + " toolchains = toolchains", + " buckal = toolchains/buckal-bundles", + " none = none", + ] + .join("\n"); + + format!( + r#"[cells] +{cells} + +[cell_aliases] + config = prelude + ovr_config = prelude + fbcode = none + fbsource = none + fbcode_macros = none + buck = none + +# Uses a copy of the prelude bundled with the buck2 binary. You can alternatively delete this +# section and vendor a copy of the prelude to the `prelude` directory of your project. +[external_cells] + prelude = bundled + +[parser] + target_platform_detector_spec = target:root//...->prelude//platforms:default \ + target:prelude//...->prelude//platforms:default \ + target:toolchains//...->prelude//platforms:default + +[build] + execution_platforms = prelude//platforms:default + default_target_platforms = prelude//platforms:default +"# + ) +} + +pub struct MegaModelConverter { + pub commit: Commit, + pub root_tree: Tree, + pub tree_maps: HashMap, + pub blob_maps: HashMap, + pub mega_trees: RefCell>, + pub mega_blobs: RefCell>, + pub raw_blobs: RefCell>, + pub refs: mega_refs::ActiveModel, +} + +impl MegaModelConverter { + fn traverse_from_root(&self) { + let root_tree = &self.root_tree; + let mut mega_tree: mega_tree::Model = root_tree.clone().into_mega_model(EntryMeta::new()); + mega_tree.commit_id = self.commit.id.to_string(); + self.mega_trees + .borrow_mut() + .insert(root_tree.id, mega_tree.clone().into()); + self.traverse_for_update(root_tree); + } + + fn traverse_for_update(&self, tree: &Tree) { + for item in &tree.tree_items { + if item.mode == TreeItemMode::Tree { + let child_tree = self.tree_maps.get(&item.id).unwrap(); + let mut mega_tree: mega_tree::Model = + child_tree.clone().into_mega_model(EntryMeta::new()); + mega_tree.commit_id = self.commit.id.to_string(); + self.mega_trees + .borrow_mut() + .insert(child_tree.id, mega_tree.clone().into()); + self.traverse_for_update(child_tree); + } else { + let blob = self.blob_maps.get(&item.id).unwrap(); + let mut mega_blob: mega_blob::Model = + blob.clone().into_mega_model(EntryMeta::new()); + mega_blob.commit_id = self.commit.id.to_string(); + self.mega_blobs + .borrow_mut() + .insert(blob.id, mega_blob.clone().into()); + + self.raw_blobs.borrow_mut().push(blob.clone()); + } + } + } + + pub fn init(mono_config: &MonoConfig) -> Self { + let (tree_maps, blob_maps, root_tree) = init_trees(mono_config); + let commit = Commit::from_tree_id(root_tree.id, vec![], "\nInit Mega Directory"); + + let mega_ref = mega_refs::Model { + id: generate_id(), + path: "/".to_owned(), + ref_name: MEGA_BRANCH_NAME.to_owned(), + ref_commit_hash: commit.id.to_string(), + ref_tree_hash: commit.tree_id.to_string(), + created_at: chrono::Utc::now().naive_utc(), + updated_at: chrono::Utc::now().naive_utc(), + is_cl: false, + }; + + let converter = MegaModelConverter { + commit, + root_tree, + tree_maps, + blob_maps, + mega_trees: RefCell::new(HashMap::new()), + mega_blobs: RefCell::new(HashMap::new()), + raw_blobs: RefCell::new(Vec::new()), + refs: mega_ref.into(), + }; + converter.traverse_from_root(); + converter + } +} diff --git a/jupiter/src/utils/converter/mod.rs b/jupiter/src/utils/converter/mod.rs new file mode 100644 index 000000000..5396c7b2f --- /dev/null +++ b/jupiter/src/utils/converter/mod.rs @@ -0,0 +1,12 @@ +mod from_db; +mod init_monorepo; +mod pack; +mod to_db; +mod traits; + +pub use init_monorepo::*; +pub use pack::*; +pub use traits::*; + +#[cfg(test)] +mod test; diff --git a/jupiter/src/utils/converter/pack.rs b/jupiter/src/utils/converter/pack.rs new file mode 100644 index 000000000..1cc22e8e5 --- /dev/null +++ b/jupiter/src/utils/converter/pack.rs @@ -0,0 +1,18 @@ +use git_internal::internal::{ + object::{ObjectTrait, blob::Blob, commit::Commit, tag::Tag, tree::Tree, types::ObjectType}, + pack::entry::Entry, +}; + +use super::traits::GitObject; + +pub fn process_entry(entry: Entry) -> GitObject { + match entry.obj_type { + ObjectType::Commit => { + GitObject::Commit(Commit::from_bytes(&entry.data, entry.hash).unwrap()) + } + ObjectType::Tree => GitObject::Tree(Tree::from_bytes(&entry.data, entry.hash).unwrap()), + ObjectType::Blob => GitObject::Blob(Blob::from_bytes(&entry.data, entry.hash).unwrap()), + ObjectType::Tag => GitObject::Tag(Tag::from_bytes(&entry.data, entry.hash).unwrap()), + _ => unreachable!("can not parse delta!"), + } +} diff --git a/jupiter/src/utils/converter/test.rs b/jupiter/src/utils/converter/test.rs new file mode 100644 index 000000000..2c4a62ac4 --- /dev/null +++ b/jupiter/src/utils/converter/test.rs @@ -0,0 +1,30 @@ +use std::str::FromStr; + +use common::config::MonoConfig; +use git_internal::{hash::ObjectHash, internal::object::commit::Commit}; + +use super::MegaModelConverter; + +#[test] +pub fn test_init_mega_dir() { + let mut mono_config = MonoConfig::default(); + if !mono_config.root_dirs.iter().any(|d| d == "toolchains") { + mono_config.root_dirs.push("toolchains".to_string()); + } + let converter = MegaModelConverter::init(&mono_config); + let mega_trees = converter.mega_trees.borrow().clone(); + let mega_blobs = converter.mega_blobs.borrow().clone(); + let dir_nums = mono_config.root_dirs.len(); + assert_eq!(mega_trees.len(), dir_nums + 2); + assert_eq!(mega_blobs.len(), dir_nums + 5); +} + +#[test] +pub fn test_init_commit() { + let commit = Commit::from_tree_id( + ObjectHash::from_str("bd4a28f2d8b2efc371f557c3b80d320466ed83f3").unwrap(), + vec![], + "\nInit Mega Directory", + ); + println!("{commit}"); +} diff --git a/jupiter/src/utils/converter/to_db.rs b/jupiter/src/utils/converter/to_db.rs new file mode 100644 index 000000000..a137e7935 --- /dev/null +++ b/jupiter/src/utils/converter/to_db.rs @@ -0,0 +1,271 @@ +use callisto::{ + git_blob, git_commit, git_tag, git_tree, mega_blob, mega_commit, mega_tag, mega_tree, +}; +use common::utils::generate_id; +use git_internal::internal::{ + metadata::EntryMeta, + object::{ObjectTrait, blob::Blob, commit::Commit, tag::Tag, tree::Tree}, +}; + +use super::traits::{IntoGitModel, IntoMegaModel}; + +impl IntoMegaModel for Blob { + type MegaTarget = mega_blob::Model; + + /// Converts a Blob object to a mega_blob::Model + /// + /// This function creates a new mega_blob::Model from a Blob object. + /// The resulting model will have a newly generated ID, the blob's ID as string, + /// and default values for size, commit_id, and name. + /// + /// # Returns + /// + /// A new mega_blob::Model instance populated with data from the blob + fn into_mega_model(self, meta: EntryMeta) -> Self::MegaTarget { + mega_blob::Model { + id: generate_id(), + blob_id: self.id.to_string(), + size: 0, + commit_id: String::new(), + name: String::new(), + pack_id: meta.pack_id.unwrap_or_default(), + file_path: meta.file_path.unwrap_or_default(), + pack_offset: meta.pack_offset.unwrap_or(0) as i64, + is_delta_in_pack: meta.is_delta.unwrap_or(false), + created_at: chrono::Utc::now().naive_utc(), + } + } +} + +impl IntoMegaModel for Commit { + type MegaTarget = mega_commit::Model; + + /// Converts a Commit object to a mega_commit::Model + /// + /// This function transforms a Commit object into a mega_commit::Model for database storage. + /// It preserves all relevant commit metadata including tree reference, parent commit IDs, + /// author information, committer information, and commit message. + /// + /// # Returns + /// + /// A new mega_commit::Model instance populated with data from the commit + /// + /// # Panics + /// + /// This function will panic if author or committer signature data cannot be converted to bytes + fn into_mega_model(self, meta: EntryMeta) -> Self::MegaTarget { + mega_commit::Model { + id: generate_id(), + commit_id: self.id.to_string(), + tree: self.tree_id.to_string(), + parents_id: self + .parent_commit_ids + .iter() + .map(|x| x.to_string()) + .collect(), + author: Some(String::from_utf8_lossy(&self.author.to_data().unwrap()).to_string()), + committer: Some( + String::from_utf8_lossy(&self.committer.to_data().unwrap()).to_string(), + ), + content: Some(self.message.clone()), + pack_id: meta.pack_id.unwrap_or_default(), + pack_offset: meta.pack_offset.unwrap_or(0) as i64, + created_at: chrono::Utc::now().naive_utc(), + } + } +} + +impl IntoMegaModel for Tag { + type MegaTarget = mega_tag::Model; + + /// Converts a Tag object to a mega_tag::Model + /// + /// This function transforms a Tag object into a mega_tag::Model for database storage. + /// It preserves all tag metadata including the referenced object hash, object type, + /// tag name, tagger information, and tag message. + /// + /// # Returns + /// + /// A new mega_tag::Model instance populated with data from the tag + /// + /// # Panics + /// + /// This function will panic if tagger signature data cannot be converted to bytes + fn into_mega_model(self, meta: EntryMeta) -> Self::MegaTarget { + mega_tag::Model { + id: generate_id(), + tag_id: self.id.to_string(), + object_id: self.object_hash.to_string(), + object_type: self.object_type.to_string(), + tag_name: self.tag_name, + tagger: String::from_utf8_lossy(&self.tagger.to_data().unwrap()).to_string(), + message: self.message, + pack_id: meta.pack_id.unwrap_or_default(), + pack_offset: meta.pack_offset.unwrap_or(0) as i64, + created_at: chrono::Utc::now().naive_utc(), + } + } +} + +impl IntoMegaModel for Tree { + type MegaTarget = mega_tree::Model; + + /// Converts a Tree object to a mega_tree::Model + /// + /// This function transforms a Tree object into a mega_tree::Model for database storage. + /// It serializes the tree structure into binary data and stores essential metadata + /// like the tree ID. Size is set to 0 and commit_id to an empty string by default. + /// + /// # Returns + /// + /// A new mega_tree::Model instance populated with data from the tree + /// + /// # Panics + /// + /// This function will panic if the tree's data cannot be serialized + fn into_mega_model(self, meta: EntryMeta) -> Self::MegaTarget { + mega_tree::Model { + id: generate_id(), + tree_id: self.id.to_string(), + sub_trees: self.to_data().unwrap(), + size: 0, + commit_id: String::new(), + pack_id: meta.pack_id.unwrap_or_default(), + pack_offset: meta.pack_offset.unwrap_or(0) as i64, + created_at: chrono::Utc::now().naive_utc(), + } + } +} + +impl IntoGitModel for Blob { + type GitTarget = git_blob::Model; + + /// Converts a Blob object to a git_blob::Model + /// + /// This function creates a new git_blob::Model from a Blob object. + /// The resulting model will have a newly generated ID, the blob's ID as string, + /// repository ID set to 0, and default values for size and name. + /// + /// # Returns + /// + /// A new git_blob::Model instance populated with data from the blob + fn into_git_model(self, meta: EntryMeta) -> Self::GitTarget { + git_blob::Model { + id: generate_id(), + repo_id: 0, + blob_id: self.id.to_string(), + size: 0, + name: None, + pack_id: meta.pack_id.unwrap_or_default(), + pack_offset: meta.pack_offset.unwrap_or(0) as i64, + file_path: meta.file_path.unwrap_or_default(), + is_delta_in_pack: meta.is_delta.unwrap_or(false), + created_at: chrono::Utc::now().naive_utc(), + } + } +} + +impl IntoGitModel for Commit { + type GitTarget = git_commit::Model; + + /// Converts a Commit object to a git_commit::Model + /// + /// This function transforms a Commit object into a git_commit::Model for Git repository + /// database storage. It preserves all relevant commit metadata including tree reference, + /// parent commit IDs, author information, committer information, and commit message. + /// The repository ID is set to 0 by default. + /// + /// # Returns + /// + /// A new git_commit::Model instance populated with data from the commit + /// + /// # Panics + /// + /// This function will panic if author or committer signature data cannot be converted to bytes + fn into_git_model(self, meta: EntryMeta) -> Self::GitTarget { + git_commit::Model { + id: generate_id(), + repo_id: 0, + commit_id: self.id.to_string(), + tree: self.tree_id.to_string(), + parents_id: self + .parent_commit_ids + .iter() + .map(|x| x.to_string()) + .collect(), + author: Some(String::from_utf8_lossy(&self.author.to_data().unwrap()).to_string()), + committer: Some( + String::from_utf8_lossy(&self.committer.to_data().unwrap()).to_string(), + ), + content: Some(self.message.clone()), + pack_id: meta.pack_id.unwrap_or_default(), + pack_offset: meta.pack_offset.unwrap_or(0) as i64, + created_at: chrono::Utc::now().naive_utc(), + } + } +} + +impl IntoGitModel for Tag { + type GitTarget = git_tag::Model; + + /// Converts a Tag object to a git_tag::Model + /// + /// This function transforms a Tag object into a git_tag::Model for Git repository + /// database storage. It preserves all tag metadata including the referenced object hash, + /// object type, tag name, tagger information, and tag message. + /// The repository ID is set to 0 by default. + /// + /// # Returns + /// + /// A new git_tag::Model instance populated with data from the tag + /// + /// # Panics + /// + /// This function will panic if tagger signature data cannot be converted to bytes + fn into_git_model(self, meta: EntryMeta) -> Self::GitTarget { + git_tag::Model { + id: generate_id(), + repo_id: 0, + tag_id: self.id.to_string(), + object_id: self.object_hash.to_string(), + object_type: self.object_type.to_string(), + tag_name: self.tag_name, + tagger: String::from_utf8_lossy(&self.tagger.to_data().unwrap()).to_string(), + message: self.message, + pack_id: meta.pack_id.unwrap_or_default(), + pack_offset: meta.pack_offset.unwrap_or(0) as i64, + created_at: chrono::Utc::now().naive_utc(), + } + } +} + +impl IntoGitModel for Tree { + type GitTarget = git_tree::Model; + + /// Converts a Tree object to a git_tree::Model + /// + /// This function transforms a Tree object into a git_tree::Model for Git repository + /// database storage. It serializes the tree structure into binary data and stores + /// essential metadata like the tree ID. The repository ID is set to 0 and size + /// is set to 0 by default. + /// + /// # Returns + /// + /// A new git_tree::Model instance populated with data from the tree + /// + /// # Panics + /// + /// This function will panic if the tree's data cannot be serialized + fn into_git_model(self, meta: EntryMeta) -> Self::GitTarget { + git_tree::Model { + id: generate_id(), + repo_id: 0, + tree_id: self.id.to_string(), + sub_trees: self.to_data().unwrap(), + size: 0, + pack_id: meta.pack_id.unwrap_or_default(), + pack_offset: meta.pack_offset.unwrap_or(0) as i64, + created_at: chrono::Utc::now().naive_utc(), + } + } +} diff --git a/jupiter/src/utils/converter/traits.rs b/jupiter/src/utils/converter/traits.rs new file mode 100644 index 000000000..2ee3a7b80 --- /dev/null +++ b/jupiter/src/utils/converter/traits.rs @@ -0,0 +1,78 @@ +use callisto::{ + git_blob, git_commit, git_tag, git_tree, mega_blob, mega_commit, mega_tag, mega_tree, +}; +use git_internal::internal::{ + metadata::EntryMeta, + object::{blob::Blob, commit::Commit, tag::Tag, tree::Tree}, +}; + +pub trait IntoMegaModel { + type MegaTarget; + fn into_mega_model(self, ext_meta: EntryMeta) -> Self::MegaTarget; +} + +pub trait IntoGitModel { + type GitTarget; + fn into_git_model(self, ext_meta: EntryMeta) -> Self::GitTarget; +} + +pub trait FromMegaModel { + type MegaSource; + fn from_mega_model(model: Self::MegaSource) -> Self; +} + +pub trait FromGitModel { + type GitSource; + fn from_git_model(model: Self::GitSource) -> Self; +} + +#[derive(PartialEq, Debug, Clone)] +pub enum GitObject { + Commit(Commit), + Tree(Tree), + Blob(Blob), + Tag(Tag), +} + +#[derive(PartialEq, Debug, Clone)] +pub enum GitObjectModel { + Commit(git_commit::Model), + Tree(git_tree::Model), + Blob(git_blob::Model, Vec), + Tag(git_tag::Model), +} + +pub enum MegaObjectModel { + Commit(mega_commit::Model), + Tree(mega_tree::Model), + Blob(mega_blob::Model, Vec), + Tag(mega_tag::Model), +} + +impl GitObject { + pub fn convert_to_mega_model(self, meta: EntryMeta) -> MegaObjectModel { + match self { + GitObject::Commit(commit) => MegaObjectModel::Commit(commit.into_mega_model(meta)), + GitObject::Tree(tree) => MegaObjectModel::Tree(tree.into_mega_model(meta)), + GitObject::Blob(blob) => { + let blob_data = blob.data.clone(); + let mega_model = blob.into_mega_model(meta); + MegaObjectModel::Blob(mega_model, blob_data) + } + GitObject::Tag(tag) => MegaObjectModel::Tag(tag.into_mega_model(meta)), + } + } + + pub fn convert_to_git_model(self, meta: EntryMeta) -> GitObjectModel { + match self { + GitObject::Commit(commit) => GitObjectModel::Commit(commit.into_git_model(meta)), + GitObject::Tree(tree) => GitObjectModel::Tree(tree.into_git_model(meta)), + GitObject::Blob(blob) => { + let blob_data = blob.data.clone(); + let git_model = blob.into_git_model(meta); + GitObjectModel::Blob(git_model, blob_data) + } + GitObject::Tag(tag) => GitObjectModel::Tag(tag.into_git_model(meta)), + } + } +} diff --git a/mono/src/api/api_common/group_permission.rs b/mono/src/api/api_common/group_permission.rs index f7d95dfce..05b37d89c 100644 --- a/mono/src/api/api_common/group_permission.rs +++ b/mono/src/api/api_common/group_permission.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use callisto::sea_orm_active_enums::ResourceTypeEnum; use ceres::{ - api_service::group_ops::EffectiveResourcePermission, + api_service::mono::EffectiveResourcePermission, model::group::{PermissionValue, ResourceTypeValue, UserEffectivePermissionResponse}, }; use http::StatusCode; @@ -29,19 +29,11 @@ pub async fn resolve_resource_context( resource_type: &str, resource_id: &str, ) -> Result<(ResourceTypeEnum, ResourceTypeValue, String), ApiError> { - let resource_type_value = ResourceTypeValue::try_from(resource_type).map_err(|err| { - tracing::warn!("invalid resource_type in request path"); - ApiError::bad_request(anyhow!(err)) - })?; - - let validated_resource_id = - resolve_resource_id(state, resource_type_value, resource_id).await?; - - Ok(( - resource_type_value.into(), - resource_type_value, - validated_resource_id, - )) + state + .monorepo() + .resolve_resource_context(resource_type, resource_id) + .await + .map_err(ApiError::from) } pub fn build_user_effective_permission_response( @@ -64,40 +56,6 @@ pub fn build_user_effective_permission_response( } } -async fn resolve_resource_id( - state: &MonoApiServiceState, - resource_type: ResourceTypeValue, - resource_id: &str, -) -> Result { - let normalized_resource_id = resource_id.trim(); - if normalized_resource_id.is_empty() { - tracing::warn!("empty resource_id in request path"); - return Err(ApiError::bad_request(anyhow!( - "resource_id must not be empty" - ))); - } - - match resource_type { - ResourceTypeValue::Note => { - let note = state - .note_stg() - .get_note_by_public_id(normalized_resource_id) - .await?; - match note { - Some(note) => Ok(note.public_id), - None => { - tracing::warn!( - resource_id = normalized_resource_id, - "note resource missing in mono notes table; falling back to raw public_id" - ); - // TODO: Remove this fallback when note resources are fully migrated into mono notes. - Ok(normalized_resource_id.to_string()) - } - } - } - } -} - fn has_permission(current: Option, required: PermissionValue) -> bool { current .map(|value| value.satisfies(required)) diff --git a/mono/src/api/mod.rs b/mono/src/api/mod.rs index 695f896ae..b51da946b 100644 --- a/mono/src/api/mod.rs +++ b/mono/src/api/mod.rs @@ -7,20 +7,18 @@ use axum::extract::FromRef; use ceres::{ api_service::{ ApiHandler, cache::GitObjectCache, import_api_service::ImportApiService, - mono_api_service::MonoApiService, state::ProtocolApiState, + mono::MonoApiService, state::ProtocolApiState, }, + application::artifact::ArtifactApplicationService, build_trigger::service::BuildTriggerService, protocol::repo::Repo, }; use common::errors::ProtocolError; -use jupiter::{ - service::webhook_service::WebhookService, - storage::{ - NotificationStorage, Storage, cl_storage::ClStorage, - conversation_storage::ConversationStorage, dynamic_sidebar_storage::DynamicSidebarStorage, - gpg_storage::GpgStorage, issue_storage::IssueStorage, note_storage::NoteStorage, - user_storage::UserStorage, webhook_storage::WebhookStorage, - }, +use jupiter::storage::{ + NotificationStorage, Storage, cl_storage::ClStorage, conversation_storage::ConversationStorage, + dynamic_sidebar_storage::DynamicSidebarStorage, gpg_storage::GpgStorage, + issue_storage::IssueStorage, note_storage::NoteStorage, user_storage::UserStorage, + webhook_storage::WebhookStorage, }; use orion_client::OrionBuildClient; use saturn::entitystore::EntityStore; @@ -81,10 +79,7 @@ impl From<&MonoApiServiceState> for MonoApiService { impl FromRef for ProtocolApiState { fn from_ref(state: &MonoApiServiceState) -> ProtocolApiState { - ProtocolApiState { - storage: state.storage.clone(), - git_object_cache: state.git_object_cache.clone(), - } + ProtocolApiState::new(state.storage.clone(), state.git_object_cache.clone()) } } @@ -125,14 +120,14 @@ impl MonoApiServiceState { self.storage.webhook_storage() } - fn webhook_svc(&self) -> WebhookService { - self.storage.webhook_service.clone() - } - fn dynamic_sidebar_stg(&self) -> DynamicSidebarStorage { self.storage.dynamic_sidebar_storage() } + pub fn artifact_app_service(&self) -> ArtifactApplicationService { + ArtifactApplicationService::from_storage(&self.storage) + } + pub fn build_trigger_service(&self) -> BuildTriggerService { BuildTriggerService::new( self.storage.clone(), diff --git a/mono/src/api/router/artifacts_router.rs b/mono/src/api/router/artifacts_router.rs index ef675e835..65757f539 100644 --- a/mono/src/api/router/artifacts_router.rs +++ b/mono/src/api/router/artifacts_router.rs @@ -70,7 +70,7 @@ pub async fn discovery( State(state): State, Path(_repo): Path, ) -> Result, ApiError> { - Ok(Json(state.storage.artifact_service.discovery_response())) + Ok(Json(state.artifact_app_service().discovery_response())) } /// List committed artifact sets for a repo (paginated). @@ -205,7 +205,7 @@ pub async fn download_object( ) -> Result { let repo = decode_path_segment(&repo); let oid = decode_path_segment(&oid); - let svc = &state.storage.artifact_service; + let svc = &state.artifact_app_service(); let model = svc .artifact_object_model_for_committed_repo_download(&repo, &oid) @@ -339,7 +339,7 @@ pub async fn head_artifact_object( ) -> Result { let repo = decode_path_segment(&repo); let oid = decode_path_segment(&oid); - let svc = &state.storage.artifact_service; + let svc = &state.artifact_app_service(); let model = svc .artifact_object_model_for_committed_repo_download(&repo, &oid) .await diff --git a/mono/src/api/router/buck_router.rs b/mono/src/api/router/buck_router.rs index ffa7c2e16..9c0561d52 100644 --- a/mono/src/api/router/buck_router.rs +++ b/mono/src/api/router/buck_router.rs @@ -144,7 +144,7 @@ async fn upload_file( })?; // Get max_file_size from BuckService - let max_size = state.storage.buck_service.max_file_size(); + let max_size = state.monorepo().buck_max_file_size(); if file_size > max_size { return Err(ApiError::with_status( StatusCode::PAYLOAD_TOO_LARGE, @@ -154,9 +154,8 @@ async fn upload_file( // Acquire permits through BuckService let (_global_permit, _large_file_permit) = state - .storage - .buck_service - .try_acquire_upload_permits(file_size) + .monorepo() + .buck_try_acquire_upload_permits(file_size) .map_err(|e| { tracing::warn!( "Buck upload rate limited: cl_link={}, file_size={}, user={}, error={}", @@ -209,9 +208,8 @@ async fn upload_file( .map_err(|e| ApiError::bad_request(anyhow::anyhow!("Failed to read body: {}", e)))?; let svc_resp = state - .storage - .buck_service - .upload_file( + .monorepo() + .upload_buck_file( &user.username, &cl_link, &file_path, diff --git a/mono/src/api/router/cl_router.rs b/mono/src/api/router/cl_router.rs index d031b1609..b5d1a8bf0 100644 --- a/mono/src/api/router/cl_router.rs +++ b/mono/src/api/router/cl_router.rs @@ -3,29 +3,24 @@ use axum::{ Json, extract::{Path, State}, }; -use callisto::sea_orm_active_enums::{ConvTypeEnum, MergeStatusEnum}; use ceres::model::{ change_list::{ - AssigneeUpdatePayload, CLDetailRes, ClFilesRes, Condition, FilesChangedPage, ListPayload, - MergeBoxRes, MuiTreeNode, UpdateBranchStatusRes, UpdateClStatusPayload, + AssigneeUpdatePayload, CLDetailRes, ClFilesRes, FilesChangedPage, ListPayload, MergeBoxRes, + MuiTreeNode, UpdateBranchStatusRes, UpdateClStatusPayload, }, conversation::ContentPayload, issue::ItemRes, label::LabelUpdatePayload, }; use common::errors::MegaError; -use jupiter::service::{cl_service::CLService, webhook_service::WebhookEvent}; use utoipa_axum::{router::OpenApiRouter, routes}; -use crate::{ - api::{ - MonoApiServiceState, - api_common::{self}, - api_doc::CL_TAG, - error::ApiError, - oauth::model::LoginUser, - }, - notification::triggers, +use crate::api::{ + MonoApiServiceState, + api_common::{self}, + api_doc::CL_TAG, + error::ApiError, + oauth::model::LoginUser, }; pub fn routers() -> OpenApiRouter { @@ -69,31 +64,7 @@ async fn reopen_cl( Path(link): Path, state: State, ) -> Result>, ApiError> { - let res = state.cl_stg().get_cl(&link).await?; - let model = res.ok_or(MegaError::Other("Not Found".to_string()))?; - - if model.status == MergeStatusEnum::Closed { - let link = model.link.clone(); - state.cl_stg().reopen_cl(model.clone()).await?; - state - .conv_stg() - .add_conversation( - &link, - &user.username, - Some(format!("{} reopen this", user.username)), - ConvTypeEnum::Reopen, - ) - .await - .unwrap(); - let updated_model = state - .cl_stg() - .get_cl(&link) - .await? - .ok_or(MegaError::Other("Not Found".to_string()))?; - state - .webhook_svc() - .dispatch(WebhookEvent::ClReopened, &updated_model); - } + state.monorepo().reopen_cl(&link, &user.username).await?; Ok(Json(CommonResult::success(None))) } @@ -114,30 +85,7 @@ async fn close_cl( Path(link): Path, state: State, ) -> Result>, ApiError> { - let res = state.cl_stg().get_cl(&link).await?; - let model = res.ok_or(MegaError::Other("Not Found".to_string()))?; - - if matches!(model.status, MergeStatusEnum::Open | MergeStatusEnum::Draft) { - let link = model.link.clone(); - state.cl_stg().close_cl(model.clone()).await?; - state - .conv_stg() - .add_conversation( - &link, - &user.username, - Some(format!("{} closed this", user.username)), - ConvTypeEnum::Closed, - ) - .await?; - let updated_model = state - .cl_stg() - .get_cl(&link) - .await? - .ok_or(MegaError::Other("Not Found".to_string()))?; - state - .webhook_svc() - .dispatch(WebhookEvent::ClClosed, &updated_model); - } + state.monorepo().close_cl(&link, &user.username).await?; Ok(Json(CommonResult::success(None))) } @@ -158,29 +106,10 @@ async fn merge( Path(link): Path, state: State, ) -> Result>, ApiError> { - let res = state.cl_stg().get_cl(&link).await?; - let model = res.ok_or(MegaError::Other("Not Found".to_string()))?; - - if model.status == MergeStatusEnum::Draft { - return Err(ApiError::from(MegaError::Other( - "CL is not ready for review".to_owned(), - ))); - } - - if model.status == MergeStatusEnum::Open { - state - .monorepo() - .merge_cl(&user.username, model.clone()) - .await?; - let updated_model = state - .cl_stg() - .get_cl(&link) - .await? - .ok_or(MegaError::Other("Not Found".to_string()))?; - state - .webhook_svc() - .dispatch(WebhookEvent::ClMerged, &updated_model); - } + state + .monorepo() + .merge_open_cl(&user.username, &link) + .await?; Ok(Json(CommonResult::success(None))) } @@ -201,31 +130,7 @@ async fn merge_no_auth( Path(link): Path, state: State, ) -> Result>, ApiError> { - let res = state.cl_stg().get_cl(&link).await?; - let model = res.ok_or(MegaError::Other("CL Not Found".to_string()))?; - - if model.status != MergeStatusEnum::Open { - return Err(ApiError::from(MegaError::Other(format!( - "CL is not in Open status, current status: {:?}", - model.status - )))); - } - - // No authentication required - using default system user - let default_username = "system"; - state - .monorepo() - .merge_cl(default_username, model.clone()) - .await?; - let updated_model = state - .cl_stg() - .get_cl(&link) - .await? - .ok_or(MegaError::Other("CL Not Found".to_string()))?; - state - .webhook_svc() - .dispatch(WebhookEvent::ClMerged, &updated_model); - + state.monorepo().merge_open_cl_no_auth(&link).await?; Ok(Json(CommonResult::success(Some( "Merge completed successfully".to_string(), )))) @@ -273,11 +178,10 @@ async fn cl_detail( Path(link): Path, state: State, ) -> Result>, ApiError> { - let cl_service: CLService = state.storage.cl_service.clone(); - let cl_details: CLDetailRes = cl_service + let cl_details = state + .monorepo() .get_cl_details(&link, user.username) - .await? - .into(); + .await?; Ok(Json(CommonResult::success(Some(cl_details)))) } @@ -408,13 +312,8 @@ async fn update_branch( ) -> Result>, ApiError> { let new_head = state .monorepo() - .update_branch(&user.username, &link) + .update_branch_with_webhook(&user.username, &link) .await?; - if let Ok(Some(cl_model)) = state.cl_stg().get_cl(&link).await { - state - .webhook_svc() - .dispatch(WebhookEvent::ClUpdated, &cl_model); - } Ok(Json(CommonResult::success(Some(new_head)))) } @@ -434,27 +333,7 @@ async fn merge_box( Path(link): Path, state: State, ) -> Result>, ApiError> { - let cl = state - .cl_stg() - .get_cl(&link) - .await? - .ok_or(MegaError::Other("CL Not Found".to_string()))?; - - let res = match cl.status { - MergeStatusEnum::Open => { - let check_res: Vec = state - .cl_stg() - .get_check_result(&link) - .await? - .into_iter() - .map(|m| m.into()) - .collect(); - MergeBoxRes::from_condition(check_res) - } - MergeStatusEnum::Draft | MergeStatusEnum::Merged | MergeStatusEnum::Closed => MergeBoxRes { - merge_requirements: None, - }, - }; + let res = state.monorepo().get_merge_box(&link).await?; Ok(Json(CommonResult::success(Some(res)))) } @@ -477,47 +356,10 @@ async fn save_comment( state: State, Json(payload): Json, ) -> Result>, ApiError> { - let conv_type = if state - .storage - .reviewer_storage() - .is_reviewer(&link, &user.username) - .await? - { - // If user is the reviewer for this cl, then the comment if of type review - ConvTypeEnum::Review - } else { - ConvTypeEnum::Comment - }; - state - .conv_stg() - .add_conversation( - &link, - &user.username, - Some(payload.content.clone()), - conv_type, - ) + .monorepo() + .save_cl_comment(&link, &user.username, &payload.content) .await?; - // Fire notification trigger - if let Err(e) = triggers::on_cl_comment_created( - &state.notification_stg(), - &state.cl_stg(), - &state.storage.reviewer_storage(), - &user.username, - &link, - &payload.content, - ) - .await - { - tracing::warn!("failed to enqueue cl comment notifications: {e}"); - } - - if let Ok(Some(cl_model)) = state.cl_stg().get_cl(&link).await { - state - .webhook_svc() - .dispatch(WebhookEvent::ClCommentCreated, &cl_model); - } - api_common::comment::check_comment_ref(user, state, &payload.content, &link).await } @@ -540,12 +382,10 @@ async fn edit_title( state: State, Json(payload): Json, ) -> Result>, ApiError> { - state.cl_stg().edit_title(&link, &payload.content).await?; - if let Ok(Some(cl_model)) = state.cl_stg().get_cl(&link).await { - state - .webhook_svc() - .dispatch(WebhookEvent::ClUpdated, &cl_model); - } + state + .monorepo() + .edit_cl_title(&link, &payload.content) + .await?; Ok(Json(CommonResult::success(None))) } @@ -604,74 +444,10 @@ async fn update_cl_status( state: State, Json(payload): Json, ) -> Result>, ApiError> { - let res = state.cl_stg().get_cl(&link).await?; - let model = res.ok_or(MegaError::Other("Not Found".to_string()))?; - - let new_status = match payload.status.to_lowercase().as_str() { - "draft" => MergeStatusEnum::Draft, - "open" => MergeStatusEnum::Open, - _ => { - return Err(ApiError::from(MegaError::Other( - "Invalid status. Only 'draft' and 'open' are supported".to_string(), - ))); - } - }; - - // Only allow Draft ↔ Open transitions - match (&model.status, &new_status) { - (MergeStatusEnum::Draft, MergeStatusEnum::Open) => { - state - .cl_stg() - .update_cl_status(model.clone(), new_status.clone()) - .await?; - state - .conv_stg() - .add_conversation( - &link, - &user.username, - Some(format!("{} marked this as ready for review", user.username)), - ConvTypeEnum::Review, - ) - .await?; - let updated_model = state - .cl_stg() - .get_cl(&link) - .await? - .ok_or(MegaError::Other("Not Found".to_string()))?; - state - .webhook_svc() - .dispatch(WebhookEvent::ClCreated, &updated_model); - } - (MergeStatusEnum::Open, MergeStatusEnum::Draft) => { - state - .cl_stg() - .update_cl_status(model.clone(), new_status.clone()) - .await?; - state - .conv_stg() - .add_conversation( - &link, - &user.username, - Some(format!("{} marked this as draft", user.username)), - ConvTypeEnum::Draft, - ) - .await?; - let updated_model = state - .cl_stg() - .get_cl(&link) - .await? - .ok_or(MegaError::Other("Not Found".to_string()))?; - state - .webhook_svc() - .dispatch(WebhookEvent::ClUpdated, &updated_model); - } - _ => { - return Err(ApiError::from(MegaError::Other( - "Invalid status transition. Only Draft ↔ Open is allowed".to_string(), - ))); - } - } - + state + .monorepo() + .update_cl_status(&link, &user.username, &payload) + .await?; Ok(Json(CommonResult::success(None))) } diff --git a/mono/src/api/router/merge_queue_router.rs b/mono/src/api/router/merge_queue_router.rs index 62d877e9f..0faa00250 100644 --- a/mono/src/api/router/merge_queue_router.rs +++ b/mono/src/api/router/merge_queue_router.rs @@ -4,8 +4,8 @@ use axum::{ extract::{Path, State}, }; use ceres::model::merge_queue::{ - AddToQueueRequest, AddToQueueResponse, QueueItem, QueueListResponse, QueueStatsResponse, - QueueStatus, QueueStatusResponse, + AddToQueueRequest, AddToQueueResponse, QueueListResponse, QueueStatsResponse, + QueueStatusResponse, }; use serde_json::{Value, json}; use utoipa_axum::{router::OpenApiRouter, routes}; @@ -27,7 +27,6 @@ pub fn routers() -> OpenApiRouter { ) } -/// Adds a CL to the merge queue #[utoipa::path( post, path = "/add", @@ -41,41 +40,16 @@ async fn add_to_queue( state: State, Json(request): Json, ) -> Result>, ApiError> { - // Use MonoApiService to add to queue AND start the background processor match state .monorepo() - .add_to_merge_queue(request.cl_link.clone()) + .add_to_merge_queue_response(request.cl_link) .await { - Ok(position) => { - let display_position = state - .storage - .merge_queue_service - .get_display_position_by_position(position) - .await - .map(Some) - .unwrap_or_else(|e| { - tracing::warn!( - "Failed to get display position after add for {}: {}", - request.cl_link, - e - ); - None - }); - - let response = AddToQueueResponse { - success: true, - position, - display_position, - message: "Added to queue".to_string(), - }; - Ok(Json(CommonResult::success(Some(response)))) - } + Ok(response) => Ok(Json(CommonResult::success(Some(response)))), Err(e) => Ok(Json(CommonResult::failed(&e.to_string()))), } } -/// Removes a CL from the merge queue #[utoipa::path( delete, path = "/remove/{cl_link}", @@ -91,28 +65,16 @@ async fn remove_from_queue( state: State, Path(cl_link): Path, ) -> Result>, ApiError> { - match state - .storage - .merge_queue_service - .remove_from_queue(&cl_link) - .await - { - Ok(removed) => { - if removed { - let response = json!({ - "success": true, - "message": "Removed from queue" - }); - Ok(Json(CommonResult::success(Some(response)))) - } else { - Ok(Json(CommonResult::failed("CL not found in queue"))) - } - } + match state.monorepo().remove_from_merge_queue(&cl_link).await { + Ok(true) => Ok(Json(CommonResult::success(Some(json!({ + "success": true, + "message": "Removed from queue" + }))))), + Ok(false) => Ok(Json(CommonResult::failed("CL not found in queue"))), Err(e) => Ok(Json(CommonResult::failed(&e.to_string()))), } } -/// Gets the current merge queue list #[utoipa::path( get, path = "/list", @@ -124,12 +86,10 @@ async fn remove_from_queue( async fn get_queue_list( state: State, ) -> Result>, ApiError> { - let items = state.storage.merge_queue_service.get_queue_list().await?; - let response = QueueListResponse::from(items); + let response = state.monorepo().get_merge_queue_list().await?; Ok(Json(CommonResult::success(Some(response)))) } -/// Gets the status of a specific CL in the queue #[utoipa::path( get, path = "/status/{cl_link}", @@ -145,50 +105,10 @@ async fn get_cl_queue_status( state: State, Path(cl_link): Path, ) -> Result>, ApiError> { - let item_model = state - .storage - .merge_queue_service - .get_cl_queue_status(&cl_link) - .await?; - - let mut item_opt: Option = item_model.map(|m| m.into()); - - if let Some(ref mut item) = item_opt { - match item.status { - QueueStatus::Waiting | QueueStatus::Testing | QueueStatus::Merging => { - let index_result = state - .storage - .merge_queue_service - .get_display_position(&item.cl_link) - .await; - - match index_result { - Ok(index) => { - item.display_position = index; - } - Err(e) => { - tracing::warn!( - "Failed to get display position for {}: {}", - item.cl_link, - e - ); - item.display_position = None; - } - } - } - _ => {} - } - } - - let response = QueueStatusResponse { - in_queue: item_opt.is_some(), - item: item_opt, - }; - + let response = state.monorepo().get_cl_queue_status(&cl_link).await?; Ok(Json(CommonResult::success(Some(response)))) } -/// Retries a failed queue item #[utoipa::path( post, path = "/retry/{cl_link}", @@ -206,27 +126,19 @@ async fn retry_queue_item( state: State, Path(cl_link): Path, ) -> Result>, ApiError> { - // Use MonoApiService to retry AND start the background processor match state.monorepo().retry_merge_queue_item(&cl_link).await { - Ok(success) => { - let response = if success { - json!({ - "success": true, - "message": "Item retried" - }) - } else { - json!({ - "success": false, - "message": "Item not found or cannot be retried" - }) - }; - Ok(Json(CommonResult::success(Some(response)))) - } + Ok(true) => Ok(Json(CommonResult::success(Some(json!({ + "success": true, + "message": "Item retried" + }))))), + Ok(false) => Ok(Json(CommonResult::success(Some(json!({ + "success": false, + "message": "Item not found or cannot be retried" + }))))), Err(e) => Ok(Json(CommonResult::failed(&e.to_string()))), } } -/// Gets queue statistics #[utoipa::path( get, path = "/stats", @@ -238,12 +150,10 @@ async fn retry_queue_item( async fn get_queue_stats( state: State, ) -> Result>, ApiError> { - let stats = state.storage.merge_queue_service.get_queue_stats().await?; - let response = QueueStatsResponse::from(stats); + let response = state.monorepo().get_merge_queue_stats().await?; Ok(Json(CommonResult::success(Some(response)))) } -/// Cancels all pending queue items #[utoipa::path( post, path = "/cancel-all", @@ -255,14 +165,9 @@ async fn get_queue_stats( async fn cancel_all_pending( state: State, ) -> Result>, ApiError> { - state - .storage - .merge_queue_service - .cancel_all_pending() - .await?; - let response = json!({ + state.monorepo().cancel_all_pending_merge_queue().await?; + Ok(Json(CommonResult::success(Some(json!({ "success": true, "message": "All pending items cancelled" - }); - Ok(Json(CommonResult::success(Some(response)))) + }))))) } diff --git a/mono/src/api/router/repo_router.rs b/mono/src/api/router/repo_router.rs index 58a30b3d4..fa4e606de 100644 --- a/mono/src/api/router/repo_router.rs +++ b/mono/src/api/router/repo_router.rs @@ -2,10 +2,12 @@ use std::path::PathBuf; use api_model::common::CommonResult; use axum::{Json, extract::State}; -use ceres::model::change_list::CloneRepoPayload; +use ceres::{api_service::mono::MonoServiceLogic, model::change_list::CloneRepoPayload}; use utoipa_axum::{router::OpenApiRouter, routes}; -use crate::api::{MonoApiServiceState, api_doc::REPO_TAG, error::ApiError}; +use crate::api::{ + MonoApiServiceState, api_doc::REPO_TAG, error::ApiError, oauth::model::LoginUser, +}; pub fn routers() -> OpenApiRouter { OpenApiRouter::new().nest( @@ -27,13 +29,15 @@ pub fn routers() -> OpenApiRouter { tag = REPO_TAG )] async fn clone_third_party_repo( + user: LoginUser, state: State, Json(payload): Json, ) -> Result>, ApiError> { - let path = PathBuf::from(payload.path); + let path = MonoServiceLogic::validate_github_sync_path(&payload.path)?; + let path = PathBuf::from(path); state .monorepo() - .sync_third_party_repo(&payload.owner, &payload.repo, path) + .sync_third_party_repo(&payload.owner, &payload.repo, path, &user.username) .await?; Ok(Json(CommonResult::success(None))) diff --git a/mono/src/git_protocol/http.rs b/mono/src/git_protocol/http.rs index ccaf825d3..40a711f04 100644 --- a/mono/src/git_protocol/http.rs +++ b/mono/src/git_protocol/http.rs @@ -9,6 +9,7 @@ use base64::Engine; use bytes::{Bytes, BytesMut}; use ceres::{ api_service::state::ProtocolApiState, + pack::into_pack_byte_stream, protocol::{PushUserInfo, ServiceType, SmartSession, TransportProtocol, smart}, }; use common::errors::ProtocolError; @@ -221,7 +222,7 @@ pub async fn git_receive_pack( let left_chunk_bytes = Bytes::copy_from_slice(&chunk[pos..]); let pack_stream = stream::once(async { Ok(left_chunk_bytes) }).chain(data_stream); report_status = pack_protocol - .git_receive_pack_stream(state, commands, Box::pin(pack_stream)) + .git_receive_pack_stream(state, commands, into_pack_byte_stream(pack_stream)) .await?; break; } else { diff --git a/mono/src/git_protocol/ssh.rs b/mono/src/git_protocol/ssh.rs index 4fd64ef45..70df46117 100644 --- a/mono/src/git_protocol/ssh.rs +++ b/mono/src/git_protocol/ssh.rs @@ -4,6 +4,7 @@ use bytes::{Bytes, BytesMut}; use ceres::{ api_service::state::ProtocolApiState, lfs::lfs_structs::Link, + pack::into_pack_byte_stream, protocol::{ ServiceType, SmartSession, TransportProtocol, smart::{self}, @@ -229,7 +230,9 @@ impl SshServer { async fn handle_receive_pack(&mut self, channel: ChannelId, session: &mut Session) { let smart_protocol = self.smart_protocol.as_mut().unwrap(); let data = self.data_combined.split().freeze(); - let mut data_stream = Box::pin(stream::once(async move { Ok(data) })); + let mut data_stream = Box::pin(stream::once(async move { + Ok::(data) + })); let mut report_status = Bytes::new(); while let Some(chunk) = data_stream.next().await { @@ -242,7 +245,11 @@ impl SshServer { let remaining_stream = stream::once(async { Ok(remaining_bytes) }).chain(data_stream); report_status = smart_protocol - .git_receive_pack_stream(&self.state, commands, Box::pin(remaining_stream)) + .git_receive_pack_stream( + &self.state, + commands, + into_pack_byte_stream(remaining_stream), + ) .await .unwrap(); break; diff --git a/mono/src/server/http_server.rs b/mono/src/server/http_server.rs index b610752f9..c1d57bd38 100644 --- a/mono/src/server/http_server.rs +++ b/mono/src/server/http_server.rs @@ -10,11 +10,13 @@ use axum::{ response::Response, routing::any, }; -use ceres::api_service::{cache::GitObjectCache, state::ProtocolApiState}; +use ceres::{ + api_service::{cache::GitObjectCache, state::ProtocolApiState}, + application::artifact::ArtifactApplicationService, +}; use common::errors::ProtocolError; use context::AppContext; use http::{HeaderName, HeaderValue, Method}; -use jupiter::service::artifact_service::ArtifactService; use orion_client::OrionBuildClient; use saturn::entitystore::EntityStore; use time::Duration; @@ -150,7 +152,7 @@ fn spawn_artifact_gc_task(ctx: AppContext, token: CancellationToken) -> Option Option { match service - .gc_unreferenced_artifact_objects_once(grace, batch_limit) + .gc_unreferenced_once(grace, batch_limit) .await { Ok(s) if s.deleted > 0 || s.candidates > 0 => { diff --git a/mono/src/server/ssh_server.rs b/mono/src/server/ssh_server.rs index 922eb45b9..bf4fce1b3 100644 --- a/mono/src/server/ssh_server.rs +++ b/mono/src/server/ssh_server.rs @@ -52,13 +52,13 @@ pub async fn start_server(ctx: AppContext, command: &SshOptions) { custom: SshCustom { ssh_port }, } = command; - let state = ProtocolApiState { - storage: ctx.storage.clone(), - git_object_cache: Arc::new(GitObjectCache { + let state = ProtocolApiState::new( + ctx.storage.clone(), + Arc::new(GitObjectCache { connection: ctx.connection.clone(), prefix: "git-object-rkyv:v1".to_string(), }), - }; + ); let mut ssh_server = SshServer { clients: Arc::new(Mutex::new(HashMap::new())), state, diff --git a/moon/apps/web/components/ClView/components/Checks/cpns/Task.tsx b/moon/apps/web/components/ClView/components/Checks/cpns/Task.tsx index 6f6a680fb..8142628b3 100644 --- a/moon/apps/web/components/ClView/components/Checks/cpns/Task.tsx +++ b/moon/apps/web/components/ClView/components/Checks/cpns/Task.tsx @@ -288,15 +288,8 @@ const TaskItem = memo(function TaskItem({ const showQueued = Boolean(isQueued) && build.status === 'Building' - const handleClick = (e: React.MouseEvent) => { - const scrollParent = e.currentTarget.closest('[data-build-list-scroll]') as HTMLElement | null - const scrollTop = scrollParent?.scrollTop ?? 0 - + const handleClick = () => { onSelectBuild(build.id) - - requestAnimationFrame(() => { - if (scrollParent) scrollParent.scrollTop = scrollTop - }) } const handleRetry = (e: React.MouseEvent) => { diff --git a/moon/apps/web/components/ClView/components/Checks/hooks/useLeftPanelScroll.ts b/moon/apps/web/components/ClView/components/Checks/hooks/useLeftPanelScroll.ts index 5c1b27f6a..3d4177540 100644 --- a/moon/apps/web/components/ClView/components/Checks/hooks/useLeftPanelScroll.ts +++ b/moon/apps/web/components/ClView/components/Checks/hooks/useLeftPanelScroll.ts @@ -1,22 +1,35 @@ -import { RefObject, useEffect, useRef } from 'react' +import { RefObject, useCallback, useEffect, useLayoutEffect, useRef } from 'react' export function useLeftPanelScroll(cl: string, buildId: string, leftPanelRef: RefObject) { const leftPanelScrollRef = useRef(0) + const pendingRestoreRef = useRef(null) + const isRestoringRef = useRef(false) const prevClRef = useRef(cl) useEffect(() => { if (prevClRef.current !== cl) { prevClRef.current = cl leftPanelScrollRef.current = 0 + pendingRestoreRef.current = null } }, [cl]) + const preserveScroll = useCallback(() => { + const panel = leftPanelRef.current + + if (panel) { + pendingRestoreRef.current = panel.scrollTop + } + }, [leftPanelRef]) + useEffect(() => { const panel = leftPanelRef.current if (!panel) return const onScroll = () => { + if (isRestoringRef.current) return + leftPanelScrollRef.current = panel.scrollTop } @@ -25,17 +38,25 @@ export function useLeftPanelScroll(cl: string, buildId: string, leftPanelRef: Re return () => panel.removeEventListener('scroll', onScroll) }, [leftPanelRef]) - useEffect(() => { + // Restore before paint so a re-render cannot reset scroll to 0 and clobber the + // saved position via the scroll listener. + useLayoutEffect(() => { const panel = leftPanelRef.current if (!panel) return - const saved = leftPanelScrollRef.current + const saved = pendingRestoreRef.current ?? leftPanelScrollRef.current + + pendingRestoreRef.current = null + + isRestoringRef.current = true + panel.scrollTop = saved + leftPanelScrollRef.current = saved requestAnimationFrame(() => { - panel.scrollTop = saved + isRestoringRef.current = false }) }, [buildId, leftPanelRef]) - return {} + return { preserveScroll } } diff --git a/moon/apps/web/components/ClView/components/Checks/index.tsx b/moon/apps/web/components/ClView/components/Checks/index.tsx index 7d9520d71..024bac72a 100644 --- a/moon/apps/web/components/ClView/components/Checks/index.tsx +++ b/moon/apps/web/components/ClView/components/Checks/index.tsx @@ -40,7 +40,12 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri defaultLeftWidthPercent } = useResizablePanels() - useLeftPanelScroll(cl, buildId, leftPanelRef) + const { preserveScroll } = useLeftPanelScroll(cl, buildId, leftPanelRef) + + const handleSelectBuild = (nextBuildId: string, taskId?: string) => { + preserveScroll() + selectBuild(nextBuildId, taskId) + } const [isDropdownOpen, setIsDropdownOpen] = useState(false) @@ -278,6 +283,7 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri key={task.task_id} onClick={(e) => { e.stopPropagation() + preserveScroll() selectTask(task.task_id) setIsDropdownOpen(false) }} @@ -326,7 +332,7 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri tasks={tasksToDisplay} logsAvailableIds={logsAvailableIds} selectedBuildId={buildId} - onSelectBuild={selectBuild} + onSelectBuild={handleSelectBuild} totalTasksCount={validTasks.length} cl={cl} /> diff --git a/moon/apps/web/components/CodeView/TreeView/SyncRepoButton.tsx b/moon/apps/web/components/CodeView/TreeView/SyncRepoButton.tsx index da792e5ed..86bf7bd64 100644 --- a/moon/apps/web/components/CodeView/TreeView/SyncRepoButton.tsx +++ b/moon/apps/web/components/CodeView/TreeView/SyncRepoButton.tsx @@ -5,6 +5,8 @@ import { Dialog } from '@gitmono/ui/Dialog' import { usePostRepoClone } from '@/hooks/usePostRepoClone' +import { formatGitHubRepoUrl, isProjectSyncPath, parseGitHubRepoUrl } from './syncUtils' + interface Props { currentPath?: string } @@ -18,24 +20,29 @@ const isFieldValid = (value: string): boolean => value.trim().length > 0 export default function SyncRepoButton({ currentPath }: Props) { const [open, setOpen] = useState(false) - const [owner, setOwner] = useState('') - const [repo, setRepo] = useState('') + const [githubUrl, setGithubUrl] = useState('') const [repoName, setRepoName] = useState('') const { mutateAsync, isPending } = usePostRepoClone() const basePath = useMemo(() => formatBasePath(currentPath), [currentPath]) + const isProjectSync = useMemo(() => isProjectSyncPath(currentPath), [currentPath]) + const parsedRepo = useMemo(() => parseGitHubRepoUrl(githubUrl), [githubUrl]) const fullPath = useMemo(() => (repoName ? `${basePath}${repoName}` : basePath), [basePath, repoName]) + const urlError = useMemo(() => { + if (!githubUrl.trim() || parsedRepo) return null + return 'Enter a valid GitHub URL (e.g. https://github.com/owner/repo)' + }, [githubUrl, parsedRepo]) + const canSubmit = useMemo( - () => isFieldValid(owner) && isFieldValid(repo) && isFieldValid(repoName) && !isPending, - [owner, repo, repoName, isPending] + () => Boolean(parsedRepo) && isFieldValid(repoName) && !isPending, + [parsedRepo, repoName, isPending] ) const resetForm = useCallback(() => { - setOwner('') - setRepo('') + setGithubUrl('') setRepoName('') }, []) @@ -50,18 +57,22 @@ export default function SyncRepoButton({ currentPath }: Props) { ) const handleSync = useCallback(async () => { + if (!parsedRepo) return + await mutateAsync({ - owner: owner.trim(), - repo: repo.trim(), + owner: parsedRepo.owner, + repo: parsedRepo.repo, path: fullPath.trim() }) handleOpenChange(false) - }, [owner, repo, fullPath, mutateAsync, handleOpenChange]) + }, [parsedRepo, fullPath, mutateAsync, handleOpenChange]) useEffect(() => { - setRepoName(repo) - }, [repo]) + if (parsedRepo?.repo) { + setRepoName(parsedRepo.repo) + } + }, [parsedRepo?.repo]) return ( <> @@ -78,30 +89,27 @@ export default function SyncRepoButton({ currentPath }: Props) {

+ {isProjectSync && ( + + Syncing under /project creates a Change List and may trigger build checks. + + )}
- setOwner(e.target.value)} - placeholder='e.g., facebook' - disabled={isPending} - /> -
- -
- setRepo(e.target.value)} - placeholder='e.g., react' + value={githubUrl} + onChange={(e) => setGithubUrl(e.target.value)} + placeholder='https://github.com/owner/repo' disabled={isPending} /> + {urlError && ( + + {urlError} + + )}
@@ -120,13 +128,13 @@ export default function SyncRepoButton({ currentPath }: Props) {
- {owner && repo && ( + {parsedRepo && (
Will sync from: - https://github.com/{owner}/{repo} + {formatGitHubRepoUrl(parsedRepo.owner, parsedRepo.repo)} To: {fullPath} diff --git a/moon/apps/web/components/CodeView/TreeView/__tests__/syncUtils.test.ts b/moon/apps/web/components/CodeView/TreeView/__tests__/syncUtils.test.ts new file mode 100644 index 000000000..a24794e36 --- /dev/null +++ b/moon/apps/web/components/CodeView/TreeView/__tests__/syncUtils.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' + +import { parseGitHubRepoUrl } from '../syncUtils' + +describe('parseGitHubRepoUrl', () => { + it('parses https github urls', () => { + expect(parseGitHubRepoUrl('https://github.com/facebook/react')).toEqual({ + owner: 'facebook', + repo: 'react' + }) + }) + + it('parses urls with .git suffix', () => { + expect(parseGitHubRepoUrl('https://github.com/facebook/react.git')).toEqual({ + owner: 'facebook', + repo: 'react' + }) + }) + + it('parses host-only urls without protocol', () => { + expect(parseGitHubRepoUrl('github.com/facebook/react')).toEqual({ + owner: 'facebook', + repo: 'react' + }) + }) + + it('parses owner/repo shorthand', () => { + expect(parseGitHubRepoUrl('facebook/react')).toEqual({ + owner: 'facebook', + repo: 'react' + }) + }) + + it('rejects invalid hosts and malformed input', () => { + expect(parseGitHubRepoUrl('https://gitlab.com/foo/bar')).toBeNull() + expect(parseGitHubRepoUrl('https://github.com/facebook')).toBeNull() + expect(parseGitHubRepoUrl('not a url')).toBeNull() + }) +}) diff --git a/moon/apps/web/components/CodeView/TreeView/syncUtils.ts b/moon/apps/web/components/CodeView/TreeView/syncUtils.ts new file mode 100644 index 000000000..bf74d936b --- /dev/null +++ b/moon/apps/web/components/CodeView/TreeView/syncUtils.ts @@ -0,0 +1,62 @@ +const GITHUB_SYNC_ROOTS = ['third-party', 'project'] as const + +export function canShowGitHubSync(path: string[], version: string): boolean { + return version === 'main' && GITHUB_SYNC_ROOTS.includes(path[0] as (typeof GITHUB_SYNC_ROOTS)[number]) +} + +export function isProjectSyncPath(currentPath?: string): boolean { + if (!currentPath) return false + return currentPath === '/project' || currentPath.startsWith('/project/') +} + +export interface ParsedGitHubRepo { + owner: string + repo: string +} + +/** Parse github.com URLs or `owner/repo` shorthand into owner and repo name. */ +export function parseGitHubRepoUrl(input: string): ParsedGitHubRepo | null { + const trimmed = input.trim() + + if (!trimmed) return null + + const shorthand = trimmed.match(/^([a-zA-Z0-9_.-]+)\/([a-zA-Z0-9_.-]+)$/) + + if (shorthand) { + return { + owner: shorthand[1], + repo: shorthand[2].replace(/\.git$/i, '') + } + } + + let url: URL + + try { + const withProtocol = trimmed.includes('://') ? trimmed : `https://${trimmed}` + + url = new URL(withProtocol) + } catch { + return null + } + + const host = url.hostname.toLowerCase() + + if (host !== 'github.com' && host !== 'www.github.com') { + return null + } + + const segments = url.pathname.split('/').filter(Boolean) + + if (segments.length < 2) return null + + const owner = segments[0] + const repo = segments[1].replace(/\.git$/i, '') + + if (!owner || !repo) return null + + return { owner, repo } +} + +export function formatGitHubRepoUrl(owner: string, repo: string): string { + return `https://github.com/${owner}/${repo}` +} diff --git a/moon/apps/web/pages/[org]/code/tree/[version]/[...path]/index.tsx b/moon/apps/web/pages/[org]/code/tree/[version]/[...path]/index.tsx index 118b952fb..30ab13d76 100644 --- a/moon/apps/web/pages/[org]/code/tree/[version]/[...path]/index.tsx +++ b/moon/apps/web/pages/[org]/code/tree/[version]/[...path]/index.tsx @@ -16,6 +16,7 @@ import BreadCrumb from '@/components/CodeView/TreeView/BreadCrumb' import CloneTabs from '@/components/CodeView/TreeView/CloneTabs' import RepoTree from '@/components/CodeView/TreeView/RepoTree' import SyncRepoButton from '@/components/CodeView/TreeView/SyncRepoButton' +import { canShowGitHubSync } from '@/components/CodeView/TreeView/syncUtils' import { AppLayout } from '@/components/Layout/AppLayout' import AuthAppProviders from '@/components/Providers/AuthAppProviders' import { useGetBlob } from '@/hooks/useGetBlob' @@ -144,7 +145,7 @@ function TreeDetailPage() { )} {canClone?.data && } - {path[0] === 'third-party' && version == 'main' && } + {canShowGitHubSync(path, version) && }
) : (