diff --git a/.github/workflows/web-deploy.yml b/.github/workflows/web-deploy.yml index 500621396..df94e78f4 100644 --- a/.github/workflows/web-deploy.yml +++ b/.github/workflows/web-deploy.yml @@ -61,24 +61,10 @@ jobs: with: registry-type: public - - name: Configure AWS Credentials (ECR Private, ap-southeast-2) - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ap-southeast-2 - - - name: Login to Amazon ECR Private - id: login-ecr-private - uses: aws-actions/amazon-ecr-login@v2 - with: - registry-type: private - - name: Build & push unified image working-directory: moon env: ECR_PUBLIC_REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} - ECR_PRIVATE_REGISTRY: ${{ steps.login-ecr-private.outputs.registry }} PLATFORM: linux/amd64 run: | set -euo pipefail @@ -88,9 +74,7 @@ jobs: SHA_TAG="${SHORT_SHA}-${ARCH_SUFFIX}" PUBLIC_IMAGE="$ECR_PUBLIC_REGISTRY/${{ env.REGISTRY_ALIAS }}/${{ env.REPOSITORY }}:$IMAGE_TAG" - PRIVATE_IMAGE="$ECR_PRIVATE_REGISTRY/${{ env.REPOSITORY }}:$IMAGE_TAG" PUBLIC_IMAGE_SHA="$ECR_PUBLIC_REGISTRY/${{ env.REGISTRY_ALIAS }}/${{ env.REPOSITORY }}:$SHA_TAG" - PRIVATE_IMAGE_SHA="$ECR_PRIVATE_REGISTRY/${{ env.REPOSITORY }}:$SHA_TAG" docker buildx build \ --platform "$PLATFORM" \ @@ -100,9 +84,7 @@ jobs: --sbom=false \ -f apps/web/Dockerfile \ -t "$PUBLIC_IMAGE" \ - -t "$PRIVATE_IMAGE" \ - -t "$PUBLIC_IMAGE_SHA" \ - -t "$PRIVATE_IMAGE_SHA" . \ + -t "$PUBLIC_IMAGE_SHA" . \ --push manifest: diff --git a/Cargo.lock b/Cargo.lock index fdf74ae1d..97bbd2159 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,9 +21,9 @@ dependencies = [ [[package]] name = "aead" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef60ac202874e574ce7a7158cc8bca7313dd344322482e4fadee288bf4a306b8" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common 0.2.2", "inout 0.2.2", @@ -72,7 +72,7 @@ version = "0.11.0-rc.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da8c919c118108f144adecad74b425b804ad075580d605d9b33c2d6d1c62a2f8" dependencies = [ - "aead 0.6.0", + "aead 0.6.1", "aes 0.9.1", "cipher 0.5.2", "ctr 0.10.1", @@ -143,9 +143,9 @@ checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" [[package]] name = "alloc-stdlib" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" dependencies = [ "alloc-no-stdlib", ] @@ -201,7 +201,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -212,7 +212,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -321,9 +321,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" [[package]] name = "as-any" @@ -364,7 +364,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -376,7 +376,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -421,7 +421,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -443,7 +443,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -454,7 +454,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -605,7 +605,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -684,7 +684,7 @@ checksum = "7b9a5040dce49a7642c97ccb1ae59567098967b5d52c29773f1299a42d23bb39" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -745,7 +745,7 @@ dependencies = [ "regex", "rustc-hash", "shlex 1.3.0", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -765,15 +765,15 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitcoin-io" -version = "0.1.100" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11301df0b06f22dea7bb1916403fdd88a371031e495c49b8f96931b28189e175" +checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953" [[package]] name = "bitcoin_hashes" -version = "0.14.100" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9901a56e133a1fc86eeb1113e2591f45f4682451ca893bff494d2f88918e3f" +checksum = "4ed83caece3afc59919481b33b472e1432d1abc4641ed9100be142ef5110b406" dependencies = [ "bitcoin-io", "hex-conservative", @@ -796,7 +796,7 @@ checksum = "f2c044f98f86f15414668d6c8187c7e4fadab1ad2b31680f648703e0fe07c555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "thiserror 2.0.18", ] @@ -817,9 +817,9 @@ dependencies = [ [[package]] name = "bitvec" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" dependencies = [ "funty", "radium", @@ -852,7 +852,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b79834656f71332577234b50bfc009996f7449e0c056884e6a02492ded0ca2f3" dependencies = [ "arrayref", - "arrayvec 0.7.6", + "arrayvec 0.7.7", "constant_time_eq", ] @@ -863,7 +863,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ "arrayref", - "arrayvec 0.7.6", + "arrayvec 0.7.7", "cc", "cfg-if", "constant_time_eq", @@ -881,9 +881,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ "hybrid-array", "zeroize", @@ -938,9 +938,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" dependencies = [ "borsh-derive", "bytes", @@ -949,22 +949,22 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" +checksum = "3ae8fb4fb5740e4b2c4884ff95f5f32f5e8479db1e8fd8eb49ddbe09eb09bb7c" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] name = "brotli" -version = "8.0.3" +version = "8.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -973,9 +973,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.1" +version = "5.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1080,7 +1080,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1162,9 +1162,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.63" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", "jobserver", @@ -1174,9 +1174,9 @@ dependencies = [ [[package]] name = "cedar-policy" -version = "4.11.1" +version = "4.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "716a5103f447735b4cef15df847dc2a7ed8752e8a95febab5a6bfe1812054e43" +checksum = "1fdd98501120bc31fc09f92d3f58c489f4ee4b846904109ac63dd8256de7b84d" dependencies = [ "cedar-policy-core", "cedar-policy-formatter", @@ -1194,9 +1194,9 @@ dependencies = [ [[package]] name = "cedar-policy-core" -version = "4.11.1" +version = "4.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7935861e764efcba26f73ac4d9d284e30b2b4a9a95d3c961e908e3ed7889e391" +checksum = "d1d41088b497e39b6789e87e05ad6549b7f64a663a4d6ecd73a748dc3a9a98a9" dependencies = [ "chrono", "educe", @@ -1222,9 +1222,9 @@ dependencies = [ [[package]] name = "cedar-policy-formatter" -version = "4.11.1" +version = "4.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca436a4803977e69b12a8e0e7f5d7c648e578b59fa0f2ed3bbe3c0054b9e036" +checksum = "fb6c28be47d77dcbc04c2a80181a2c0db9725ed60d24832c6401396cc2ae114b" dependencies = [ "cedar-policy-core", "itertools 0.14.0", @@ -1249,7 +1249,7 @@ dependencies = [ "chrono", "common", "futures", - "git-internal 0.7.5", + "git-internal 0.7.6", "hex", "io-orbit", "jupiter", @@ -1263,7 +1263,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.11.0", - "sysinfo 0.39.3", + "sysinfo 0.39.5", "tempfile", "tokio", "tokio-stream", @@ -1357,7 +1357,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ - "block-buffer 0.12.0", + "block-buffer 0.12.1", "crypto-common 0.2.2", "inout 0.2.2", "zeroize", @@ -1405,7 +1405,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1452,7 +1452,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -1481,7 +1481,7 @@ dependencies = [ "config", "directories", "envsubst", - "git-internal 0.7.5", + "git-internal 0.7.6", "idgenerator", "pgp", "redis", @@ -1714,6 +1714,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.0", +] + [[package]] name = "crc24" version = "0.1.6" @@ -1805,13 +1815,13 @@ dependencies = [ [[package]] name = "crypto-bigint" -version = "0.7.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42a0d26b245348befa0c121944541476763dcc46ede886c88f9d12e1697d27c3" +checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" dependencies = [ "cpubits", "ctutils", - "getrandom 0.4.2", + "getrandom 0.4.3", "hybrid-array", "num-traits", "rand_core 0.10.1", @@ -1837,7 +1847,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "hybrid-array", "rand_core 0.10.1", ] @@ -1848,7 +1858,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f" dependencies = [ - "crypto-bigint 0.7.3", + "crypto-bigint 0.7.5", "rand_core 0.10.1", ] @@ -1931,7 +1941,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1991,7 +2001,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2004,7 +2014,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2015,7 +2025,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2026,7 +2036,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2072,6 +2082,38 @@ dependencies = [ "generic-array 0.14.7", ] +[[package]] +name = "defmt" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b" +dependencies = [ + "defmt-parser", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "delegate" version = "0.13.5" @@ -2080,7 +2122,7 @@ checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2147,7 +2189,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2168,7 +2210,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2178,7 +2220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2191,7 +2233,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2213,7 +2255,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.117", + "syn 2.0.118", "unicode-xid", ] @@ -2259,7 +2301,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer 0.12.0", + "block-buffer 0.12.1", "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", @@ -2294,7 +2336,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2317,7 +2359,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2465,7 +2507,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2509,7 +2551,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "102d3643d30dd8b559613c5cced68317199597fffb278cdc88daa2ef7fafc935" dependencies = [ "base16ct 1.0.0", - "crypto-bigint 0.7.3", + "crypto-bigint 0.7.5", "crypto-common 0.2.2", "digest 0.11.3", "ff 0.14.0", @@ -2594,7 +2636,7 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2614,7 +2656,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2626,7 +2668,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2685,7 +2727,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2809,7 +2851,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", - "spin", + "spin 0.9.8", ] [[package]] @@ -2952,7 +2994,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3044,17 +3086,15 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", - "wasip2", - "wasip3", "wasm-bindgen", ] @@ -3125,9 +3165,9 @@ dependencies = [ [[package]] name = "git-internal" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a86cdc63286d078791391ab9b2aba6c2c6401b39941b9cd67d3d17972f3f595" +checksum = "514e1c3227f3ade4a4ea2fadcf80936b034cf75ec52e9deabea34594804622c9" dependencies = [ "ahash 0.8.12", "async-trait", @@ -3157,8 +3197,8 @@ dependencies = [ "sea-orm", "serde", "serde_json", - "sha1 0.11.0", - "sha2 0.11.0", + "sha1 0.10.6", + "sha2 0.10.9", "similar", "tempfile", "thiserror 2.0.18", @@ -3232,9 +3272,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", @@ -3335,7 +3375,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3444,7 +3484,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f" dependencies = [ - "arrayvec 0.7.6", + "arrayvec 0.7.7", ] [[package]] @@ -3599,11 +3639,10 @@ dependencies = [ "hyper", "hyper-util", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -3652,7 +3691,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "system-configuration", "tokio", "tower-service", @@ -3766,12 +3805,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "idea" version = "0.5.1" @@ -3865,7 +3898,7 @@ checksum = "c727f80bfa4a6c6e2508d2f05b6f4bfce242030bd88ed15ae5331c5b5d30fba7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4040,10 +4073,11 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.28" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" +checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46" dependencies = [ + "defmt", "jiff-static", "log", "portable-atomic", @@ -4053,13 +4087,13 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.28" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" +checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4089,7 +4123,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4108,7 +4142,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4123,9 +4157,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.100" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" dependencies = [ "cfg-if", "futures-util", @@ -4155,7 +4189,7 @@ dependencies = [ "chrono", "common", "futures", - "git-internal 0.7.5", + "git-internal 0.7.6", "hex", "hmac 0.13.0", "idgenerator", @@ -4238,9 +4272,9 @@ dependencies = [ [[package]] name = "kvm-bindings" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b3c06ff73c7ce03e780887ec2389d62d2a2a9ddf471ab05c2ff69207cd3f3b4" +checksum = "11cf0ca75d59e9d298647c59cf6c5286fa048120caa77972a7a504a0824d234f" dependencies = [ "vmm-sys-util 0.15.0", ] @@ -4295,15 +4329,9 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.8", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "lettre" version = "0.11.22" @@ -4629,9 +4657,9 @@ checksum = "5be1cf190319c74ba3e45923624626ae2e43fe42ad7e60ff38ded81044c37630" [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "logos" @@ -4653,7 +4681,7 @@ dependencies = [ "quote", "regex-automata", "regex-syntax 0.8.11", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4725,6 +4753,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "md5" version = "0.7.0" @@ -4739,15 +4777,15 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", ] @@ -4781,7 +4819,7 @@ checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4903,7 +4941,7 @@ dependencies = [ "ctrlc", "ed25519-dalek 2.2.0", "futures", - "git-internal 0.7.5", + "git-internal 0.7.6", "http", "jemallocator", "jupiter", @@ -4962,7 +5000,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5081,7 +5119,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5177,7 +5215,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5254,14 +5292,16 @@ dependencies = [ [[package]] name = "object_store" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +checksum = "765784b4390c6bcf80316e5a22f4e3661b639c9d8c83246856643c27d8ce9dbe" dependencies = [ "async-trait", + "aws-lc-rs", "base64", "bytes", "chrono", + "crc-fast", "form_urlencoded", "futures-channel", "futures-core", @@ -5270,14 +5310,14 @@ dependencies = [ "http-body-util", "humantime", "hyper", - "itertools 0.14.0", - "md-5", + "itertools 0.15.0", + "md-5 0.11.0", + "nix 0.31.3", "parking_lot 0.12.5", "percent-encoding", "quick-xml", "rand 0.10.1", - "reqwest 0.12.28", - "ring", + "reqwest 0.13.4", "rustls-pki-types", "serde", "serde_json", @@ -5289,6 +5329,7 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", ] [[package]] @@ -5352,7 +5393,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5400,7 +5441,7 @@ dependencies = [ [[package]] name = "orion" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anyhow", "api-model", @@ -5419,7 +5460,6 @@ dependencies = [ "td_util", "td_util_buck", "tempfile", - "thiserror 2.0.18", "tokio", "tokio-tungstenite", "tokio-util", @@ -5427,7 +5467,6 @@ dependencies = [ "tracing-subscriber", "tungstenite", "url", - "utoipa", "uuid", ] @@ -5525,7 +5564,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5715,7 +5754,7 @@ dependencies = [ "regex", "regex-syntax 0.7.5", "structmeta 0.2.0", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5729,7 +5768,7 @@ dependencies = [ "regex", "regex-syntax 0.8.11", "structmeta 0.3.0", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5872,7 +5911,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5938,7 +5977,7 @@ dependencies = [ "idea", "k256", "log", - "md-5", + "md-5 0.10.6", "nom 8.0.0", "num-bigint-dig", "num-traits", @@ -6015,7 +6054,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6220,16 +6259,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] name = "primefield" -version = "0.14.0-rc.11" +version = "0.14.0-rc.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d7e42f46a29abc16fb621a3466ee453358ebaae48a9e515f287e0af052ed8f" +checksum = "2db02b39ea98560a1fec81df6266f3c1ef7fdde06ac5ef17f69aee6101602630" dependencies = [ - "crypto-bigint 0.7.3", + "crypto-bigint 0.7.5", "crypto-common 0.2.2", "ff 0.14.0", "rand_core 0.10.1", @@ -6294,7 +6333,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6314,7 +6353,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "version_check", "yansi", ] @@ -6336,7 +6375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -6345,7 +6384,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.117", + "syn 2.0.118", "tempfile", ] @@ -6356,10 +6395,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6418,7 +6457,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6516,9 +6555,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.4" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" dependencies = [ "memchr", "serde", @@ -6526,9 +6565,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -6537,7 +6576,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -6546,9 +6585,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", @@ -6575,16 +6614,16 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -6670,7 +6709,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20 0.10.0", - "getrandom 0.4.2", + "getrandom 0.4.3", "rand_core 0.10.1", ] @@ -6837,7 +6876,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6927,7 +6966,6 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", - "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", @@ -6936,16 +6974,14 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-rustls", - "tokio-util", "tower", "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams 0.4.2", "web-sys", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -6988,7 +7024,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams 0.5.0", + "wasm-streams", "web-sys", ] @@ -7115,14 +7151,14 @@ checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] name = "ron" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4147b952f3f819eca0e99527022f7d6a8d05f111aeb0a62960c74eb283bec8fc" +checksum = "81116b9531d61eabc41aeb228e4b6b2435bcca3233b98cf3b3077d4e6e9debb3" dependencies = [ "bitflags 2.13.0", "once_cell", @@ -7160,7 +7196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b2aa4ba0d89f73d1e332df05be0eeab8840351c36ca5654341dfdb57bb3caf" dependencies = [ "const-oid 0.10.2", - "crypto-bigint 0.7.3", + "crypto-bigint 0.7.5", "crypto-primes", "digest 0.11.3", "pkcs1 0.8.0-rc.4", @@ -7248,7 +7284,7 @@ dependencies = [ "bytes", "cbc 0.2.1", "cipher 0.5.2", - "crypto-bigint 0.7.3", + "crypto-bigint 0.7.5", "ctr 0.10.1", "curve25519-dalek 5.0.0-rc.0", "data-encoding", @@ -7262,7 +7298,7 @@ dependencies = [ "flate2", "futures", "generic-array 1.4.3", - "getrandom 0.4.2", + "getrandom 0.4.3", "ghash 0.6.0", "hex-literal 1.1.0", "hmac 0.13.0", @@ -7383,7 +7419,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.117", + "syn 2.0.118", "walkdir", ] @@ -7409,11 +7445,11 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.42.0" +version = "1.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c5108e3d4d903e21aac27f12ba5377b6b34f9f44b325e4894c7924169d06995" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" dependencies = [ - "arrayvec 0.7.6", + "arrayvec 0.7.7", "borsh", "bytes", "num-traits", @@ -7477,14 +7513,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "aws-lc-rs", "log", @@ -7545,7 +7581,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7663,7 +7699,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "scorpiofs" version = "0.2.2" -source = "git+https://github.com/web3infra-foundation/scorpiofs.git?rev=900309f0abc397ccf8b9565ae0da2e3ef1e65618#900309f0abc397ccf8b9565ae0da2e3ef1e65618" +source = "git+https://github.com/web3infra-foundation/scorpiofs.git#230f62a28598100b2bb94379de20bdec9d4566c5" dependencies = [ "async-recursion", "async-trait", @@ -7730,7 +7766,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7790,7 +7826,7 @@ dependencies = [ "proc-macro2", "quote", "sea-bae", - "syn 2.0.117", + "syn 2.0.118", "unicode-ident", ] @@ -7853,7 +7889,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "thiserror 2.0.18", ] @@ -7877,7 +7913,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8014,7 +8050,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8101,7 +8137,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8184,7 +8220,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8418,7 +8454,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8438,7 +8474,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8450,6 +8486,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + [[package]] name = "spki" version = "0.7.3" @@ -8534,7 +8576,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8557,7 +8599,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.117", + "syn 2.0.118", "tokio", "url", ] @@ -8589,7 +8631,7 @@ dependencies = [ "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", @@ -8633,7 +8675,7 @@ dependencies = [ "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "num-bigint", "once_cell", @@ -8702,7 +8744,7 @@ version = "0.3.0-rc.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10db6f219196a8528f9ec904d9d45cdad692d65b0e57e72be4dedd1c5fddce36" dependencies = [ - "aead 0.6.0", + "aead 0.6.1", "aes 0.9.1", "aes-gcm 0.11.0-rc.4", "cbc 0.2.1", @@ -8736,7 +8778,7 @@ checksum = "7abf34aa716da5d5b4c496936d042ea282ab392092cd68a72ef6a8863ff8c96a" dependencies = [ "base64ct", "bytes", - "crypto-bigint 0.7.3", + "crypto-bigint 0.7.5", "ctutils", "digest 0.11.3", "pem-rfc7468 1.0.0", @@ -8785,7 +8827,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8850,7 +8892,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive 0.2.0", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8862,7 +8904,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive 0.3.0", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8873,7 +8915,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8884,7 +8926,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8912,7 +8954,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8940,9 +8982,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -8966,7 +9008,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8985,9 +9027,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.39.3" +version = "0.39.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21d0d938c10fcda3e897e28aaddf4ab462375d411f4378cd63b1c945f69aba96" +checksum = "2c8bd2130a9b60bee2581bf82cfe89ee836424d1f37dcfa4ce21509611684673" dependencies = [ "libc", "memchr", @@ -9085,10 +9127,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9097,7 +9139,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9136,7 +9178,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9147,7 +9189,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9170,9 +9212,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.49" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" dependencies = [ "deranged", "num-conv", @@ -9190,9 +9232,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.29" +version = "0.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" +checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" dependencies = [ "num-conv", "time-core", @@ -9250,7 +9292,7 @@ checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9288,7 +9330,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9521,7 +9563,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9695,7 +9737,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9770,7 +9812,7 @@ checksum = "96cbd06a7b648f1603e60d75d9ed295d096b340d30e9f9324f4b512b5d40cd92" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10038,7 +10080,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10065,7 +10107,7 @@ version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", "rand 0.10.1", "serde_core", @@ -10185,7 +10227,7 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5924018406ce0063cd67f8e008104968b74b563ee1b85dde3ed1f7cb87d3dbd" dependencies = [ - "arrayvec 0.7.6", + "arrayvec 0.7.7", "memchr", ] @@ -10216,20 +10258,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] @@ -10240,9 +10273,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ "cfg-if", "once_cell", @@ -10254,9 +10287,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.73" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" dependencies = [ "js-sys", "wasm-bindgen", @@ -10264,9 +10297,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -10274,61 +10307,26 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "wasm-streams" version = "0.5.0" @@ -10342,23 +10340,11 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.13.0", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver", -] - [[package]] name = "web-sys" -version = "0.3.100" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ "js-sys", "wasm-bindgen", @@ -10376,9 +10362,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" dependencies = [ "rustls-pki-types", ] @@ -10389,14 +10375,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" dependencies = [ "rustls-pki-types", ] @@ -10458,7 +10444,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -10566,7 +10552,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10577,7 +10563,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10926,100 +10912,12 @@ dependencies = [ "memchr", ] -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck 0.5.0", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck 0.5.0", - "indexmap 2.14.0", - "prettyplease", - "syn 2.0.117", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.117", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags 2.13.0", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "writeable" version = "0.6.3" @@ -11106,7 +11004,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -11127,7 +11025,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11147,28 +11045,28 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11201,7 +11099,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11220,9 +11118,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" +checksum = "977347db8caa080403f6b6b7c1cda9479a8e869316f7e13a59b19076a40f94e3" [[package]] name = "zmij" diff --git a/Cargo.toml b/Cargo.toml index 12b599107..cd6498baa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ orion-client = { path = "clients/orion-client" } context = { path = "context" } -git-internal = "0.7.5" +git-internal = "0.7.6" libvault-core = "0.1.0" #==== @@ -67,7 +67,7 @@ russh = "0.61.2" tower-http = "0.7.0" tower = "0.5.3" tower-sessions = { version = "0.15", features = ["memory-store"] } -time = { version = "0.3.49", features = ["serde"] } +time = { version = "0.3.51", features = ["serde"] } lettre = { version = "0.11", default-features = false, features = [ "builder", "smtp-transport", @@ -99,7 +99,7 @@ uuid = "1.23.3" regex = "1.12.3" ed25519-dalek = "2.2.0" ctrlc = "3.5.2" -cedar-policy = "4.11.0" +cedar-policy = "4.11.2" secp256k1 = "0.31.1" pgp = "0.19.0" base64 = "0.22.1" @@ -111,7 +111,7 @@ tempfile = "3.27.0" dashmap = "6.2.1" once_cell = "1.21.4" serial_test = "3.5.0" -sysinfo = "0.39.3" +sysinfo = "0.39.5" http = "1.4.1" url = "2.5.8" jemallocator = "0.5.4" @@ -126,8 +126,8 @@ envsubst = "0.2.1" directories = "6.0.0" redis = "1.2.3" redis-test = "1.0.4" -rustls = "0.23" -object_store = "0.13.2" +rustls = "0.23.41" +object_store = "0.14.0" parse-display = "0.11.0" qlean = "0.3.0" toml = "1.1.2" diff --git a/docker/README.md b/docker/README.md index 0010f4ab1..d1f3b1cb8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -7,8 +7,8 @@ - [Demo Walk-Through](#demo-walk-through) - [Service Endpoints](#service-endpoints) - [FAQ](#faq) -- [Stopping & Cleanup](#stopping--cleanup) -- [Log Streaming](#log-streaming) +- [Stopping and Cleanup](#stopping-and-cleanup) +- [View logs](#view-logs) - [Architecture Overview](#architecture-overview) --- @@ -90,10 +90,23 @@ The main configurable environment variables include: - **Service Images**: - `MEGA_ENGINE_IMAGE`: Mega backend image (default: `public.ecr.aws/m8q5m4u3/mega/mono-engine:latest`) - - `MEGA_UI_IMAGE`: Mega UI image (default: `public.ecr.aws/m8q5m4u3/mega/mega-ui:demo-latest`) + - `MEGA_UI_IMAGE`: Mega UI unified image (default: `public.ecr.aws/m8q5m4u3/mega/mega-ui:latest`) - `CAMPSITE_API_IMAGE`: Campsite API image (default: `public.ecr.aws/m8q5m4u3/mega/campsite-api:latest`) + - `CAMPSITE_HOST_PORT`: Host port for Campsite API (default: `18080`, maps to container port `8080`; avoids conflict with local tools such as Cursor on `8080`) - `CAMPSITE_RUN_MIGRATIONS`: Whether to run database migrations when the container starts; `1` (default) to run, can be changed to `0` after the first successful migration to skip and speed up subsequent starts. +- **Mega UI Runtime URLs** (injected at container start; must be browser-reachable): + - `NEXT_PUBLIC_API_URL`: Campsite API for the **browser** (default: `http://api.gitmono.local:18080`) + - `NEXT_PUBLIC_INTERNAL_API_URL`: Campsite API for **Next.js SSR inside the mega_ui container** (default: `http://api.gitmono.local:8080` — Docker network alias + container port `8080`, not host port `18080`) + - `NEXT_PUBLIC_MONO_API_URL`: Mega / mono API (default: `http://git.gitmono.local:8000`) + - `NEXT_PUBLIC_ORION_API_URL`: Orion Server API (default: `http://orion.gitmono.local:8004`) + - `NEXT_PUBLIC_AUTH_URL`: OAuth / auth endpoint (default: `http://auth.gitmono.local:18080`) + - `NEXT_PUBLIC_WEB_URL`: Web UI origin (default: `http://app.gitmono.local`) + - `NEXT_PUBLIC_SYNC_URL`: Real-time sync WebSocket URL (optional; leave empty if not used) + - `NEXT_PUBLIC_CRATES_PRO_URL`: Crates Pro API URL (optional; leave empty if not used) + + The UI image is **environment-agnostic**: it is built once with placeholder URLs and `docker-entrypoint.sh` rewrites them from the variables above when `mega_ui` starts. You no longer need per-environment image tags such as `demo-latest`. + - **RustFS Configuration**: - `RUSTFS_ACCESS_KEY`: RustFS access key (default: `rustfsadmin`) - `RUSTFS_SECRET_KEY`: RustFS secret key (default: `rustfsadmin`) @@ -111,6 +124,13 @@ Execute in the project root directory: docker compose -f docker/demo/docker-compose.demo.yml up -d ``` +Or, if you maintain a local `.env` under `docker/demo/`: + +```bash +cd docker/demo +docker compose -f docker-compose.demo.yml up -d +``` + This command will: 1. Pull the required Docker images (may take a long time for the first run) @@ -118,7 +138,9 @@ This command will: 3. Start all services in dependency order: - First, start infrastructure services (PostgreSQL, MySQL, Redis, RustFS) - Then, start application services (Mega, Orion Server, Campsite API) - - Finally, start client services (Mega UI, Orion Build Client) + - Finally, start `mega_ui`, which injects `NEXT_PUBLIC_*` URLs at container start and serves the web UI on port 80 + +On first `mega_ui` start you should see log lines such as `[entrypoint] injecting runtime environment` followed by `NEXT_PUBLIC_* applied` — that confirms the unified image picked up your demo URLs. ### 4. Check service status @@ -224,7 +246,7 @@ Log in with the following credentials: | **Mega UI** | | Web Frontend UI | | **Mega API** | | Mega backend API | | **Orion Server** | | Orion build server API | -| **Campsite API** | | Campsite OAuth/SSO API | +| **Campsite API** | | Campsite OAuth/SSO API | | **PostgreSQL** | localhost:5432 | Database (used by Mega & Orion, mapped to host port 5432 in demo) | | **MySQL** | localhost:3306 | Database (used by Campsite API, mapped to host port 3306 in demo) | | **Redis** | localhost:6379 | Cache service (mapped to host port 6379 in demo) | @@ -233,14 +255,32 @@ Log in with the following credentials: ### API Health Check Endpoints -- **Mega API**: `GET http://api.gitmono.lcoal:8000/api/v1/status` +- **Mega API**: `GET http://api.gitmono.local:8000/api/v1/status` - **Orion Server**: `GET http://orion.gitmono.local:8004/v2/health` -- **Campsite API**: `GET http://api.gitmono.local:8080/health` +- **Campsite API**: `GET http://api.gitmono.local:18080/health` --- ## FAQ +### Mega UI points at wrong API URLs + +**Problem**: After opening , API calls fail or target placeholder hostnames such as `rt-api.placeholder.local`. + +**Cause**: The unified UI image did not receive `NEXT_PUBLIC_*` values at container start (older image without `docker-entrypoint.sh`, or missing environment block). + +**Solution**: + +1. Ensure `mega_ui` uses `public.ecr.aws/m8q5m4u3/mega/mega-ui:latest` (or a locally built unified image). +2. Confirm `docker/demo/.env` includes the `NEXT_PUBLIC_*` block from `.env.example`, or rely on the defaults in `docker-compose.demo.yml`. +3. Restart the UI container and check logs: + + ```bash + docker compose -f docker/demo/docker-compose.demo.yml logs mega_ui | grep entrypoint + ``` + + You should see `NEXT_PUBLIC_* applied` for each configured variable. + ### Port Conflict **Issue**: Docker reports the port is already allocated @@ -526,11 +566,28 @@ The demo environment includes the following services: - **Application services**: - `mega`: Mega backend (Rust) - - `mega_ui`: Mega Web UI (Next.js) + - `mega_ui`: Mega Web UI (Next.js, unified image with runtime `NEXT_PUBLIC_*` injection) - `orion_server`: Orion build server (Rust) - - `orion_build_client`: Orion build client (based on the orion-client image) - `campsite_api`: Campsite API (Ruby/Rails, built locally by default; if you have the encrypted development credentials configured you can pull the pre-built image directly via `CAMPSITE_API_IMAGE=public.ecr.aws/m8q5m4u3/mega/campsite-api:latest`) +### Mega UI: unified image + runtime configuration + +Previously each environment shipped its own UI image tag (`demo-latest`, `gitmono-latest`, etc.) with URLs baked in at build time. The current flow uses **one** `mega-ui:latest` image for every environment: + +1. **Build** (`moon/apps/web/Dockerfile`): compile with placeholder URLs from `moon/apps/web/.env.runtime`. +2. **Start** (`moon/apps/web/docker-entrypoint.sh`): replace placeholders in the compiled assets with `NEXT_PUBLIC_*` values from the container environment, then run `node server.js`. +3. **Configure locally**: set the `NEXT_PUBLIC_*` block in `docker/demo/.env` (see `.env.example`). Defaults in `docker-compose.demo.yml` already target the `gitmono.local` demo hostnames. + +To build the UI image locally instead of pulling from ECR: + +```bash +./scripts/demo/build-demo-images-local.sh mega-ui +# then in docker/demo/.env: +# MEGA_UI_IMAGE=public.ecr.aws/m8q5m4u3/mega/mega-ui:latest- +``` + +Replace `` with `arm64` or `amd64` to match your machine. + For a detailed architecture diagram and dependency list, see the [Mega / Orion Demo architecture design document](./mega-orion-demo-compose-arch.md). --- diff --git a/docker/demo/.env.example b/docker/demo/.env.example index 63eb811e4..c128903b0 100644 --- a/docker/demo/.env.example +++ b/docker/demo/.env.example @@ -51,10 +51,32 @@ S3_SECRET_ACCESS_KEY=rustfsadmin # You can override these to use local images or different tags MEGA_ENGINE_IMAGE=public.ecr.aws/m8q5m4u3/mega/mono-engine:latest -MEGA_UI_IMAGE=public.ecr.aws/m8q5m4u3/mega/mega-ui:demo-latest +# Unified web image (same tag for all environments; URLs injected at container start). +# Remote default: mega-ui:latest +# Local build (build-demo-images-local.sh): mega-ui:latest- e.g. latest-arm64 +MEGA_UI_IMAGE=public.ecr.aws/m8q5m4u3/mega/mega-ui:latest ORION_SERVER_IMAGE=public.ecr.aws/m8q5m4u3/mega/orion-server:latest CAMPSITE_API_IMAGE=public.ecr.aws/m8q5m4u3/mega/campsite-api:latest +# ---------------------------------------------------------------------------- +# Mega UI Runtime Configuration (unified image) +# ---------------------------------------------------------------------------- +# The web image is built once with placeholder URLs. docker-entrypoint.sh +# rewrites them with these values when the container starts. URLs must be +# reachable from the browser (use gitmono.local hostnames, not in-container +# service names). + +NEXT_PUBLIC_API_URL=http://api.gitmono.local:18080 +# Server-side (SSR) from mega_ui container — container network port, not host port +NEXT_PUBLIC_INTERNAL_API_URL=http://api.gitmono.local:8080 +NEXT_PUBLIC_MONO_API_URL=http://git.gitmono.local:8000 +NEXT_PUBLIC_ORION_API_URL=http://orion.gitmono.local:8004 +NEXT_PUBLIC_AUTH_URL=http://auth.gitmono.local:18080 +NEXT_PUBLIC_WEB_URL=http://app.gitmono.local +# Optional — leave empty when sync / crates-pro are not part of the demo stack +NEXT_PUBLIC_SYNC_URL= +NEXT_PUBLIC_CRATES_PRO_URL= + # ---------------------------------------------------------------------------- # Mega Backend Service Configuration # ---------------------------------------------------------------------------- @@ -76,7 +98,7 @@ MEGA_LOG__PRINT_STD=true MEGA_BUILD__ENABLE_BUILD=true MEGA_BUILD__ORION_SERVER=http://orion_server:8004 -MEGA_OAUTH__CAMPSITE_API_DOMAIN=http://api.gitmono.local:8080 +MEGA_OAUTH__CAMPSITE_API_DOMAIN=http://api.gitmono.local:18080 MEGA_OAUTH__UI_DOMAIN=http://app.gitmono.local MEGA_OAUTH__COOKIE_DOMAIN=gitmono.local MEGA_OAUTH__ALLOWED_CORS_ORIGINS="http://app.gitmono.local" @@ -100,6 +122,9 @@ ORION_MONOBASE_URL=http://mega:8000 # Campsite API Configuration # ---------------------------------------------------------------------------- +# Host port mapped to campsite_api container port 8080 (default 18080 to avoid local 8080 conflicts) +CAMPSITE_HOST_PORT=18080 + CAMPSITE_REDIS_URL=redis://redis:6379 CAMPSITE_RAILS_ENV=demo CAMPSITE_SERVER_COMMAND=bundle exec puma diff --git a/docker/demo/docker-compose.demo.yml b/docker/demo/docker-compose.demo.yml index c445be82f..a55dd1018 100644 --- a/docker/demo/docker-compose.demo.yml +++ b/docker/demo/docker-compose.demo.yml @@ -201,20 +201,33 @@ services: restart: "no" # --------------------------------------------------------------------------- - # Mega UI (Next.js) + # Mega UI (Next.js, unified image) # --------------------------------------------------------------------------- - # Use pre-built image from ECR Public: public.ecr.aws/m8q5m4u3/mega/mega-ui:{env}-latest - # Available environments: staging, openatom, gitmono - # Set MEGA_UI_IMAGE environment variable to use ECR image, or tag the ECR image as mega:mono-ui-latest-release - # Default uses local tag (mega:mono-ui-latest-release) if ECR image is tagged locally + # Single environment-agnostic image: built with placeholder URLs (.env.runtime) + # and configured at container start by docker-entrypoint.sh using the + # NEXT_PUBLIC_* variables below. Override MEGA_UI_IMAGE to pin a tag or use + # a locally built image (see scripts/demo/build-demo-images-local.sh). mega_ui: - image: ${MEGA_UI_IMAGE:-public.ecr.aws/m8q5m4u3/mega/mega-ui:demo-latest} + image: ${MEGA_UI_IMAGE:-public.ecr.aws/m8q5m4u3/mega/mega-ui:latest} container_name: mega-demo-ui depends_on: mega: condition: service_healthy orion_server: condition: service_healthy + environment: + # Runtime injection for the unified web image (browser-facing URLs). + # Browser-facing Campsite URL (host port 18080). + NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-http://api.gitmono.local:18080} + # SSR / server-side calls from mega_ui run inside Docker; use container port 8080. + NEXT_PUBLIC_INTERNAL_API_URL: ${NEXT_PUBLIC_INTERNAL_API_URL:-http://api.gitmono.local:8080} + NEXT_PUBLIC_MONO_API_URL: ${NEXT_PUBLIC_MONO_API_URL:-http://git.gitmono.local:8000} + NEXT_PUBLIC_ORION_API_URL: ${NEXT_PUBLIC_ORION_API_URL:-http://orion.gitmono.local:8004} + NEXT_PUBLIC_AUTH_URL: ${NEXT_PUBLIC_AUTH_URL:-http://auth.gitmono.local:18080} + NEXT_PUBLIC_WEB_URL: ${NEXT_PUBLIC_WEB_URL:-http://app.gitmono.local} + # Optional: leave unset when the demo stack has no sync / crates-pro service. + NEXT_PUBLIC_SYNC_URL: ${NEXT_PUBLIC_SYNC_URL:-} + NEXT_PUBLIC_CRATES_PRO_URL: ${NEXT_PUBLIC_CRATES_PRO_URL:-} ports: # Container listens on 3000, host port is 80 - "80:3000" @@ -321,8 +334,8 @@ services: exec ${CAMPSITE_SERVER_COMMAND:-bundle exec puma} ports: - # Container listens on 8080, host port is 8080 - - "8080:8080" + # Container listens on 8080; host port 18080 avoids conflict with Cursor / other local services on 8080 + - "${CAMPSITE_HOST_PORT:-18080}:8080" healthcheck: # Campsite API health check # NOTE: avoid curl dependency; use bash built-in /dev/tcp diff --git a/docker/deployment/.env.example b/docker/deployment/.env.example index 9412fd39a..a7f9d2e8d 100644 --- a/docker/deployment/.env.example +++ b/docker/deployment/.env.example @@ -51,11 +51,25 @@ S3_SECRET_ACCESS_KEY=rustfsadmin # You can override these to use local images or different tags MEGA_ENGINE_IMAGE=ghcr.io/web3infra-foundation/mono-engine:latest -# waiting for next release for demo image -MEGA_UI_IMAGE=public.ecr.aws/m8q5m4u3/mega/mega-ui:demo-latest +# Unified web image (same tag for all environments; URLs injected at container start) +MEGA_UI_IMAGE=public.ecr.aws/m8q5m4u3/mega/mega-ui:latest ORION_SERVER_IMAGE=ghcr.io/web3infra-foundation/orion-server:latest CAMPSITE_API_IMAGE=ghcr.io/web3infra-foundation/campsite-api:latest +# ---------------------------------------------------------------------------- +# Mega UI Runtime Configuration (unified image) +# ---------------------------------------------------------------------------- + +NEXT_PUBLIC_API_URL=http://api.gitmono.local:18080 +# Server-side (SSR) from mega_ui container — container network port, not host port +NEXT_PUBLIC_INTERNAL_API_URL=http://api.gitmono.local:8080 +NEXT_PUBLIC_MONO_API_URL=http://git.gitmono.local:8000 +NEXT_PUBLIC_ORION_API_URL=http://orion.gitmono.local:8004 +NEXT_PUBLIC_AUTH_URL=http://auth.gitmono.local:18080 +NEXT_PUBLIC_WEB_URL=http://app.gitmono.local +NEXT_PUBLIC_SYNC_URL= +NEXT_PUBLIC_CRATES_PRO_URL= + # ---------------------------------------------------------------------------- # Mega Backend Service Configuration # ---------------------------------------------------------------------------- @@ -77,7 +91,7 @@ MEGA_LOG__PRINT_STD=true MEGA_BUILD__ENABLE_BUILD=true MEGA_BUILD__ORION_SERVER=http://orion_server:8004 -MEGA_OAUTH__CAMPSITE_API_DOMAIN=http://api.gitmono.local:8080 +MEGA_OAUTH__CAMPSITE_API_DOMAIN=http://api.gitmono.local:18080 MEGA_OAUTH__UI_DOMAIN=http://app.gitmono.local MEGA_OAUTH__COOKIE_DOMAIN=gitmono.local MEGA_OAUTH__ALLOWED_CORS_ORIGINS="http://app.gitmono.local" diff --git a/io-orbit/Cargo.toml b/io-orbit/Cargo.toml index 40def9e07..a2c843b71 100644 --- a/io-orbit/Cargo.toml +++ b/io-orbit/Cargo.toml @@ -11,7 +11,7 @@ path = "src/lib.rs" [dependencies] common = { workspace = true } -object_store = { workspace = true, features = ["cloud", "aws", "gcp"] } +object_store = { workspace = true, features = ["cloud-base", "aws", "gcp"] } futures = { workspace = true } bytes = { workspace = true } async-trait = { workspace = true } diff --git a/moon/Caddyfile b/moon/Caddyfile index cff616c08..341b09055 100644 --- a/moon/Caddyfile +++ b/moon/Caddyfile @@ -1,10 +1,30 @@ +# Local HTTPS reverse proxy for Next.js dev. +# +# Browser → Caddy (:443, HTTPS) → Next.js dev (127.0.0.1:80) +# +# Prerequisites: +# 1. Add to /etc/hosts: +# 127.0.0.1 local.xuanwu.openatom.cn +# 2. TLS certs in this directory (moon/): +# ./local.xuanwu.openatom.cn.pem +# ./local.xuanwu.openatom.cn-key.pem +# 3. Start Next.js dev on port 80 before Caddy. +# +# Usage (run from moon/): +# sudo caddy run --config Caddyfile # foreground +# sudo caddy start --config Caddyfile # background +# sudo caddy stop +# caddy validate --config Caddyfile +# caddy fmt --overwrite Caddyfile +# +# Open: https://local.xuanwu.openatom.cn { - # Next.js dev already owns port 80, so don't let Caddy bind it for - # the HTTP->HTTPS redirect vhost. - auto_https disable_redirects + # Next.js dev already owns port 80, so don't let Caddy bind it for + # the HTTP->HTTPS redirect vhost. + auto_https disable_redirects } https://local.xuanwu.openatom.cn { - tls ./local.xuanwu.openatom.cn.pem ./local.xuanwu.openatom.cn-key.pem - reverse_proxy 127.0.0.1:80 -} \ No newline at end of file + tls ./local.xuanwu.openatom.cn.pem ./local.xuanwu.openatom.cn-key.pem + reverse_proxy 127.0.0.1:80 +} diff --git a/moon/apps/web/components/ClView/components/Checks/LogViewer.tsx b/moon/apps/web/components/ClView/components/Checks/LogViewer.tsx index ab60d51c6..36593dfdb 100644 --- a/moon/apps/web/components/ClView/components/Checks/LogViewer.tsx +++ b/moon/apps/web/components/ClView/components/Checks/LogViewer.tsx @@ -12,6 +12,7 @@ import { import { Virtuoso, type VirtuosoHandle } from 'react-virtuoso' import { parseAnsi, type AnsiSegment } from './ansi' +import { applyErrorHighlight, isErrorLogLine } from './hooks/logUtils' export interface LogViewerProps { text: string @@ -79,12 +80,13 @@ const LogRow = memo(function LogRow({ searchQuery, getSegments }: LogRowProps) { - const segments = getSegments(line) + const isErrorLine = isErrorLogLine(line) + const segments = isErrorLine ? applyErrorHighlight(getSegments(line)) : getSegments(line) return (
) } + +/** Keep LogViewer mounted but visually hidden so Virtuoso retains scroll position. */ +export const CachedLogPanel = memo(function CachedLogPanel({ + text, + height, + visible +}: LogViewerProps & { visible: boolean }) { + return ( +
+ +
+ ) +}) diff --git a/moon/apps/web/components/ClView/components/Checks/cpns/Task.tsx b/moon/apps/web/components/ClView/components/Checks/cpns/Task.tsx index 013e34405..6f6a680fb 100644 --- a/moon/apps/web/components/ClView/components/Checks/cpns/Task.tsx +++ b/moon/apps/web/components/ClView/components/Checks/cpns/Task.tsx @@ -1,14 +1,13 @@ -import { useMemo, useState } from 'react' +import { memo, useMemo, useState } from 'react' import { CheckIcon, ChevronDownIcon, ClockIcon, FileDirectoryIcon, SyncIcon, XIcon } from '@primer/octicons-react' import { format } from 'date-fns' -import { useAtom } from 'jotai' import { StatusProjectRelativePath } from '@gitmono/types/generated' import { LoadingSpinner } from '@gitmono/ui/Spinner' -import { buildIdAtom } from '@/components/Issues/utils/store' import { usePostRetryBuild } from '@/hooks/SSE/usePostRetryBuild' +import { TERMINAL_BUILD_STATUSES } from '../hooks/logUtils' import { BuildDTO, getLatestBuildId, isTaskQueued, Status, TaskInfoDTO } from './store' /** @@ -24,13 +23,13 @@ const formatDateTime = (isoDate: string): string => { } } -type LogStatus = 'idle' | 'loading' | 'success' | 'empty' | 'error' - export interface TreeRootProps { path?: string prName?: string tasks: TaskInfoDTO[] - logStatus: Record + logsAvailableIds: Set + selectedBuildId: string + onSelectBuild: (buildId: string, taskId?: string) => void totalTasksCount?: number cl: string } @@ -38,7 +37,16 @@ export interface TreeRootProps { /** * Tree Root Component - Top level node showing the path */ -export const TreeRoot = ({ path, prName, tasks, logStatus, totalTasksCount, cl }: TreeRootProps) => { +export const TreeRoot = ({ + path, + prName, + tasks, + logsAvailableIds, + selectedBuildId, + onSelectBuild, + totalTasksCount, + cl +}: TreeRootProps) => { const [isExpanded, setIsExpanded] = useState(true) // Show the total number of tasks in dropdown (displayed as "builds") @@ -71,7 +79,9 @@ export const TreeRoot = ({ path, prName, tasks, logStatus, totalTasksCount, cl } key={t.task_id} list={t} prName={prName} - logStatus={logStatus} + logsAvailableIds={logsAvailableIds} + selectedBuildId={selectedBuildId} + onSelectBuild={onSelectBuild} isLast={index === tasks.length - 1} cl={cl} /> @@ -88,13 +98,17 @@ export const TreeRoot = ({ path, prName, tasks, logStatus, totalTasksCount, cl } export const Task = ({ list, prName, - logStatus, + logsAvailableIds, + selectedBuildId, + onSelectBuild, isLast, cl }: { list: TaskInfoDTO prName?: string - logStatus: Record + logsAvailableIds: Set + selectedBuildId: string + onSelectBuild: (buildId: string, taskId?: string) => void isLast?: boolean cl: string }) => { @@ -225,7 +239,9 @@ export const Task = ({ key={i.id} build={i} seq={index + 1} - logStatus={logStatus[i.id]} + hasLogs={logsAvailableIds.has(i.id)} + isSelected={selectedBuildId === i.id} + onSelectBuild={(buildId) => onSelectBuild(buildId, list.task_id)} isLast={index === orderedBuilds.length - 1} cl={cl} clId={list.cl_id} @@ -243,10 +259,12 @@ export const Task = ({ /** * TaskItem Component - Build item showing individual builds */ -export const TaskItem = ({ +const TaskItem = memo(function TaskItem({ build, seq, - logStatus, + hasLogs, + isSelected, + onSelectBuild, isLast, cl, clId, @@ -256,23 +274,29 @@ export const TaskItem = ({ }: { build: BuildDTO seq?: number - logStatus?: LogStatus + hasLogs?: boolean + isSelected: boolean + onSelectBuild: (buildId: string) => void isLast?: boolean cl: string clId?: number changes?: StatusProjectRelativePath[] isQueued?: boolean isLatestBuild?: boolean -}) => { - const [buildId, setBuildId] = useAtom(buildIdAtom) +}) { const { mutate: retryBuild, isPending: isRetrying } = usePostRetryBuild(cl) - // A not-yet-started build (queued, waiting for a worker) reports as "Building" - // because it has no end_at; surface it distinctly so users know it has no logs. const showQueued = Boolean(isQueued) && build.status === 'Building' - const handleClick = (build_id: string) => { - setBuildId(build_id) + const handleClick = (e: React.MouseEvent) => { + const scrollParent = e.currentTarget.closest('[data-build-list-scroll]') as HTMLElement | null + const scrollTop = scrollParent?.scrollTop ?? 0 + + onSelectBuild(build.id) + + requestAnimationFrame(() => { + if (scrollParent) scrollParent.scrollTop = scrollTop + }) } const handleRetry = (e: React.MouseEvent) => { @@ -287,12 +311,8 @@ export const TaskItem = ({ }) } - // Retry only the task's latest build, and only when it has failed or been - // interrupted. Being the latest finished build already guarantees no newer - // build is in flight; the backend enforces the same invariant. - const canRetry = Boolean(isLatestBuild) && (build.status === 'Failed' || build.status === 'Interrupted') - const isSelected = buildId === build.id - const isHighlighted = logStatus === 'success' + const canRetry = Boolean(isLatestBuild) && TERMINAL_BUILD_STATUSES.has(build.status) + const isHighlighted = hasLogs let bgClass = 'bg-white dark:bg-gray-800/30' let textColor = 'text-gray-600 dark:text-gray-400' @@ -308,7 +328,7 @@ export const TaskItem = ({ return (
handleClick(build.id)} + onClick={handleClick} className={`group flex cursor-pointer items-center gap-2 py-2 pl-9 pr-3 transition-all hover:bg-gray-100 dark:hover:bg-gray-700/30 ${bgClass} ${ isSelected ? `border-l-2 ${borderColor}` : 'border-l-2 border-transparent' } ${!isLast ? 'border-b border-gray-100 dark:border-gray-800' : ''}`} @@ -356,7 +376,7 @@ export const TaskItem = ({ {isSelected &&
}
) -} +}) export const identifyStatus = (status: Status[keyof Status]) => { switch (status) { diff --git a/moon/apps/web/components/ClView/components/Checks/cpns/store.ts b/moon/apps/web/components/ClView/components/Checks/cpns/store.ts index 54e838c7e..967ee207b 100644 --- a/moon/apps/web/components/ClView/components/Checks/cpns/store.ts +++ b/moon/apps/web/components/ClView/components/Checks/cpns/store.ts @@ -1,4 +1,5 @@ import { atom } from 'jotai' +import { atomFamily } from 'jotai/utils' import { StatusProjectRelativePath, TargetState } from '@gitmono/types/generated' @@ -80,6 +81,37 @@ export const getLatestBuildId = (task: TaskInfoDTO): string | undefined => { return latest?.id } +/** Most recent build across all tasks (by start time). */ +export const getLatestBuildIdFromTasks = (tasks: TaskInfoDTO[]): string | undefined => { + let latest: BuildDTO | undefined + + tasks.forEach((task) => { + task.build_list?.forEach((build) => { + if (!latest || new Date(build.start_at).getTime() > new Date(latest.start_at).getTime()) { + latest = build + } + }) + }) + + return latest?.id +} + +export const getAllBuildIds = (tasks: TaskInfoDTO[]): Set => { + const ids = new Set() + + tasks.forEach((task) => { + task.build_list?.forEach((build) => { + if (build.id) ids.add(build.id) + }) + }) + + return ids +} + +export const findTaskIdByBuildId = (tasks: TaskInfoDTO[], buildId: string): string | undefined => { + return tasks.find((task) => task.build_list?.some((build) => build.id === buildId))?.task_id +} + /** * Collect the build ids that are still queued (waiting for a worker). These have * no logs yet, so callers should avoid fetching logs for them and instead show a @@ -99,8 +131,9 @@ export const getQueuedBuildIds = (tasks: TaskInfoDTO[]): Set => { return ids } -export const logsAtom = atom>({}) +export const buildIdAtomFamily = atomFamily((_cl: string) => atom('')) +export const logsAtomFamily = atomFamily((_cl: string) => atom>({})) + export const statusAtom = atom>({}) export const loadingAtom = atom(true) -export const statusMapAtom = atom>(new Map()) export const tabAtom = atom<'conversation' | 'check' | 'filechange'>('conversation') diff --git a/moon/apps/web/components/ClView/components/Checks/cpns/tasksSignature.ts b/moon/apps/web/components/ClView/components/Checks/cpns/tasksSignature.ts new file mode 100644 index 000000000..1932b0796 --- /dev/null +++ b/moon/apps/web/components/ClView/components/Checks/cpns/tasksSignature.ts @@ -0,0 +1,17 @@ +import { TaskInfoDTO } from './store' + +/** + * Stable signature so effects only re-run when build ids/statuses or target states + * change. Target states must be included: a build can stay `Building` while + * targets move from `Uninitialized` (queued) to `Building` (worker picked up). + */ +export function getTasksSignature(tasks: TaskInfoDTO[]): string { + return tasks + .map((t) => { + const builds = t.build_list?.map((b) => `${b.id}:${b.status}`).join(',') ?? '' + const targets = t.targets?.map((tg) => tg.state).join(',') ?? '' + + return `${t.task_id}:${builds}:${targets}` + }) + .join('|') +} diff --git a/moon/apps/web/components/ClView/components/Checks/hooks/logUtils.ts b/moon/apps/web/components/ClView/components/Checks/hooks/logUtils.ts new file mode 100644 index 000000000..8fd96f8f0 --- /dev/null +++ b/moon/apps/web/components/ClView/components/Checks/hooks/logUtils.ts @@ -0,0 +1,114 @@ +import { GetBuildsLogsV2Data } from '@gitmono/types/generated' + +import { type AnsiSegment } from '../ansi' +import { BuildDTO, BuildStatus, TaskInfoDTO } from '../cpns/store' + +export type LogStatus = 'idle' | 'loading' | 'success' | 'empty' | 'error' + +export const TERMINAL_BUILD_STATUSES = new Set(['Completed', 'Failed', 'Interrupted']) + +export const MAX_MOUNTED_LOG_PANELS = 6 + +/** Default log foreground from ansi.ts — used to decide when to apply error highlighting. */ +const DEFAULT_LOG_FG = '#d4d4d4' + +export const ERROR_LOG_FG = '#f14c4c' + +/** Heuristics for plain-text build errors (buck2, rustc, scorpio, etc.). */ +const ERROR_LINE_PATTERNS: RegExp[] = [ + /\bBUILD FAILED\b/, + /\bAction failed\b/i, + /\(os error \d+\)/i, + /\bNo such file or directory\b/i, + /\bPermission denied\b/i, + /\bBad file descriptor\b/i, + /\bInput\/output error\b/i, + /\btransport endpoint is not connected\b/i, + /\bfatal:\s/i, + /\bpanicked at\b/i, + /^\s*error\[/i, + /^\[error\]/i, + /: error:/i +] + +function stripAnsiSequences(line: string): string { + // eslint-disable-next-line no-control-regex -- strip SGR for pattern matching only + return line.replace(/\x1b\[[0-9;?]*m/g, '') +} + +/** True when a log line looks like a build/runtime error in plain text. */ +export function isErrorLogLine(line: string): boolean { + const plain = stripAnsiSequences(line).trim() + + if (!plain) return false + + return ERROR_LINE_PATTERNS.some((pattern) => pattern.test(plain)) +} + +export function isDefaultLogColor(color: string | undefined): boolean { + return !color || color === DEFAULT_LOG_FG +} + +export function applyErrorHighlight(segments: AnsiSegment[]): AnsiSegment[] { + return segments.map((seg) => + isDefaultLogColor(seg.style.color as string | undefined) + ? { ...seg, style: { ...seg.style, color: ERROR_LOG_FG } } + : seg + ) +} + +export function parseBuildLogResponse(res: GetBuildsLogsV2Data | null | undefined): { + status: LogStatus + text: string +} { + if (!res || !res.data) { + return { status: 'empty', text: '' } + } + + if (Array.isArray(res.data) && res.data.length === 0) { + return { status: 'empty', text: '' } + } + + if (res.len === 0) { + return { status: 'empty', text: '' } + } + + const text = Array.isArray(res.data) ? res.data.join('\n') : String(res.data || '') + + if (!text) { + return { status: 'empty', text: '' } + } + + return { status: 'success', text } +} + +export function findBuildInTasks(tasks: TaskInfoDTO[], buildId: string): BuildDTO | undefined { + for (const task of tasks) { + const build = task.build_list?.find((b) => b.id === buildId) + + if (build) return build + } + + return undefined +} + +/** Adjacent build ids in chronological order within the same task. */ +export function getAdjacentBuildIds(tasks: TaskInfoDTO[], buildId: string): string[] { + for (const task of tasks) { + const list = [...(task.build_list ?? [])].sort( + (a, b) => new Date(a.start_at).getTime() - new Date(b.start_at).getTime() + ) + const index = list.findIndex((b) => b.id === buildId) + + if (index === -1) continue + + const adjacent: string[] = [] + + if (index > 0) adjacent.push(list[index - 1].id) + if (index < list.length - 1) adjacent.push(list[index + 1].id) + + return adjacent + } + + return [] +} diff --git a/moon/apps/web/components/ClView/components/Checks/hooks/syncChecksUrl.ts b/moon/apps/web/components/ClView/components/Checks/hooks/syncChecksUrl.ts new file mode 100644 index 000000000..e346d0fcc --- /dev/null +++ b/moon/apps/web/components/ClView/components/Checks/hooks/syncChecksUrl.ts @@ -0,0 +1,37 @@ +/** Update tab=check and build query params without triggering a Next.js route transition. */ +export function syncChecksUrl(buildId: string) { + if (typeof window === 'undefined' || !buildId) return + + const url = new URL(window.location.href) + + if (url.searchParams.get('tab') === 'check' && url.searchParams.get('build') === buildId) { + return + } + + url.searchParams.set('tab', 'check') + url.searchParams.set('build', buildId) + + const href = `${url.pathname}${url.search}${url.hash}` + + window.history.replaceState(window.history.state, '', href) +} + +export function readBuildFromUrl(routerBuild: string | string[] | undefined): string | undefined { + if (typeof routerBuild === 'string' && routerBuild) return routerBuild + + if (typeof window !== 'undefined') { + return new URLSearchParams(window.location.search).get('build') ?? undefined + } + + return undefined +} + +export function readTabFromUrl(routerTab: string | string[] | undefined): string | undefined { + if (typeof routerTab === 'string' && routerTab) return routerTab + + if (typeof window !== 'undefined') { + return new URLSearchParams(window.location.search).get('tab') ?? undefined + } + + return undefined +} diff --git a/moon/apps/web/components/ClView/components/Checks/hooks/useBuildSelection.ts b/moon/apps/web/components/ClView/components/Checks/hooks/useBuildSelection.ts new file mode 100644 index 000000000..4bdfe5dfb --- /dev/null +++ b/moon/apps/web/components/ClView/components/Checks/hooks/useBuildSelection.ts @@ -0,0 +1,122 @@ +import { useCallback, useEffect, useRef } from 'react' +import { atom, useAtom } from 'jotai' +import { atomFamily } from 'jotai/utils' +import { useRouter } from 'next/router' + +import { + buildIdAtomFamily, + findTaskIdByBuildId, + getAllBuildIds, + getLatestBuildId, + getLatestBuildIdFromTasks, + TaskInfoDTO +} from '../cpns/store' +import { readBuildFromUrl, syncChecksUrl } from './syncChecksUrl' + +const selectedTaskIdAtomFamily = atomFamily((_cl: string) => atom(null)) + +export function useBuildSelection(cl: string, tasks: TaskInfoDTO[] | undefined) { + const router = useRouter() + const [buildId, setBuildId] = useAtom(buildIdAtomFamily(cl)) + const [selectedTaskId, setSelectedTaskId] = useAtom(selectedTaskIdAtomFamily(cl)) + const taskBuildMemoryRef = useRef>(new Map()) + const prevClRef = useRef(cl) + const syncedBuildRef = useRef(null) + + useEffect(() => { + if (prevClRef.current === cl) return + + prevClRef.current = cl + syncedBuildRef.current = null + taskBuildMemoryRef.current.clear() + setBuildId('') + setSelectedTaskId(null) + }, [cl, setBuildId, setSelectedTaskId]) + + useEffect(() => { + if (!router.isReady || !tasks || tasks.length === 0) return + + const allBuildIds = getAllBuildIds(tasks) + const queryBuild = readBuildFromUrl(router.query.build) + + let targetBuildId: string | undefined + + if (queryBuild && allBuildIds.has(queryBuild)) { + targetBuildId = queryBuild + } else if (buildId && allBuildIds.has(buildId)) { + targetBuildId = buildId + } else { + targetBuildId = getLatestBuildIdFromTasks(tasks) + } + + if (targetBuildId && targetBuildId !== buildId) { + setBuildId(targetBuildId) + } + + const taskId = targetBuildId ? findTaskIdByBuildId(tasks, targetBuildId) : undefined + const validTasks = tasks.filter((t) => t.build_list && t.build_list.length > 0) + + if (taskId && taskId !== selectedTaskId) { + setSelectedTaskId(taskId) + } else if (!selectedTaskId && validTasks.length > 0) { + setSelectedTaskId(validTasks[0].task_id) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [tasks, router.isReady, router.query.build, cl]) + + useEffect(() => { + if (!buildId) return + + if (syncedBuildRef.current === buildId) return + + const urlBuild = typeof window !== 'undefined' ? new URLSearchParams(window.location.search).get('build') : null + const urlTab = typeof window !== 'undefined' ? new URLSearchParams(window.location.search).get('tab') : null + + if (urlTab === 'check' && urlBuild === buildId) { + syncedBuildRef.current = buildId + return + } + + syncedBuildRef.current = buildId + syncChecksUrl(buildId) + }, [buildId]) + + const selectBuild = useCallback( + (nextBuildId: string, taskId?: string) => { + if (taskId) { + taskBuildMemoryRef.current.set(taskId, nextBuildId) + } + + setBuildId(nextBuildId) + }, + [setBuildId] + ) + + const selectTask = useCallback( + (taskId: string) => { + setSelectedTaskId(taskId) + + const task = tasks?.find((t) => t.task_id === taskId) + + if (!task) return + + const remembered = taskBuildMemoryRef.current.get(taskId) + const rememberedValid = remembered && task.build_list?.some((b) => b.id === remembered) + + const nextBuildId = rememberedValid ? remembered : getLatestBuildId(task) + + if (nextBuildId) { + taskBuildMemoryRef.current.set(taskId, nextBuildId) + setBuildId(nextBuildId) + } + }, + [tasks, setBuildId, setSelectedTaskId] + ) + + return { + buildId, + selectBuild, + selectedTaskId, + selectTask + } +} diff --git a/moon/apps/web/components/ClView/components/Checks/hooks/useLeftPanelScroll.ts b/moon/apps/web/components/ClView/components/Checks/hooks/useLeftPanelScroll.ts new file mode 100644 index 000000000..5c1b27f6a --- /dev/null +++ b/moon/apps/web/components/ClView/components/Checks/hooks/useLeftPanelScroll.ts @@ -0,0 +1,41 @@ +import { RefObject, useEffect, useRef } from 'react' + +export function useLeftPanelScroll(cl: string, buildId: string, leftPanelRef: RefObject) { + const leftPanelScrollRef = useRef(0) + const prevClRef = useRef(cl) + + useEffect(() => { + if (prevClRef.current !== cl) { + prevClRef.current = cl + leftPanelScrollRef.current = 0 + } + }, [cl]) + + useEffect(() => { + const panel = leftPanelRef.current + + if (!panel) return + + const onScroll = () => { + leftPanelScrollRef.current = panel.scrollTop + } + + panel.addEventListener('scroll', onScroll, { passive: true }) + + return () => panel.removeEventListener('scroll', onScroll) + }, [leftPanelRef]) + + useEffect(() => { + const panel = leftPanelRef.current + + if (!panel) return + + const saved = leftPanelScrollRef.current + + requestAnimationFrame(() => { + panel.scrollTop = saved + }) + }, [buildId, leftPanelRef]) + + return {} +} diff --git a/moon/apps/web/components/ClView/components/Checks/hooks/useLogCache.ts b/moon/apps/web/components/ClView/components/Checks/hooks/useLogCache.ts new file mode 100644 index 000000000..d90c23a88 --- /dev/null +++ b/moon/apps/web/components/ClView/components/Checks/hooks/useLogCache.ts @@ -0,0 +1,211 @@ +import { useCallback, useEffect, useMemo, useRef } from 'react' +import { useQuery, useQueryClient } from '@tanstack/react-query' + +import { GetBuildsLogsV2Data } from '@gitmono/types/generated' + +import { getBuildLogQueryKey, getBuildLogQueryOptions, isTerminalBuildStatus } from '@/hooks/SSE/useGetHTTPLog' + +import { useTaskSSE } from '../../../hook/useSSM' +import { getQueuedBuildIds, TaskInfoDTO } from '../cpns/store' +import { getTasksSignature } from '../cpns/tasksSignature' +import { + findBuildInTasks, + getAdjacentBuildIds, + LogStatus, + parseBuildLogResponse, + TERMINAL_BUILD_STATUSES +} from './logUtils' + +export function useLogCache(cl: string, buildId: string, tasks: TaskInfoDTO[] | undefined) { + const queryClient = useQueryClient() + const { eventSourcesRef, setEventSource, closeEventSource, logsMap, setLogsMap } = useTaskSSE(cl) + const tasksSignatureRef = useRef('') + + const queuedBuildIds = useMemo(() => (tasks ? getQueuedBuildIds(tasks) : new Set()), [tasks]) + + const currentBuild = useMemo( + () => (tasks && buildId ? findBuildInTasks(tasks, buildId) : undefined), + [tasks, buildId] + ) + + const isQueued = Boolean(buildId && queuedBuildIds.has(buildId)) + const isBuilding = currentBuild?.status === 'Building' + const isTerminal = isTerminalBuildStatus(currentBuild?.status) + + const httpEnabled = Boolean(buildId) && !isQueued && !isBuilding + + const { + data: httpLog, + isLoading: isHttpLoading, + isError: isHttpError, + isFetching: isHttpFetching, + refetch: refetchHttpLog + } = useQuery({ + ...getBuildLogQueryOptions(buildId, isTerminal), + enabled: httpEnabled + }) + + // Manage SSE connections only when the tasks signature changes. + useEffect(() => { + if (!tasks?.length) return + + const signature = getTasksSignature(tasks) + + if (signature === tasksSignatureRef.current) return + + tasksSignatureRef.current = signature + + const buildingIds = new Set() + + tasks.forEach((task) => { + task.build_list?.forEach((build) => { + if (build.status === 'Building' && !queuedBuildIds.has(build.id)) { + buildingIds.add(build.id) + } + }) + }) + + buildingIds.forEach((id) => { + setEventSource(id) + }) + + Object.keys(eventSourcesRef.current).forEach((id) => { + if (!buildingIds.has(id)) { + closeEventSource(id) + } + }) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [tasks, queuedBuildIds]) + + // Merge HTTP log into logsMap (never replace longer SSE snapshot). + useEffect(() => { + if (!httpEnabled || !buildId || !httpLog) return + + const { text } = parseBuildLogResponse(httpLog) + + if (!text) return + + setLogsMap((prev) => { + const current = prev[buildId] ?? '' + const next = current.length > text.length ? current : text + + if (prev[buildId] === next) return prev + + return { ...prev, [buildId]: next } + }) + }, [httpEnabled, buildId, httpLog, setLogsMap]) + + // Prefetch adjacent terminal builds on idle. + useEffect(() => { + if (!buildId || !tasks?.length) return + + const adjacent = getAdjacentBuildIds(tasks, buildId) + + adjacent.forEach((id) => { + const build = findBuildInTasks(tasks, id) + + if (!build || build.status === 'Building' || queuedBuildIds.has(id)) return + + const terminal = TERMINAL_BUILD_STATUSES.has(build.status) + + void queryClient.prefetchQuery(getBuildLogQueryOptions(id, terminal)) + }) + }, [buildId, tasks, queryClient, queuedBuildIds]) + + // Hydrate logsMap from react-query cache (e.g. prefetched adjacent builds). + useEffect(() => { + if (!tasks?.length) return + + const updates: Record = {} + + tasks.forEach((task) => { + task.build_list?.forEach((build) => { + const cached = queryClient.getQueryData(getBuildLogQueryKey(build.id)) + + if (!cached) return + + const { text } = parseBuildLogResponse(cached) + + if (text) updates[build.id] = text + }) + }) + + if (Object.keys(updates).length === 0) return + + setLogsMap((prev) => { + let changed = false + const next = { ...prev } + + Object.entries(updates).forEach(([id, text]) => { + if (!next[id]) { + next[id] = text + changed = true + } + }) + + return changed ? next : prev + }) + }, [tasks, queryClient, setLogsMap]) + + const getLogStatus = useCallback( + (id: string): LogStatus => { + if (queuedBuildIds.has(id)) return 'idle' + + if (logsMap[id]) return 'success' + + if (id !== buildId) { + const cached = queryClient.getQueryData(getBuildLogQueryKey(id)) + + if (cached) { + const { status } = parseBuildLogResponse(cached) + + return status + } + + return 'idle' + } + + if (isBuilding) { + return logsMap[id] ? 'success' : 'loading' + } + + if (isHttpError) return 'error' + + if (isHttpLoading || isHttpFetching) return 'loading' + + if (httpLog) return parseBuildLogResponse(httpLog).status + + return 'idle' + }, + [buildId, httpLog, isBuilding, isHttpError, isHttpFetching, isHttpLoading, logsMap, queryClient, queuedBuildIds] + ) + + const currentLogStatus = buildId ? getLogStatus(buildId) : 'idle' + + const logsAvailableIds = useMemo(() => { + const ids = new Set() + + Object.entries(logsMap).forEach(([id, text]) => { + if (text) ids.add(id) + }) + + return ids + }, [logsMap]) + + const retryLog = useCallback(() => { + if (!buildId) return + + void queryClient.invalidateQueries({ queryKey: getBuildLogQueryKey(buildId) }) + void refetchHttpLog() + }, [buildId, queryClient, refetchHttpLog]) + + return { + logsMap, + logsAvailableIds, + getLogStatus, + currentLogStatus, + isQueued, + isBuilding, + retryLog + } +} diff --git a/moon/apps/web/components/ClView/components/Checks/hooks/useMountedLogPanels.ts b/moon/apps/web/components/ClView/components/Checks/hooks/useMountedLogPanels.ts new file mode 100644 index 000000000..a086a7f26 --- /dev/null +++ b/moon/apps/web/components/ClView/components/Checks/hooks/useMountedLogPanels.ts @@ -0,0 +1,25 @@ +import { useEffect, useState } from 'react' + +import { MAX_MOUNTED_LOG_PANELS } from './logUtils' + +/** LRU list of build ids with mounted LogViewer panels. */ +export function useMountedLogPanels(buildId: string, logsMap: Record) { + const [mountedIds, setMountedIds] = useState([]) + + useEffect(() => { + if (!buildId || !logsMap[buildId]) return + + setMountedIds((prev) => { + const withLogs = prev.filter((id) => logsMap[id] && id !== buildId) + const next = [...withLogs, buildId] + + while (next.length > MAX_MOUNTED_LOG_PANELS) { + next.shift() + } + + return next + }) + }, [buildId, logsMap]) + + return mountedIds.filter((id) => logsMap[id]) +} diff --git a/moon/apps/web/components/ClView/components/Checks/hooks/useResizablePanels.ts b/moon/apps/web/components/ClView/components/Checks/hooks/useResizablePanels.ts new file mode 100644 index 000000000..6227821a8 --- /dev/null +++ b/moon/apps/web/components/ClView/components/Checks/hooks/useResizablePanels.ts @@ -0,0 +1,124 @@ +import { useCallback, useEffect, useRef, useState } from 'react' + +const MIN_LEFT_WIDTH = 200 +const MAX_LEFT_WIDTH_PERCENT = 0.7 +const DEFAULT_LEFT_WIDTH_PERCENT = 0.2 + +export function useResizablePanels() { + const containerRef = useRef(null) + const leftPanelRef = useRef(null) + const rightPanelRef = useRef(null) + const logContainerRef = useRef(null) + const scrollPositionRef = useRef(0) + const startWidthRef = useRef(0) + + const [leftWidth, setLeftWidth] = useState(null) + const [isDragging, setIsDragging] = useState(false) + const [logViewerHeight, setLogViewerHeight] = useState(0) + + useEffect(() => { + if (containerRef.current && leftWidth === null) { + setLeftWidth(containerRef.current.offsetWidth * DEFAULT_LEFT_WIDTH_PERCENT) + } + }, [leftWidth]) + + useEffect(() => { + const el = containerRef.current + + if (!el) return + + const update = () => setLogViewerHeight(el.clientHeight) + + update() + + const observer = new ResizeObserver(update) + + observer.observe(el) + + return () => observer.disconnect() + }, []) + + const handleMouseMove = useCallback((e: MouseEvent) => { + if (!containerRef.current || !leftPanelRef.current) return + + const containerRect = containerRef.current.getBoundingClientRect() + const newLeftWidth = e.clientX - containerRect.left + const maxWidth = containerRect.width * MAX_LEFT_WIDTH_PERCENT + const clampedWidth = Math.max(MIN_LEFT_WIDTH, Math.min(newLeftWidth, maxWidth)) + + leftPanelRef.current.style.width = `${clampedWidth}px` + }, []) + + const handleMouseUp = useCallback(() => { + document.removeEventListener('mousemove', handleMouseMove) + document.removeEventListener('mouseup', handleMouseUp) + document.body.style.cursor = '' + document.body.style.userSelect = '' + + if (rightPanelRef.current) { + rightPanelRef.current.style.display = 'block' + } + + if (leftPanelRef.current) { + setLeftWidth(leftPanelRef.current.offsetWidth) + } + + setIsDragging(false) + }, [handleMouseMove]) + + const handleMouseDown = useCallback( + (e: React.MouseEvent) => { + e.preventDefault() + + if (rightPanelRef.current) { + rightPanelRef.current.style.display = 'none' + } + + if (logContainerRef.current) { + const scrollEl = logContainerRef.current.querySelector('.log-viewer-scroll') + + if (scrollEl) { + scrollPositionRef.current = scrollEl.scrollTop + } + } + + if (leftPanelRef.current) { + startWidthRef.current = leftPanelRef.current.offsetWidth + } + + requestAnimationFrame(() => { + setIsDragging(true) + }) + + document.addEventListener('mousemove', handleMouseMove) + document.addEventListener('mouseup', handleMouseUp) + document.body.style.cursor = 'col-resize' + document.body.style.userSelect = 'none' + }, + [handleMouseMove, handleMouseUp] + ) + + useEffect(() => { + if (!isDragging && logContainerRef.current && scrollPositionRef.current > 0) { + requestAnimationFrame(() => { + const scrollEl = logContainerRef.current?.querySelector('.log-viewer-scroll') + + if (scrollEl) { + scrollEl.scrollTop = scrollPositionRef.current + } + }) + } + }, [isDragging]) + + return { + containerRef, + leftPanelRef, + rightPanelRef, + logContainerRef, + leftWidth, + isDragging, + logViewerHeight, + handleMouseDown, + defaultLeftWidthPercent: DEFAULT_LEFT_WIDTH_PERCENT + } +} diff --git a/moon/apps/web/components/ClView/components/Checks/index.tsx b/moon/apps/web/components/ClView/components/Checks/index.tsx index 8b4af95fc..7d9520d71 100644 --- a/moon/apps/web/components/ClView/components/Checks/index.tsx +++ b/moon/apps/web/components/ClView/components/Checks/index.tsx @@ -1,298 +1,49 @@ -import { memo, useCallback, useEffect, useRef, useState } from 'react' -import { format } from 'date-fns' -import { useAtom } from 'jotai' +import { memo, useState } from 'react' import { LoadingSpinner } from '@gitmono/ui' -import { buildIdAtom } from '@/components/Issues/utils/store' import { useGetClTask } from '@/hooks/SSE/useGetClTask' -import { fetchHTTPLog } from '@/hooks/SSE/useGetHTTPLog' -import { useTaskSSE } from '../../hook/useSSM' -import { BuildDTO, getQueuedBuildIds, statusMapAtom, TaskInfoDTO } from './cpns/store' +import { getQueuedBuildIds, TaskInfoDTO } from './cpns/store' import { TreeRoot } from './cpns/Task' -import { LogViewer } from './LogViewer' - -type LogStatus = 'idle' | 'loading' | 'success' | 'empty' | 'error' - -const MIN_LEFT_WIDTH = 200 -const MAX_LEFT_WIDTH_PERCENT = 0.7 -const DEFAULT_LEFT_WIDTH_PERCENT = 0.2 +import { useBuildSelection } from './hooks/useBuildSelection' +import { useLeftPanelScroll } from './hooks/useLeftPanelScroll' +import { useLogCache } from './hooks/useLogCache' +import { useMountedLogPanels } from './hooks/useMountedLogPanels' +import { useResizablePanels } from './hooks/useResizablePanels' +import { CachedLogPanel } from './LogViewer' + +const LogLoadingState = ({ label = 'Loading logs...' }: { label?: string }) => ( +
+
+ + {label} +
+
+) const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: string }) => { - const [buildid, setBuildId] = useAtom(buildIdAtom) - const { eventSourcesRef, setEventSource, closeEventSource, logsMap, setLogsMap } = useTaskSSE() - const [statusMap, _setStatusMap] = useAtom(statusMapAtom) const { data: tasks, isError: isTasksError, isLoading: isTasksLoading } = useGetClTask(cl) - const [logStatus, setLogStatus] = useState>({}) - const [selectedTaskId, setSelectedTaskId] = useState(null) - const [isDropdownOpen, setIsDropdownOpen] = useState(false) - const [hoveredTaskId, setHoveredTaskId] = useState(null) - const [tooltipPosition, setTooltipPosition] = useState<{ top: number; left: number } | null>(null) - - // Resizable panel state - const containerRef = useRef(null) - const leftPanelRef = useRef(null) - const rightPanelRef = useRef(null) - const [leftWidth, setLeftWidth] = useState(null) - const [isDragging, setIsDragging] = useState(false) - const scrollPositionRef = useRef(0) - const logContainerRef = useRef(null) - const startWidthRef = useRef(0) - // Explicit pixel height for LogViewer (avoids layout jump while the panel mounts). - const [logViewerHeight, setLogViewerHeight] = useState(0) - - // Initialize left width based on container width - useEffect(() => { - if (containerRef.current && leftWidth === null) { - setLeftWidth(containerRef.current.offsetWidth * DEFAULT_LEFT_WIDTH_PERCENT) - } - }, [leftWidth]) - - // Track the container height and pass it to LogViewer as a fixed pixel value. - useEffect(() => { - const el = containerRef.current - - if (!el) return - - const update = () => setLogViewerHeight(el.clientHeight) - - update() - - const observer = new ResizeObserver(update) - - observer.observe(el) - - return () => observer.disconnect() - }, []) - - const handleMouseMove = useCallback((e: MouseEvent) => { - if (!containerRef.current || !leftPanelRef.current) return - - // Directly manipulate DOM without triggering React re-render - const containerRect = containerRef.current.getBoundingClientRect() - const newLeftWidth = e.clientX - containerRect.left - const maxWidth = containerRect.width * MAX_LEFT_WIDTH_PERCENT - const clampedWidth = Math.max(MIN_LEFT_WIDTH, Math.min(newLeftWidth, maxWidth)) - - // Update DOM directly for smooth dragging - leftPanelRef.current.style.width = `${clampedWidth}px` - }, []) - - const handleMouseUp = useCallback(() => { - // Remove event listeners - document.removeEventListener('mousemove', handleMouseMove) - document.removeEventListener('mouseup', handleMouseUp) - document.body.style.cursor = '' - document.body.style.userSelect = '' - - // Show right panel immediately using DOM - if (rightPanelRef.current) { - rightPanelRef.current.style.display = 'block' - } - - // Update React state only once when dragging ends - if (leftPanelRef.current) { - const finalWidth = leftPanelRef.current.offsetWidth - - setLeftWidth(finalWidth) - } - setIsDragging(false) - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []) - - const handleMouseDown = useCallback( - (e: React.MouseEvent) => { - e.preventDefault() - - // Immediately hide right panel using DOM (no re-render) - if (rightPanelRef.current) { - rightPanelRef.current.style.display = 'none' - } - - // Save scroll position - if (logContainerRef.current) { - const scrollEl = logContainerRef.current.querySelector('.log-viewer-scroll') - - if (scrollEl) { - scrollPositionRef.current = scrollEl.scrollTop - } - } - - // Save current width - if (leftPanelRef.current) { - startWidthRef.current = leftPanelRef.current.offsetWidth - } - - // Update state asynchronously (won't block dragging) - requestAnimationFrame(() => { - setIsDragging(true) - }) - - // Add event listeners immediately - document.addEventListener('mousemove', handleMouseMove) - document.addEventListener('mouseup', handleMouseUp) - document.body.style.cursor = 'col-resize' - document.body.style.userSelect = 'none' - }, - [handleMouseMove, handleMouseUp] - ) - - useEffect(() => { - if (!isDragging) { - // Restore scroll position after dragging ends - if (logContainerRef.current && scrollPositionRef.current > 0) { - // Use requestAnimationFrame to ensure DOM is updated - requestAnimationFrame(() => { - const scrollEl = logContainerRef.current?.querySelector('.log-viewer-scroll') - - if (scrollEl) { - scrollEl.scrollTop = scrollPositionRef.current - } - }) - } - } - }, [isDragging]) - - // Reset scroll position when buildid changes - useEffect(() => { - // Clear saved scroll position when switching to a different build - scrollPositionRef.current = 0 - - // Reset scroll to top for new build - if (logContainerRef.current) { - requestAnimationFrame(() => { - const scrollEl = logContainerRef.current?.querySelector('.log-viewer-scroll') - - if (scrollEl) { - scrollEl.scrollTop = 0 - } - }) - } - }, [buildid]) - - useEffect(() => { - if (!tasks || tasks.length === 0) return - - const builds = tasks.flatMap((task: TaskInfoDTO) => - (task.build_list ?? []).map((build: BuildDTO) => ({ - build_id: build.id, - status: build.status - })) - ) - - const validBuilds = builds.filter((b) => Boolean(b.build_id)) - - if (validBuilds.length === 0) return - - // Queued builds (waiting for a worker) have no logs yet; don't fetch them. - const queuedIds = getQueuedBuildIds(tasks) - const buildsToFetch = validBuilds.filter((b) => !queuedIds.has(b.build_id)) - const buildingIds = new Set(buildsToFetch.filter((b) => b.status === 'Building').map((b) => b.build_id)) - - buildingIds.forEach((buildId) => { - setEventSource(buildId) - }) - - Object.keys(eventSourcesRef.current).forEach((buildId) => { - if (!buildingIds.has(buildId)) { - closeEventSource(buildId) - } - }) - - buildsToFetch.forEach((b) => { - setLogStatus((prev) => ({ ...prev, [b.build_id]: 'loading' })) - }) - - const fetchLogs = async () => { - const logsResult = await Promise.allSettled( - buildsToFetch.map(async ({ build_id }) => { - try { - const res = await fetchHTTPLog(build_id) - - return { id: build_id, res, error: null } - } catch (error) { - return { id: build_id, res: null, error } - } - }) - ) - - const newLogStatus: Record = {} - const fetchedLogsMap: Record = {} - - logsResult.forEach((item) => { - if (item.status === 'fulfilled' && item.value) { - const { id, res, error } = item.value - - if (error) { - // fetchHTTPLog threw an error - newLogStatus[id] = 'error' - } else if (!res || !res.data) { - // Response is null/undefined - newLogStatus[id] = 'empty' - } else if (Array.isArray(res.data) && res.data.length === 0) { - // Response data is empty array - newLogStatus[id] = 'empty' - } else if (res.len === 0) { - // Response len is 0 - newLogStatus[id] = 'empty' - } else { - // Success case - const logText = Array.isArray(res.data) ? res.data.join('\n') : String(res.data || '') - - newLogStatus[id] = 'success' - fetchedLogsMap[id] = logText - } - } else { - // Promise.allSettled rejected (shouldn't happen with try-catch, but defensive) - const id = buildsToFetch[logsResult.indexOf(item)]?.build_id - - if (id) { - newLogStatus[id] = 'error' - } - } - }) - - setLogsMap((prev) => { - const next = { ...prev } - - Object.entries(fetchedLogsMap).forEach(([id, fetchedLog]) => { - const currentLog = next[id] ?? '' - - // HTTP reads can lag behind SSE appends while a build is running; never - // replace a longer live log with an older persisted snapshot. - next[id] = currentLog.length > fetchedLog.length ? currentLog : fetchedLog - }) - - return next - }) - setLogStatus((prev) => ({ ...prev, ...newLogStatus })) - } - - if (buildsToFetch.length > 0) { - fetchLogs() - } - - if (!buildid && validBuilds.length > 0) { - setBuildId(validBuilds[0].build_id) - } - - return () => { - statusMap.clear() - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [tasks]) - - // Initialize selected task - useEffect(() => { - const validTasks = tasks?.filter((t) => t.build_list && t.build_list.length > 0) || [] + const { buildId, selectBuild, selectedTaskId, selectTask } = useBuildSelection(cl, tasks) + const { logsMap, logsAvailableIds, currentLogStatus, isQueued, retryLog } = useLogCache(cl, buildId, tasks) + const mountedPanelIds = useMountedLogPanels(buildId, logsMap) + + const { + containerRef, + leftPanelRef, + rightPanelRef, + logContainerRef, + leftWidth, + isDragging, + logViewerHeight, + handleMouseDown, + defaultLeftWidthPercent + } = useResizablePanels() + + useLeftPanelScroll(cl, buildId, leftPanelRef) - if (validTasks.length > 0 && !selectedTaskId) { - setSelectedTaskId(validTasks[0].task_id) - } - }, [tasks, selectedTaskId]) + const [isDropdownOpen, setIsDropdownOpen] = useState(false) - // Helper functions for dropdown const getTaskFileName = (task: TaskInfoDTO) => { if (!task.targets || task.targets.length === 0) return task.task_name || 'Unnamed Task' @@ -309,14 +60,6 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri return parts[parts.length - 1] || 'Unnamed Task' } - const formatDateTime = (isoDate: string): string => { - try { - return format(new Date(isoDate), 'yyyy-MM-dd HH:mm') - } catch { - return isoDate - } - } - const getTaskStatus = (task: TaskInfoDTO) => { if (!task.targets || task.targets.length === 0) return null @@ -349,7 +92,6 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri return null } - // Handle tasks loading state if (isTasksLoading) { return (
@@ -368,7 +110,6 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri ) } - // Handle tasks error or empty state if (isTasksError) { return (
@@ -388,7 +129,7 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri const selectedTask = validTasks.find((t) => t.task_id === selectedTaskId) const tasksToDisplay = selectedTask ? [selectedTask] : validTasks - if (!isTasksLoading && (!tasks || tasks.length === 0 || validTasks.length === 0)) { + if (!tasks || tasks.length === 0 || validTasks.length === 0) { return (
@@ -403,11 +144,10 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri ) } - const queuedBuildIds = getQueuedBuildIds(tasks ?? []) + const queuedBuildIds = getQueuedBuildIds(tasks) - // Render log viewer with status handling const renderLogContent = () => { - if (!buildid) { + if (!buildId) { return (
Select a build to view logs @@ -415,8 +155,7 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri ) } - // Queued build: not started yet, so there are no logs to show. - if (queuedBuildIds.has(buildid)) { + if (queuedBuildIds.has(buildId) || isQueued) { return (
@@ -427,42 +166,46 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri ) } - const status = logStatus[buildid] - - if (logsMap[buildid]) { - return ( -
- 0 ? logViewerHeight : 'auto'} /> -
- ) - } + const viewerHeight = logViewerHeight > 0 ? logViewerHeight : 'auto' + const hasCachedCurrentBuild = Boolean(logsMap[buildId]) + const isLoadingCurrentBuild = + !hasCachedCurrentBuild && (currentLogStatus === 'loading' || currentLogStatus === 'idle') - // If status is undefined or idle, user needs to select a build - if (!status || status === 'idle') { + if (mountedPanelIds.length > 0) { return ( -
- Select a build to view logs +
+ {mountedPanelIds.map((id) => ( + + ))} + {isLoadingCurrentBuild ? ( +
+ +
+ ) : null}
) } - if (status === 'loading') { - return ( -
- Loading logs... -
- ) + if (isLoadingCurrentBuild || currentLogStatus === 'loading') { + return } - if (status === 'error') { + if (currentLogStatus === 'error') { return ( -
+
Failed to fetch logs +
) } - if (status === 'empty') { + if (currentLogStatus === 'empty') { return (
No logs available @@ -470,7 +213,6 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri ) } - // Fallback: show select prompt return (
Select a build to view logs @@ -479,218 +221,131 @@ const Checks = ({ cl, path, prName }: { cl: string; path?: string; prName?: stri } return ( - <> -
- {/* Header with Task Selector */} -
-
-

[] tasks status interface

- - {/* Task Selector Dropdown - Only show if multiple tasks */} - {validTasks.length > 1 && ( -
- - - {/* Dropdown Menu */} - {isDropdownOpen && ( - <> - {/* Backdrop */} -
setIsDropdownOpen(false)} /> - - {/* Dropdown List */} -
- {validTasks.map((task) => { - const fileName = getTaskFileName(task) - const status = getTaskStatus(task) - const isSelected = task.task_id === selectedTaskId - const targetsCount = task.targets?.length || 0 - - return ( -
{ - e.stopPropagation() - setSelectedTaskId(task.task_id) - setIsDropdownOpen(false) - setHoveredTaskId(null) - }} - onMouseEnter={(e) => { - setHoveredTaskId(task.task_id) - - const rect = e.currentTarget.getBoundingClientRect() - - setTooltipPosition({ - top: rect.top, - left: rect.right + 8 - }) - }} - onMouseLeave={() => { - setHoveredTaskId(null) - setTooltipPosition(null) - }} - className={`flex cursor-pointer items-center justify-between px-3 py-2.5 transition-colors ${ - isSelected ? 'bg-blue-50 dark:bg-blue-900/30' : 'hover:bg-gray-100 dark:hover:bg-gray-700' - }`} - > -
-
- - {fileName} - - {status && ( - • {status.status} - )} -
- - {targetsCount} {targetsCount === 1 ? 'target' : 'targets'} + )} + + ) : ( + Select a task... + )} +
+ + + + + + {isDropdownOpen && ( + <> +
setIsDropdownOpen(false)} /> +
+ {validTasks.map((task) => { + const fileName = getTaskFileName(task) + const status = getTaskStatus(task) + const isSelected = task.task_id === selectedTaskId + const targetsCount = task.targets?.length || 0 + + return ( +
{ + e.stopPropagation() + selectTask(task.task_id) + setIsDropdownOpen(false) + }} + className={`flex cursor-pointer items-center justify-between px-3 py-2.5 transition-colors ${ + isSelected ? 'bg-blue-50 dark:bg-blue-900/30' : 'hover:bg-gray-100 dark:hover:bg-gray-700' + }`} + > +
+
+ + {fileName} + {status && ( + • {status.status} + )}
- {isSelected && ( - - - - )} + + {targetsCount} {targetsCount === 1 ? 'target' : 'targets'} +
- ) - })} -
- - {/* Tooltip for hovered task */} - {hoveredTaskId && tooltipPosition && ( -
- {(() => { - const task = validTasks.find((t) => t.task_id === hoveredTaskId) - - if (!task) return null - - const fileName = getTaskFileName(task) - const status = getTaskStatus(task) - const targetsCount = task.targets?.length || 0 - - return ( -
-
- {fileName} - {status && ( - - {status.status} - - )} -
-
-
Task ID: {task.task_id}
-
Created: {formatDateTime(task.created_at)}
-
Targets: {targetsCount}
-
-
- ) - })()} -
- )} - - )} -
- )} -
+
+ ) + })} +
+ + )} +
+ )}
+
-
-
- -
- {/* Resizer handle */} -
+
+ -
- {renderLogContent()} -
- {isDragging && ( -
- Resizing... -
- )}
+
+
+ {renderLogContent()} +
+ {isDragging && ( +
+ Resizing... +
+ )}
- +
) } diff --git a/moon/apps/web/components/ClView/hook/useSSM.ts b/moon/apps/web/components/ClView/hook/useSSM.ts index 2873a9853..9d7d81594 100644 --- a/moon/apps/web/components/ClView/hook/useSSM.ts +++ b/moon/apps/web/components/ClView/hook/useSSM.ts @@ -3,7 +3,7 @@ import { useAtom } from 'jotai' import { orionApiClient } from '@/utils/queryClient' -import { loadingAtom, logsAtom, statusAtom } from '../components/Checks/cpns/store' +import { loadingAtom, logsAtomFamily, statusAtom } from '../components/Checks/cpns/store' /** * Get SSE URL for task output streaming @@ -50,12 +50,12 @@ export const useSSM = () => { // together at most once per interval. const LOG_FLUSH_INTERVAL_MS = 150 -export const useTaskSSE = () => { +export const useTaskSSE = (cl: string) => { const eventSourcesRef = useRef>({}) // Per-task "force flush remaining buffer + cancel pending timer". Used when a // stream ends or the component unmounts so we never drop the trailing lines. const flushersRef = useRef void>>({}) - const [logsMap, setLogsMap] = useAtom(logsAtom) + const [logsMap, setLogsMap] = useAtom(logsAtomFamily(cl)) const [_, setLoading] = useAtom(loadingAtom) const [_status, setStatus] = useAtom(statusAtom) @@ -163,7 +163,6 @@ export const useTaskSSE = () => { eventSourcesRef.current = {} flushersRef.current = {} setLoading(false) - setLogsMap({}) } // eslint-disable-next-line react-hooks/exhaustive-deps }, []) diff --git a/moon/apps/web/components/Issues/utils/store.tsx b/moon/apps/web/components/Issues/utils/store.tsx index a0ed88336..0d27fc16d 100644 --- a/moon/apps/web/components/Issues/utils/store.tsx +++ b/moon/apps/web/components/Issues/utils/store.tsx @@ -25,5 +25,3 @@ export const FALSE_EDIT_VAL = -1 export const editIdAtom = atom(0) export const refreshAtom = atom(0) - -export const buildIdAtom = atom('') diff --git a/moon/apps/web/hooks/SSE/useGetClTask.ts b/moon/apps/web/hooks/SSE/useGetClTask.ts index f33f013b0..67e001760 100644 --- a/moon/apps/web/hooks/SSE/useGetClTask.ts +++ b/moon/apps/web/hooks/SSE/useGetClTask.ts @@ -32,8 +32,11 @@ const toTargetState = (state: string): TargetState => { } export function useGetClTask(cl: string, params?: RequestParams) { + // NOTE(P3): Each task currently triggers 2 extra API calls (buildEvents + targets). + // A future Orion `GET /cl/{link}/tasks-full` aggregate endpoint would remove this N+1. return useQuery({ queryKey: [...orionApiClient.task.getTaskByClV2().requestKey(cl), params], + enabled: Boolean(cl), // Poll while any task is still queued (Uninitialized) or actively building, // so the tree transitions automatically once a worker picks it up, the build // finishes, or the server times it out (-> Interrupted). Stops when idle. diff --git a/moon/apps/web/hooks/SSE/useGetHTTPLog.ts b/moon/apps/web/hooks/SSE/useGetHTTPLog.ts index 9239a49ac..52c7a7fca 100644 --- a/moon/apps/web/hooks/SSE/useGetHTTPLog.ts +++ b/moon/apps/web/hooks/SSE/useGetHTTPLog.ts @@ -2,18 +2,34 @@ import { useQuery } from '@tanstack/react-query' import { GetBuildsLogsV2Data, RequestParams } from '@gitmono/types/generated' +import { BuildStatus } from '@/components/ClView/components/Checks/cpns/store' +import { TERMINAL_BUILD_STATUSES } from '@/components/ClView/components/Checks/hooks/logUtils' import { orionApiClient } from '@/utils/queryClient' -export function useGetHTTPLog(buildId: string, params?: RequestParams) { - const request = orionApiClient.builds.getBuildsLogsV2() +const request = orionApiClient.builds.getBuildsLogsV2() - return useQuery({ - queryKey: [...request.requestKey(buildId), params], +export function getBuildLogQueryKey(buildId: string, params?: RequestParams) { + return [...request.requestKey(buildId), params] as const +} + +export function getBuildLogQueryOptions(buildId: string, isTerminal: boolean, params?: RequestParams) { + return { + queryKey: getBuildLogQueryKey(buildId, params), queryFn: () => request.request(buildId, params), - enabled: Boolean(buildId) - }) + enabled: Boolean(buildId), + staleTime: isTerminal ? Number.POSITIVE_INFINITY : 0, + gcTime: 1000 * 60 * 30 + } +} + +export function useGetHTTPLog(buildId: string, isTerminal: boolean, params?: RequestParams) { + return useQuery(getBuildLogQueryOptions(buildId, isTerminal, params)) } export const fetchHTTPLog = (buildId: string, params?: RequestParams) => { - return orionApiClient.builds.getBuildsLogsV2().request(buildId, params) + return request.request(buildId, params) +} + +export function isTerminalBuildStatus(status: BuildStatus | undefined): boolean { + return Boolean(status && TERMINAL_BUILD_STATUSES.has(status)) } diff --git a/moon/apps/web/next.config.js b/moon/apps/web/next.config.js index b7bc672ae..c5f260092 100644 --- a/moon/apps/web/next.config.js +++ b/moon/apps/web/next.config.js @@ -49,7 +49,7 @@ const cspResourcesByDirective = { // Local demo environments 'http://*.gitmono.local:8004', 'http://*.gitmono.local:8000', - 'http://*.gitmono.local:8080', + 'http://*.gitmono.local:18080', // Local development environments 'http://*.gitmono.test:3001', 'http://*.gitmono.test:8000', diff --git a/moon/apps/web/pages/[org]/cl/[link]/index.tsx b/moon/apps/web/pages/[org]/cl/[link]/index.tsx index f76fc0a5f..9cf473972 100644 --- a/moon/apps/web/pages/[org]/cl/[link]/index.tsx +++ b/moon/apps/web/pages/[org]/cl/[link]/index.tsx @@ -238,7 +238,21 @@ const CLDetailPage: PageWithLayout = () => { }) } - const [tab] = useAtom(tabAtom) + const [tab, setTab] = useAtom(tabAtom) + const openedBuildTabRef = useRef(false) + + // Open Checks tab when landing with ?tab=check or ?build= shared link. + useEffect(() => { + if (!router.isReady || openedBuildTabRef.current) return + + const build = router.query.build + const tab = router.query.tab + + if (tab === 'check' || (typeof build === 'string' && build)) { + setTab('check') + openedBuildTabRef.current = true + } + }, [router.isReady, router.query.build, router.query.tab, setTab]) const renderStatusPill = () => { if (!clDetail?.status) return null diff --git a/orion-scheduler/FUSE_MOUNT_ISSUES.md b/orion-scheduler/FUSE_MOUNT_ISSUES.md deleted file mode 100644 index ca2e8ef62..000000000 --- a/orion-scheduler/FUSE_MOUNT_ISSUES.md +++ /dev/null @@ -1,127 +0,0 @@ -# Antares FUSE 挂载问题分析 - -本文档记录在 `orion-scheduler` 驱动的 buck2 构建过程中，构建工作目录所在的 antares/scorpio FUSE 挂载（`/data/scorpio/antares/mnt/-N/`）出现的稳定性问题。这些问题不会每次都直接中断构建，但会放大失败、干扰诊断，长构建下尤其明显。 - -> 首次记录来源：线上测试服务 `git.xuanwu.openatom.cn`（k3s 集群 `buck2hub` 命名空间），CL `UYXIYYNJ` 第三次构建 `019ec954-e2b3-7520-b780-a14382f6ed24`（2026-06-15 03:30 → 04:07，exit_code=1）。 - ---- - -## 问题汇总 - - -| 现象 | errno | 出现次数（该次构建） | 影响 | 优先级 | -| --------------------------- | --------------------- | ---------- | --------------------- | --- | -| 刚写出的文件立即 `metadata()` 报不存在 | `os error 2` (ENOENT) | 45 | 本地 action 失败、可能误判构建失败 | 高 | -| 收尾 flush `event.jsonl` 句柄失效 | `os error 9` (EBADF) | 2 | buck2 无法写自身事件日志，收尾报错 | 中 | -| `Action failed` 总数 | — | 47 | 干扰诊断、放大失败面 | 高 | - - -> 注意：本次构建的**决定性失败**是工具链问题（worker 缺少 `jlink`，见文末「与构建失败的关系」），FUSE 问题是次要但需要单独跟踪的稳定性隐患。 - ---- - -## 1. 写后立即读取返回 ENOENT（一致性问题） - -### 问题现象 - -构建过程中大量本地 action（如 `write linker_wrapper.sh`）刚写出文件，buck2 紧接着对同一路径做 `metadata()`（设置可执行权限）时却报“文件不存在”： - -``` -Action failed: root//third-party/rust/crates/thiserror/2.0.18:thiserror-build-script-build (write linker_wrapper.sh) -`write_file` setting executable `/data/scorpio/antares/mnt/019ec954-.../.../linker_wrapper.sh`: - metadata(/data/scorpio/antares/mnt/019ec954-.../.../linker_wrapper.sh): No such file or directory (os error 2) -``` - -同类还有 `remove_file(...): No such file or directory (os error 2)`。 - -### 根因分析 - -`/data/scorpio/antares/mnt/...` 是 antares/scorpio 提供的 FUSE 工作目录。错误模式是典型的 **“写后读不一致”（read-after-write inconsistency）**： - -- buck2 在挂载点 `write_file` 成功后，立刻 `stat`/`metadata` 同一路径； -- FUSE 层因元数据缓存、异步落盘或目录项尚未可见，返回 ENOENT； -- 高并发（buck2 同时跑成百上千个 action）下概率显著上升。 - -### 影响 - -- 这些是本地 `write_file` action，buck2 会记为 `Action failed`； -- 即使部分被重试，也会拖慢构建、污染日志，难以与“真实的构建错误”区分。 - -### 建议排查 / 修复方向 - -- 检查 antares/scorpio FUSE 实现的元数据缓存策略，确认 `write` 后 `lookup`/`getattr` 的一致性保证；必要时关闭/缩短 attr/entry cache 的 TTL。 -- 确认是否存在异步写入（writeback）导致目录项延迟可见；写关键控制文件时考虑同步语义。 -- 评估并发压力下的表现，复现“写后立即 stat”的竞态。 - ---- - -## 2. 收尾 flush `event.jsonl` 报 Bad file descriptor - -### 问题现象 - -构建末尾 buck2 flush 自身事件日志失败，随后整体 `BUILD FAILED`： - -``` -BUILD FAILED - WARN buck2_event_log::writer: Failed to flush log file at - `/data/scorpio/antares/mnt/019ec954-.../event.jsonl`: Bad file descriptor (os error 9) -Command failed: Error flushing log file at /data/scorpio/antares/mnt/019ec954-.../event.jsonl - -Caused by: - Bad file descriptor (os error 9) -``` - -### 根因分析 - -`Bad file descriptor (EBADF)` 表示文件句柄在 flush 时已失效。在 FUSE 挂载上通常意味着： - -- 挂载在构建收尾阶段被卸载 / 重新挂载，或连接中断（transport endpoint 变化）； -- 句柄被底层提前关闭，而上层仍持有并写入。 - -该 `event.jsonl` 是 buck2 自己的事件日志，位于挂载工作目录内，因此挂载的句柄稳定性直接影响 buck2 收尾。 - -### 影响 - -- 即使所有目标都已构建，buck2 也会因无法落盘事件日志而以失败收尾； -- 句柄失效往往与挂载生命周期管理（卸载时机）相关。 - -### 建议排查 / 修复方向 - -- 检查 antares 挂载的生命周期：是否在构建进程仍在写入时就触发了卸载 / 清理。 -- 确认 `fusermount` 卸载时机晚于构建进程完全退出（参考 `orion` 中 antares 卸载相关逻辑）。 -- 评估把 buck2 的 event-log 输出目录放到挂载之外的本地稳定路径，避免受挂载卸载影响。 - ---- - -## 与构建失败的关系（区分主因） - -需要明确区分：CL `UYXIYYNJ` 该次构建的**决定性失败原因不是 FUSE**，而是 worker 的 Java 17 运行时缺少 `jlink`，导致 `create_jdk_system_image` 工具链 action 失败： - -``` -Action failed: root//project/buck2_test/toolchains:jdk_system_image (create_jdk_system_image) -FileNotFoundError: [Errno 2] No such file or directory: - PosixPath('/usr/local/java-runtime/impl/17/bin/jlink') -``` - -本文档聚焦的是**次要但需独立跟踪**的 FUSE 挂载稳定性问题：它会产生大量噪声 action 失败、并在收尾阶段以 EBADF 破坏事件日志，从而放大整体失败面、增加定位成本。即便修复了 `jlink`，FUSE 问题仍可能在长构建中独立导致失败，建议单独跟进。 - ---- - -## 复现与取证 - -- 构建本地日志（orion-server `mix` 模式）：`///.log` - - 示例：`/tmp/megadir/buck2ctl/019ebaad-2fa4-7310-a027-13a1616483af/019ec954-e2b3-7520-b780-a14382f6ed24.log` -- 关键字检索： - -```bash -grep -niE "bad file descriptor|os error 9|os error 2|fuse|transport endpoint|input/output error|scorpio|antares|Action failed|BUILD FAILED" -``` - -- 统计错误规模： - -```bash -grep -c "os error 2" # ENOENT 写后读不一致 -grep -c "os error 9" # EBADF 句柄失效 -grep -c "Action failed" -``` - diff --git a/orion-scheduler/README.md b/orion-scheduler/README.md index 0804600dc..965a102f3 100644 --- a/orion-scheduler/README.md +++ b/orion-scheduler/README.md @@ -130,6 +130,8 @@ sudo modprobe nbd max_part=8 sudo bash scripts/build-custom-image.sh ``` +> 镜像发布到**调用者**的 qlean 目录（`sudo` 从 `orion` 运行时即 `/home/orion/.local/share/qlean/images/`，不是 `/root/...`）。`orion-scheduler` 与 `cargo run` 也读该路径。可用 `OUTPUT_DIR=...` 覆盖。 + > 基础镜像下载源默认指向 USTC 镜像（`https://mirrors.ustc.edu.cn/debian-cdimage/cloud/trixie/latest`）。如需更换为官方源或其他镜像，通过 `BASE_MIRROR_URL` 覆盖（注意 `sudo` 需加 `-E` 传递环境变量）： > > ```bash diff --git a/orion-scheduler/TARGET_DISCOVERY_SCOPE.md b/orion-scheduler/TARGET_DISCOVERY_SCOPE.md deleted file mode 100644 index 514ca38bd..000000000 --- a/orion-scheduler/TARGET_DISCOVERY_SCOPE.md +++ /dev/null @@ -1,274 +0,0 @@ -# Target Discovery 扫描范围过大问题分析 - -本文档记录 Orion worker 在做 Buck2 增量构建时，**target discovery（目标发现）与影响传播范围过大**，导致一个纯 Rust 改动（如 `rk8s`）会拉起与之无关的 JVM / Android toolchain，并因 worker 镜像缺少这些环境而构建失败的问题。 - -涉及代码主要在 `orion` crate（worker 端），而非 `orion-scheduler` 本身；本文放在 `orion-scheduler/` 下作为问题归档与修改方案说明。 - -> 重要澄清：`project/buck2_test/toolchains:jdk_system_image`、`__android_sdk_tools__` 等 **不代表 `buck2_test` 业务代码依赖 JVM/Android**。它们只是 JVM/Android toolchain / platform helper target 的**定义位置**恰好在 `project/buck2_test/toolchains` 这个包下。这些 helper 被拉进来，是因为 Orion 统一使用了**全语言默认平台** `prelude//platforms:default`，并且影响传播不过滤 toolchain/platform 节点——而不是 `rk8s` 或 `buck2_test` 真的需要 Java/Android。 - ---- - -## 1. 问题现象 - -CL（例如 `#UYXIYYNJ`）只新增了 `rk8s` 目录下的 Rust 代码、`rk8s/third-party/rust/crates/**` 的三方依赖 BUCK，以及 `rk8s/toolchains` 的 Buck 定义，**与 `project/buck2_test` 无关**。但 worker 构建时却出现： - -``` -Action failed: root//project/buck2_test/toolchains:jdk_system_image (create_jdk_system_image) - FileNotFoundError: [Errno 2] No such file or directory: - PosixPath('/usr/local/java-runtime/impl/17/bin/jlink') -``` - -以及后续： - -``` -FileNotFoundError: [Errno 2] No such file or directory: - buck-out/.../__android_sdk_tools__/core-for-system-modules.jar -``` - -也就是说：**改 `rk8s`（纯 Rust）却触发了定义在 `project/buck2_test/toolchains` 下的 JDK / Android toolchain helper 构建**。 - -几个关键事实： - -- `.buckconfig` 里只有 4 个 cell：`root` / `prelude` / `toolchains` / `none`。`rk8s` 和 `project/buck2_test` 都在 **`root` cell 下**，是目录而非独立 cell。 - -```1:5:.buckconfig -[cells] - root = . - prelude = prelude - toolchains = toolchains - none = none -``` - -- 因此“按 cell 收窄”无效：只要扫 `root//...` 就会把 `project/buck2_test` 一起带进来。 -- 本次 worker 日志里的改动列表全部位于 `rk8s/**`，**没有看到 `.buckconfig` 变更**。所以本次更可能是“默认平台 + 不过滤 toolchain 的影响传播”导致，而非 `.buckconfig` 触发的 select-all（见 2.3，仍是潜在放大源之一）。 - ---- - -## 2. 根因分析 - -worker 端的目标发现流程在 [orion/src/buck_controller.rs](orion/src/buck_controller.rs)：`get_build_targets()` 对 **base（改动前）** 和 **diff（改动后）** 两份快照分别跑 `buck2 targets` 构建完整图，再由 `collect_impacted_targets()` 计算受影响集合，最后 `buck2 build` 这些 target。 - -放大效应来自下面几条**叠加**的路径。 - -### 2.1 发现阶段：对所有 cell 跑 `//...` - -`get_repo_targets()` 用 `get_all_cell_patterns()` 拼出所有 cell 的 `//...` 模式并整体查询： - -```348:356:orion/src/buck_controller.rs - // If cells info is provided, query all cells; otherwise just query root cell - if let Some(cells_info) = cells { - let cell_patterns = cells_info.get_all_cell_patterns(repo_path); - tracing::debug!("Querying targets for cells: {:?}", cell_patterns); - command.args(&cell_patterns); - } else { - // Default: only query root cell - command.arg("//..."); - } -``` - -`get_all_cell_patterns()` 会返回 `root//...`、`toolchains//...`（排除 `prelude` / `none` 与不存在的目录）： - -```278:304:orion/buck/cells.rs - pub fn get_all_cell_patterns(&self, project_root: &Path) -> Vec { - self.cells - .iter() - .filter(|(cell_name, cell_data)| { - let cell_str = cell_name.as_str(); - - // Exclude known special/placeholder cells - if cell_str == "prelude" || cell_str == "none" { - return false; - } - - // Check if the cell directory actually exists - let cell_path = project_root.join(cell_data.path.as_str()); - if !cell_path.exists() { - return false; - } - - true - }) - .map(|(cell_name, _)| format!("{}//...", cell_name.as_str())) - .collect() - } -``` - -后果：查询 `root//...` 会**解析整个 `root` cell 的所有包**（含 `project/buck2_test/toolchains`），查询 `toolchains//...` 会**初始化整个 `toolchains` cell**，即便本次只改了 `rk8s`。 - -### 2.2 关键根因：固定全语言默认平台 + 不过滤 toolchain 的影响传播 - -这是“为什么会真正去 build JVM/Android helper”的核心。 - -1. 发现与构建都钉死在 `prelude//platforms:default` 这一**全语言默认平台**： - -```193:207:orion/buck/run.rs -pub fn targets_arguments() -> &'static [&'static str] { - &[ - "targets", - "--target-platforms", - "prelude//platforms:default", - "--streaming", - "--keep-going", - ... - ] -} -``` - -```1203:1217:orion/src/buck_controller.rs - .arg("build") - .args(["--event-log", EVENT_LOG_FILE]) - .args(&targets) - .arg("--target-platforms") - .arg("prelude//platforms:default") - // Avoid failing the whole build when a target is explicitly incompatible - // with the selected platform (e.g., macOS-only crates on Linux builders). - .arg("--skip-incompatible-targets") -``` - -该默认平台会把 Java/Android/CXX 等各语言 toolchain 一并纳入 configuration / toolchain resolution，相关 helper target 就定义在 `project/buck2_test/toolchains`。 - -2. 影响传播 `recursive_target_changes` 的 `follow_rule_type` 传的是 `|_| true`，即**沿所有 rule type 的依赖边无差别传播**，不区分业务 target、toolchain target、platform helper： - -```467:469:orion/src/buck_controller.rs -fn collect_impacted_targets(base: &Targets, diff: &Targets, changes: &Changes) -> Vec { - let immediate = diff::immediate_target_changes(base, diff, changes, false); - let recursive = diff::recursive_target_changes(diff, changes, &immediate, None, |_| true); -``` - -3. 叠加效果：`rk8s` 新增了大量三方 Rust crate（其中包含 `jni`、`jni-sys`、`rustls-platform-verifier-android` 等 **Android/JNI 相关 Rust crate**）。这些新增 target 在 `prelude//platforms:default` 下做 configuration / toolchain resolution 时，会把对应的 JVM/Android toolchain helper 作为依赖引入；而 rdeps 传播又不过滤这些 helper，于是 `project/buck2_test/toolchains:jdk_system_image` 等被选入 `buck2 build`，最终在缺环境的 worker 上 `jlink` / `core-for-system-modules.jar` 报错。 - -> 结论：引入点是 **“全语言默认平台 + 不过滤 toolchain/platform 的影响传播”**，叠加 `rk8s` 三方 crate 中的 Android/JNI 依赖；不是 `buck2_test` 业务代码主动依赖 Java/Android。 - -### 2.3 次要放大源：`.buckconfig` 变更触发 select-all - -若某个 CL 改了根 `.buckconfig`（或 `.buckroot` / `.bazelrc` / `.buckversion`，常见于注册新 cell/toolchain），`immediate_target_changes()` 会进入“通用文件”分支，把**整张图里所有带 Buck 依赖的目标**都标记为受影响： - -```248:261:orion/src/repo/diff.rs - if changes.cell_paths().any(is_buckconfig_change) { - let mut ret = GraphImpact::from_non_recursive( - diff.targets() - .map(|t| (t, ImpactTraceData::new(t, RootImpactKind::UniversalFile))) - .filter(|(t, _)| { - is_target_with_buck_dependencies(t) - || is_target_with_changed_ci_srcs(t, changes) - }) - .filter(|(t, _)| matches_ci_srcs_must_match(&t.ci_srcs_must_match, changes)) - .collect(), - ); - ret.sort(); - return ret; - } -``` - -本次 CL 未见 `.buckconfig` 改动，所以这不是当次直接原因，但属于同类“放大为全图”的潜在风险，需一并关注。 - -### 2.4 路径如何叠加 - -```mermaid -flowchart TD - Change["rk8s 改动\n(新增 Rust 三方 crate, 含 jni/android)"] --> Discovery["发现阶段\nget_repo_targets()\nroot//... + toolchains//..."] - Discovery --> FullGraph["解析整个 root cell + toolchains cell"] - FullGraph --> Select["选择阶段\ncollect_impacted_targets()"] - - Platform["固定默认平台\nprelude//platforms:default\n(全语言 toolchain resolution)"] --> Select - NoFilter["rdeps 传播 follow_rule_type = |_| true\n(不过滤 toolchain/platform helper)"] --> Select - BuckcfgBranch{".buckconfig 改动?\n(本次=否)"} -->|"是 → select-all"| Select - - Select --> Build["buck2 build 选入\nproject/buck2_test/toolchains:jdk_system_image\n__android_sdk_tools__"] - Build --> Fail["worker 缺 JDK/Android\njlink / core-for-system-modules.jar\nFileNotFoundError"] -``` - ---- - -## 3. 现状小结 - -| 维度 | 当前行为 | 问题 | -|------|----------|------| -| 发现范围 | 全 cell `//...`（`root` + `toolchains`） | 解析/初始化全仓所有包与无关 toolchain | -| cell 粒度收窄 | 不适用 | `rk8s` 与 `project/buck2_test` 同在 `root` cell | -| 默认平台 | 固定 `prelude//platforms:default` | 拉入全语言 toolchain resolution（含 JVM/Android helper） | -| 影响传播 | `recursive_target_changes(..., \|_\| true)` | 不过滤 toolchain/platform helper，沿所有边传播 | -| `.buckconfig` 改动 | 触发 select-all | 任意小改动放大为全图构建（本次未触发，但是风险） | -| worker 镜像 | 已补装 JDK17 + Android SDK（治标） | 仅掩盖问题，构建仍在做无关工作、慢且脆 | - -worker 镜像里补装 JDK / Android（见 [scripts/build-custom-image.sh](orion-scheduler/scripts/build-custom-image.sh)）能让构建“不报错”，但并未解决“一个 Rust 改动为何要构建 Java/Android”这一根本问题——构建仍然变慢、变脆，且引入了不必要的环境依赖。 - ---- - -## 4. 修改方案 - -目标：**让 target discovery 与影响传播只覆盖与本次改动相关的范围，不再为了 `rk8s` 改动而扫描/初始化全仓所有 cell，并避免把无关语言的 toolchain/platform helper 选入构建。** - -下面几项可独立或组合落地。 - -### 方案 A：按改动路径收窄发现范围 - -在 `get_repo_targets()` / `get_all_cell_patterns()` 之外新增一条“按改动收窄”的路径： - -1. 由 `changes` 推导出受影响的目录/包，把 `root//...` 收窄为 `root//rk8s/...` 这类**子树模式**（每个改动文件 → 其所在包/目录子树）。 -2. **仅当**改动实际触及某个 cell（如 `toolchains/`）时，才把该 cell 的模式纳入查询；否则不查 `toolchains//...`。 - -- 优点：直接消除“扫全仓 + 初始化无关 toolchain cell”。 -- 代价/风险：**可能漏掉跨目录反向依赖**（rdeps）。若 `project/foo` 依赖 `rk8s/bar`，改 `rk8s/bar` 时 `project/foo` 不会被发现。仅在“子项目相互隔离”假设下安全。 -- 实现落点：`orion/buck/cells.rs` 增加 `get_scoped_patterns(changes)`；`get_repo_targets()` 接收 `changes` 并传入。 - -### 方案 B：过滤 toolchain / platform helper 的影响传播（与 A 互补，强烈建议） - -把 `collect_impacted_targets()` 里 `recursive_target_changes` 的 `follow_rule_type` 从 `|_| true` 改为**跳过 toolchain / platform / 配置类 rule type**，并在最终 build 列表里剔除 toolchain helper target（如 `*/toolchains:*`、`jdk_system_image`、`__android_sdk_tools__` 等）。 - -- 优点：即便它们出现在图里，也不会被当作“要 build 的业务 target”。直接消除 `jlink` / `core-for-system-modules.jar` 这类失败。 -- 代价：需要准确识别哪些 rule type 属于 toolchain/platform（可基于 rule_type 前缀或 package 路径规则），避免误伤真实业务 target。 -- 实现落点：[orion/src/buck_controller.rs](orion/src/buck_controller.rs) 的 `collect_impacted_targets()`；[orion/src/repo/diff.rs](orion/src/repo/diff.rs) 的 `recursive_target_changes()`。 - -### 方案 C：收敛默认平台 / 按语言选择 toolchain - -不再对所有发现/构建钉死 `prelude//platforms:default`，而是按改动涉及的语言/子项目选择更窄的 platform，或显式裁剪 default platform 引用的 toolchain，使纯 Rust 改动不触发 Java/Android toolchain resolution。 - -- 优点：从“全语言 toolchain resolution”这一源头止血。 -- 代价：需要梳理 `prelude//platforms:default` 的定义与各语言 toolchain 装配方式，改动面与风险相对较高，需配合 Buck 侧配置。 -- 实现落点：`orion/buck/run.rs::targets_arguments()` 与 `buck2 build` 的 `--target-platforms` 参数，以及仓库 `prelude//platforms` / `toolchains` 定义。 - -### 方案 D：按路径收窄 + `buck2 uquery rdeps` 补齐（保正确性） - -在方案 A 基础上，对收窄得到的“种子目标”再跑 `buck2 uquery "rdeps(, )"` 补回跨目录反向依赖。 - -- 优点：不漏建，保持 CI 正确性。 -- 代价：rdeps 查询仍需触碰较大范围的图，初始化节省有限；实现更复杂。 - -### 方案 E：收敛 `.buckconfig` select-all（防回归） - -让 `.buckconfig` 等通用文件变更不再无脑 select-all，而是限定在受影响 cell / 子树内。本次非直接原因，但建议一并收敛以防后续回归。 - -### 方案对比 - -| 方案 | 不再扫全仓 | 不 build 无关 toolchain | 从源头去 JVM/Android resolution | 保持跨目录 rdeps 正确 | 复杂度 | -|------|-----------|-------------------------|----------------------------------|------------------------|--------| -| A 路径收窄 | 是 | 部分 | 否 | 否（可能漏建） | 低 | -| B 过滤 toolchain 传播 | 否 | 是 | 否（图里仍解析） | 是 | 低-中 | -| C 收敛默认平台 | 否 | 是 | 是 | 是 | 高 | -| D A + uquery rdeps | 部分 | 部分 | 否 | 是 | 高 | -| E 收敛 select-all | 否 | 部分 | 否 | 视实现而定 | 低-中 | - ---- - -## 5. 推荐与待确认 - -推荐组合：**A（按路径收窄发现）+ B（过滤 toolchain/platform helper 传播）**。A 降低扫描/初始化范围，B 直接阻止把无关语言 toolchain helper 选入 build；两者改动面小、对当次失败最直接。若要从源头根治，再逐步推进 **C**。 - -待确认问题： - -1. 收窄范围的单元用“改动文件所在目录子树”还是 webhook 已提供的 `repo` / `repo_prefix` 子项目边界？ -2. 是否接受方案 A 可能漏掉的跨目录反向依赖（即子项目是否真的相互隔离），还是需要 D 来兜底？ -3. 方案 B 中“toolchain/platform helper”的识别标准：按 rule type 前缀、按 `*/toolchains:*` 包路径，还是两者结合？ -4. 是否需要同时推进 C（收敛默认平台），从源头去掉纯 Rust 改动的 JVM/Android toolchain resolution？ - ---- - -## 6. 相关代码索引 - -- [orion/src/buck_controller.rs](orion/src/buck_controller.rs)：`get_build_targets()`、`get_repo_targets()`、`collect_impacted_targets()`（`recursive_target_changes(..., |_| true)`）、`buck2 build`（固定 `--target-platforms prelude//platforms:default`）。 -- [orion/buck/run.rs](orion/buck/run.rs)：`targets_arguments()`（发现阶段也固定 `prelude//platforms:default`）。 -- [orion/buck/cells.rs](orion/buck/cells.rs)：`get_all_cell_patterns()`、`unresolve()`（路径 → cell 的映射，可复用做路径收窄）。 -- [orion/src/repo/diff.rs](orion/src/repo/diff.rs)：`immediate_target_changes()`（`.buckconfig` → select-all）、`recursive_target_changes()`（rdeps 传播，`follow_rule_type`）。 -- [orion/src/repo/changes.rs](orion/src/repo/changes.rs)：`is_package_level_file()`（`.buckconfig` 等判定）。 -- [orion-scheduler/scripts/build-custom-image.sh](orion-scheduler/scripts/build-custom-image.sh)：worker 镜像中 JDK17 / Android SDK 的安装（当前“治标”措施）。 diff --git a/orion-scheduler/TESTING.md b/orion-scheduler/TESTING.md index 28bd414f3..0c96a2001 100644 --- a/orion-scheduler/TESTING.md +++ b/orion-scheduler/TESTING.md @@ -1,258 +1,174 @@ # 测试方法 -本文档包含本地调试、API 测试、服务管理和常见问题排查的方法。 +本地调试、API 测试、服务管理和常见问题排查。 ## 前提条件 -首次使用前需要先生成 SSH 密钥并构建本地镜像： - ```bash -# 1. 生成 SSH 密钥对（用于 VM 访问，路径配置在 target_config.json 的 ssh_public_key_path） +# SSH 密钥（路径写入 target_config.json 的 ssh_public_key_path） ssh-keygen -t ed25519 -f ~/.ssh/orion_vm_access -N "" -C "orion-scheduler" -# 2. 构建自定义镜像（包含 Rust 工具链和 Buck2） -sudo bash ~/mega/orion-scheduler/scripts/build-custom-image.sh - -# 3. 复制并编辑配置文件 +# 配置文件 cp orion-scheduler/target_config.json.template orion-scheduler/target_config.json -# 编辑 target_config.json，填入本机实际路径 +# 编辑 target_config.json，填入本机路径 ``` -## 1. 本地调试 +自定义 VM 镜像的构建与上传见 [§4 构建镜像并上传到 S3](#4-构建镜像并上传到-s3)。 -### 构建与运行 +--- -```bash -# 构建调试版本 -cargo build - -# 运行服务（调试模式） -sudo env "PATH=$PATH" "RUSTUP_HOME=$RUSTUP_HOME" "CARGO_HOME=$CARGO_HOME" "HOME=$HOME" cargo run -``` - -### 调试流程 +## 1. 快速开始 ```bash -# 1. 发送 webhook 请求（使用默认 Debian 镜像） -curl -X POST http://localhost:8080/webhook \ - -H "Content-Type: application/json" \ - -d '{"target": "aws-gitmega"}' +# 构建并启动 scheduler（需 root/KVM） +cargo build -p orion-scheduler +sudo env "PATH=$PATH" "RUSTUP_HOME=$RUSTUP_HOME" "CARGO_HOME=$CARGO_HOME" "HOME=$HOME" \ + cargo run -p orion-scheduler -# 1b. 发送 webhook 请求（指定本地自定义镜像） +# 触发 VM + Orion worker（推荐：本地 debian-13-buck2 镜像） curl -X POST http://localhost:8080/webhook \ -H "Content-Type: application/json" \ -d '{ - "target": "aws-gitmega", + "target": "k3s-buck2hub", "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", - "image_digest": "sha256:e3219324738ef9492042f021fa13b44dc668507f2bd254b5c3470f1d7cdfcce4", - "image_disk_gb": 20, - "image_cpus": 2, - "image_memory_mb": 4096 - }' - -# 1c. 发送 webhook 请求（指定远程镜像） -curl -X POST http://localhost:8080/webhook \ - -H "Content-Type: application/json" \ - -d '{ - "target": "aws-gitmega", - "image_url": "https://gitmega.s3.ap-southeast-2.amazonaws.com/images/debian-13-buck2.qcow2?response-content-disposition=inline&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEDgaDmFwLXNvdXRoZWFzdC0yIkYwRAIgQmiSRJW4dcJqZ1YlbTo64NAZipaYlDxezUtQoVpn2R4CICJxDmSVTUWvGXxGJxSVCm59TFian4l%2B95P4lRM9X5VLKtYDCAEQABoMNTM1MDAyODcyMDczIgytK%2FEFzG5xpCYPutAqswM2%2FRYbsun0%2FTUNc44myCbtG8Zl9vGxs0zoHA8PUK5yxWVugKy7wE8maQyBRsnRxj97YvDd64HDWJgy%2F6ZJRzRIkn5O4gOjOfACr2RibrAF951%2FnIz8gyiESic8DUVBV8K0xLT%2FOXOIvY9DdhwNXP5O1CG63IRE%2FEoIAEwDrJl4Fr3tW868bdzRUEiYwwclvWQ17i8Gw2xnbJ%2FLUTnuWOcoI3tECZam2VHs1Fi00YyIhTnZRmKcqirxIar8%2BGv7JrXrMd0Nup8s12zGjsZWJ%2FeNxVWzmh4A4K43enJT%2BAwHhQ%2FEfTVZh%2F4CxYXbOHTiSVVjvtpJ4QgiiQR6VyiY5Wp2UkEHguiQC8MIOemYIuZFdSoDWvs1HjofbJ13%2FdySYg1fRvlnmteJyE%2F4J6vJ83Rt3W4GeqTntKFIqC6xlhKbYx0Wektf1p%2F1qMXKFmNI%2BvIDVU5xS9Prm6NKWkUeKDBC6t%2FdxDXjzyRJNVenmGPg%2BHufyUEb2gyKDdgXxxg%2BBYK%2Bwt97tvqRh8s58khV36v8Nt55NlrWMIAi84q8AJFKtL227wDmRXl%2FZGsXBYMR776qrt8wlte50AY63wKNrXwyrKpRVq0LjQ5Rd0nklMtriQ9deJo1NNT6CR8ZJr2x17Cf3JSM1EbQwCRdHpJavob7bVJyrIhVY%2B8zIFp70XsZZWhcnd6P32ymnttMAvCQUZB%2FxcHN%2BDlzB1AtATRWtXihVq0ExD2%2BLbXCx8H0Lkdxpd87EOZ7d095DK9zFXz7FAlXlPiGGVSyQWB%2FUfQzk%2BDv1%2FInTIjjSnzYQD5dmrPwXVtbEBx6zolpTCYGyVLpnasgsFTv1nUR9c7e7POCxVjEy8WO%2FhAZwA%2FP8FDxnmlxUatyD7zTPohPWJVpymprxABhaZrUQLTL9SBuU462Wfc6sv6Mmm7RVKkIqGrDzW3OdnJtVCeYUdEhXUHlis0cbQsFzce9bM0LnynlBd67mqOyfpZdeJ531NPvgdukOzfCE2BITZ%2BjXWMDN%2F3Ndo%2FaSoC%2B9lT%2FwYnhxkMQh4cSKog3pPsvllJHNG4RILs%3D&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIAXZEFH7EE6E2HVVIM%2F20260521%2Fap-southeast-2%2Fs3%2Faws4_request&X-Amz-Date=20260521T082130Z&X-Amz-Expires=14400&X-Amz-SignedHeaders=host&X-Amz-Signature=99471e1c7bab465f0242fc9bb86aabfbdbf271d079ca0db780b9ba29503e3d86", - "image_digest": "sha256:677ef198bb2a8a30bb3a593b1b70efb9a14f6e06a1193df47d4e028bce0445d6", - "image_disk_gb": 20, - "image_cpus": 4, - "image_memory_mb": 8192 + "image_digest": "sha256:753c28888c9d30fe4baef55c1d1dfa9a39431595eca940b7ad85d78d84f3d7a5", + "image_disk_gb": 30, + "image_cpus": 8, + "image_memory_mb": 16000 }' -# 2. 检查服务状态（VM 应保持运行状态） +# VM 与日志 curl http://localhost:8080/status - -# 3. 持续监控日志（SSE 流，每 2 秒刷新，适合终端监控） curl -N http://localhost:8080/logs/orion/stream ``` -## 2. API 测试 +`image_path` 支持 `~/...` 或绝对路径。其他 webhook 变体（默认镜像、远程 `image_url`）见 [§2 Webhook](#webhook)。 + +--- + +## 2. API 参考 ### 健康检查 ```bash curl http://localhost:8080/health -# 响应: {"status": "healthy", "service": "orion-scheduler"} +# {"status": "healthy", "service": "orion-scheduler"} ``` ### Webhook ```bash -# GET - 健康检查 +# GET curl http://localhost:8080/webhook -# 响应: {"status": "ok", "vm_id": null, "error": null, "orion_log_file": null} -# POST - 触发部署（keep-alive 模式，使用默认 Debian 镜像） +# POST — 默认 Debian 镜像 curl -X POST http://localhost:8080/webhook \ -H "Content-Type: application/json" \ - -d '{"target": "gcp-buck2hub"}' -# 响应: {"status": "ok", "vm_id": "orion-vm-xxx", "error": null, "orion_log_file": null} + -d '{"target": "aws-gitmega"}' -# POST - 指定本地镜像 +# POST — 本地自定义镜像（字段同 §1 快速开始） curl -X POST http://localhost:8080/webhook \ -H "Content-Type: application/json" \ - -d '{ - "target": "gcp-buck2hub", - "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", - "image_digest": "sha256:abcd1234...", - "image_disk_gb": 20, - "image_cpus": 4, - "image_memory_mb": 8192 - }' + -d '{"target": "gcp-buck2hub", "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", "image_digest": "sha256:...", "image_disk_gb": 20, "image_cpus": 4, "image_memory_mb": 8192}' + +# POST — 远程镜像（image_url + image_digest + 可选 disk/cpus/memory） +curl -X POST http://localhost:8080/webhook \ + -H "Content-Type: application/json" \ + -d '{"target": "aws-gitmega", "image_url": "https://...", "image_digest": "sha256:...", "image_disk_gb": 20, "image_cpus": 4, "image_memory_mb": 8192}' ``` ### VM 状态 ```bash -# 获取 VM 状态（keep-alive 模式，VM 持续运行） curl http://localhost:8080/status -# 响应: {"status": "running", "vm_id": "orion-vm-xxx", "vm_ip": "192.168.221.87", "uptime_secs": 60, "log_file": "/var/log/orion-scheduler/..."} +# {"status": "running", "vm_id": "orion-vm-xxx", "vm_ip": "192.168.221.x", "uptime_secs": 60, ...} ``` -### 日志端点 +### SSH 进入 VM +```bash +VM_IP=$(curl -s http://localhost:8080/status | jq -r .vm_ip) +ssh -i ~/.ssh/orion_vm_access root@$VM_IP +``` -| 端点 | 响应格式 | 特点 | 使用场景 | -| ------------------------ | ---- | ------- | -------------- | -| `GET /logs/orion/stream` | SSE | 每 2 秒推送 | `curl -N` 持续监控 | +### 日志 +| 端点 | 格式 | 说明 | +|------|------|------| +| `GET /logs/orion/stream` | SSE | 每 2 秒推送；`curl -N` 持续监控 | ```bash curl -N http://localhost:8080/logs/orion/stream ``` +服务端调试日志：`RUST_LOG=debug cargo run -p orion-scheduler`；systemd 部署用 `journalctl -u orion-scheduler -f`。 + ### Scorpio 状态 ```bash curl http://localhost:8080/scorpio/status -# 响应: {"status": "ok", "directories": {...}, "mounts": "...", "orion_process": "...", "scorpio_process": "..."} ``` ### 关闭 ```bash -# 优雅关闭（停止 VM 并退出） curl -X POST http://localhost:8080/shutdown -# 响应: {"status": "ok", "message": "Shutdown initiated, VM will be stopped"} +# 仅停 VM，scheduler 继续运行 ``` +--- + ## 3. 服务管理 -### 启动服务 +### 停止与检查 ```bash -cargo run -p orion-scheduler -``` +# 优雅：先关 VM（见上），再停 scheduler +kill -TERM -### 停止服务 - -```bash -# 停止 orion-scheduler 服务进程 +# 强制（不关闭 VM） pkill -9 -f orion-scheduler +sudo pkill -9 -f qemu-system-x86 # 清理残留 QEMU -# 停止所有 QEMU 进程（如果有残留的 VM） -sudo pkill -9 -f qemu-system-x86 - -# 验证进程已停止 ps aux | grep -E "orion-scheduler|qemu-system" | grep -v grep - -# 检查端口是否释放 fuser 8080/tcp 2>/dev/null || echo "Port 8080 is free" ``` -### 检查服务状态 +### 信号与关闭方式 -```bash -# 检查 HTTP API 状态 -curl http://localhost:8080/status - -# 检查 orion-scheduler 进程 -ps aux | grep orion-scheduler | grep -v grep - -# 检查 QEMU 进程 -ps aux | grep qemu | grep -v grep -``` +| 操作 | VM | scheduler | 说明 | +|------|-----|-----------|------| +| `Ctrl+C` / SIGTERM / SIGQUIT | 停止 | 停止 | 先关 VM 再退出 | +| `POST /shutdown` | 停止 | **继续** | 仅关 VM，适合换 CL 重跑 | +| `pkill -9 -f orion-scheduler` | 可能残留 | 停止 | 不优雅 | -### 优雅关闭对比 - - -| 操作 | VM | 服务器 | 说明 | -| ----------------------------- | --- | ---- | ---------------------- | -| `Ctrl+C` | 停止 | 停止 | 关闭 VM 后退出服务 | -| SIGTERM | 停止 | 停止 | 关闭 VM 后退出服务 | -| SIGQUIT | 停止 | 停止 | 关闭 VM 后退出服务 | -| `POST /shutdown` | 停止 | 继续运行 | 仅关闭 VM，服务保持运行 | -| `pkill -9 -f orion-scheduler` | - | 停止 | **不优雅**：直接杀死进程，不会关闭 VM | +--- - -```bash -# 关闭 VM，服务继续运行（推荐） -curl -X POST http://localhost:8080/shutdown - -# 发送 SIGTERM 信号（关闭 VM 并停止服务） -kill -TERM - -# 强制杀死进程（不优雅） -pkill -9 -f orion-scheduler -``` - -## 4. 查看日志 - -```bash -# 服务端日志 -RUST_LOG=debug cargo run 2>&1 | grep -E '\[orion|webhook|vm' - -# Orion 实时 SSE 流（持续刷新，Ctrl+C 退出） -curl -N http://localhost:8080/logs/orion/stream - -# systemd 日志（如服务以 systemd 运行） -journalctl -u orion-scheduler -f -``` - -## 5. 构建镜像并上传到 S3 - -### 5.1 构建本地镜像 +## 4. 构建镜像并上传到 S3 ```bash sudo modprobe nbd max_part=8 sudo bash ~/mega/orion-scheduler/scripts/build-custom-image.sh +# 输出 sha256:，用作 webhook 的 image_digest ``` -构建完成后会输出： - -``` -sha256:<镜像SHA256值> -``` - -### 5.2 上传到 S3 - ```bash -# 上传镜像到 S3 -aws s3 cp ~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2 \ - s3://gitmega/images/debian-13-buck2.qcow2 - -# 上传时显示进度 aws s3 cp ~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2 \ s3://gitmega/images/debian-13-buck2.qcow2 --progress - ``` -> **注意**：`image_digest` 使用构建脚本输出的 `sha256:` 值（上传前本地文件的 hash），上传后内容不变 hash 保持一致。 - -## 6. 常见问题排查 - +`image_digest` 使用构建脚本输出的本地文件 hash；上传前后内容不变则 hash 一致。 -| 问题 | 排查方法 | -| ------------------- | ------------------------------------------------------------------------------------------------------------- | -| KVM 权限错误 | 检查 `/dev/kvm` 权限，确保用户在 `kvm` 组 | -| QEMU 网络桥接失败 | 检查 `/etc/qemu/bridge.conf` 是否配置 `allow qlbr0` | -| VM 启动超时 | 检查 cloud-init 是否正常，SSH 是否可连接 | -| Orion 启动失败 | `curl -N http://localhost:8080/logs/orion/stream` 实时查看日志 | -| Scorpio 挂载问题 | `curl http://localhost:8080/scorpio/status` 检查挂载状态 | -| VM 已关闭但状态显示 running | 重启服务或检查 VM 是否异常退出 | -| 需要 SSH 进入 VM 调试 | Orion-scheduler 会自动注入 `ssh_public_key_path` 配置的公钥对应的私钥访问权限。使用 `ssh -i ~/.ssh/orion_vm_access root@` 连接 | +--- +## 5. 常见问题排查 +| 问题 | 排查 | +|------|------| +| KVM 权限错误 | `/dev/kvm` 权限；用户是否在 `kvm` 组 | +| QEMU 桥接失败 | `/etc/qemu/bridge.conf` 是否 `allow qlbr0` | +| VM 启动超时 | cloud-init、SSH 是否可达 | +| Orion 启动失败 | `curl -N http://localhost:8080/logs/orion/stream` | +| Scorpio 挂载问题 | `curl http://localhost:8080/scorpio/status` | +| 状态仍 running 但 VM 已死 | 重启 scheduler 或查 QEMU 进程 | +| 进 VM 调试 | [SSH 进入 VM](#ssh-进入-vm) | diff --git a/orion-scheduler/scripts/build-custom-image.sh b/orion-scheduler/scripts/build-custom-image.sh index 43b85fe27..d39a57ddd 100755 --- a/orion-scheduler/scripts/build-custom-image.sh +++ b/orion-scheduler/scripts/build-custom-image.sh @@ -2,19 +2,37 @@ # Build a custom Debian image with buck2 and Rust toolchain pre-installed. # # Pre-installs Rust toolchain, apt packages, and buck2 directly into the image -# via chroot. No VM boot needed, so this is significantly faster than the -# previous cloud-init based approach. +# via chroot. Also caches the prelude cpython_archive tarball behind a local +# GitHub HTTPS mirror so toolchains//:cpython_archive can fetch without hitting +# github.com during worker builds (no toolchains repo changes required). # # Usage: sudo ./build-custom-image.sh # # Note: Must run as root because qemu-nbd / mount / chroot need it. +# Images are published to the invoking user's ~/.local/share/qlean/images +# (e.g. /home/orion/... when run as `sudo -u` or `sudo` from user orion), +# not /root/. Override with OUTPUT_DIR=... if needed. set -eo pipefail # ============================================================================ # Configuration # ============================================================================ -OUTPUT_DIR="${OUTPUT_DIR:-$HOME/.local/share/qlean/images}" +# qlean / orion-scheduler run as the login user (orion), not root. When this +# script is invoked via sudo, $HOME is /root — publish to the invoking user's +# qlean images dir unless OUTPUT_DIR is set explicitly. +if [ -n "${OUTPUT_DIR:-}" ]; then + : # caller override +elif [ -n "${SUDO_USER:-}" ] && [ "$SUDO_USER" != "root" ]; then + _qlean_home="$(getent passwd "$SUDO_USER" | cut -d: -f6)" + OUTPUT_DIR="${_qlean_home}/.local/share/qlean/images" +elif [ -n "${SUDO_UID:-}" ] && [ "$SUDO_UID" != "0" ]; then + _qlean_home="$(getent passwd "$SUDO_UID" | cut -d: -f6)" + OUTPUT_DIR="${_qlean_home}/.local/share/qlean/images" +else + OUTPUT_DIR="${HOME}/.local/share/qlean/images" +fi +unset _qlean_home IMAGE_NAME="debian-13-buck2" IMAGE_DIR="$OUTPUT_DIR/$IMAGE_NAME" BASE_DIR="$OUTPUT_DIR/debian-13-generic-amd64" @@ -38,38 +56,88 @@ BASE_CHECKSUM_URL="${BASE_MIRROR_URL%/}/SHA512SUMS" IMAGE_SIZE="15G" -RUST_VERSION="1.95.0" +RUST_VERSION="1.96.0" RUST_ARCH="x86_64-unknown-linux-gnu" +# Host-side cache for the Rust toolchain tarball. Pre-place the file here to +# skip the download (useful when the network is flaky/blocked): +# curl -fL -o /tmp/rust-1.96.0-x86_64-unknown-linux-gnu.tar.gz "$RUST_TARBALL_URL" RUST_TARBALL="/tmp/rust-${RUST_VERSION}-${RUST_ARCH}.tar.gz" RUST_TARBALL_URL="https://static.rust-lang.org/dist/rust-${RUST_VERSION}-${RUST_ARCH}.tar.gz" BUCK2_VERSION="2026-04-15" BUCK2_ARCH="x86_64-unknown-linux-musl" BUCK2_URL="https://github.com/facebook/buck2/releases/download/${BUCK2_VERSION}/buck2-${BUCK2_ARCH}.zst" +# Host-side cache for the buck2 archive. Pre-place the file here to skip the +# in-chroot GitHub download entirely (useful when GitHub is flaky/blocked): +# curl -fL -o /tmp/buck2-x86_64-unknown-linux-musl.zst "$BUCK2_URL" +BUCK2_TARBALL="/tmp/buck2-${BUCK2_ARCH}.zst" +# Host-side apt caches, bind-mounted into the chroot so repeated image builds +# reuse already-downloaded indexes and .deb packages. They live on the host and +# are unmounted before the image is sealed, so cached data is NOT baked into the +# final image. Override with APT_LISTS_DIR / APT_CACHE_DIR. +APT_LISTS_DIR="${APT_LISTS_DIR:-/var/cache/orion-image/apt-lists}" +APT_CACHE_DIR="${APT_CACHE_DIR:-/var/cache/orion-image/apt-archives}" + +# CPython tarball for prelude remote_python_toolchain / toolchains//:cpython_archive. +# Baked into a local GitHub HTTPS mirror (see chroot) so buck2 http_archive keeps +# the upstream URL but reads the file from disk on workers. +CPYTHON_VERSION="3.13.6" +CPYTHON_BUILD="20250807" +CPYTHON_ARCH="x86_64-unknown-linux-gnu" +CPYTHON_TARBALL="/tmp/cpython-${CPYTHON_VERSION}-${CPYTHON_ARCH}.tar.gz" +CPYTHON_TARBALL_URL="https://github.com/astral-sh/python-build-standalone/releases/download/${CPYTHON_BUILD}/cpython-${CPYTHON_VERSION}+${CPYTHON_BUILD}-${CPYTHON_ARCH}-install_only_stripped.tar.gz" +CPYTHON_MIRROR_REL_PATH="astral-sh/python-build-standalone/releases/download/${CPYTHON_BUILD}/cpython-${CPYTHON_VERSION}+${CPYTHON_BUILD}-${CPYTHON_ARCH}-install_only_stripped.tar.gz" +CPYTHON_SHA256="e3e280d4b1ead63de6ebc9816de71792fc8c71b7a6a999ea82f937047beba037" ROOT_SSH_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9LTEGIaaad0XP4qUfBoVRgeOg+G36jIWIiqIWP/k4g" NBD_DEV="/dev/nbd0" NBD_PART="${NBD_DEV}p1" +# When run via sudo, root creates files under the invoking user's OUTPUT_DIR. +# qlean / orion-scheduler must be able to read/write debian-13-buck2.json. +fix_qlean_ownership() { + local owner="${SUDO_USER:-}" + if [ -z "$owner" ] || [ "$owner" = "root" ]; then + return 0 + fi + echo "[build-custom-image] chown $owner: $OUTPUT_DIR ($IMAGE_NAME artifacts)" + chown -R "$owner:$owner" "$IMAGE_DIR" "$BASE_DIR" 2>/dev/null || true + chown "$owner:$owner" "$PUBLISHED_IMAGE" "$PUBLISHED_JSON" 2>/dev/null || true + chmod 644 "$PUBLISHED_JSON" "$IMAGE_DIR/checksums" 2>/dev/null || true + chmod 644 "$PUBLISHED_IMAGE" "$CUSTOM_IMAGE" 2>/dev/null || true +} + # ============================================================================ # Cleanup trap — always release mounts and NBD even on failure / Ctrl-C # ============================================================================ MOUNT_DIR="" +BUILD_STAGE="init" + +log_stage() { + BUILD_STAGE="$1" + echo "[build-custom-image] >>> stage: $BUILD_STAGE" +} + +log_cmd() { + echo "[build-custom-image] \$ $*" +} + cleanup() { local rc=$? set +e + if [ $rc -ne 0 ]; then + echo "[build-custom-image] FAILED with exit code $rc (stage: ${BUILD_STAGE:-unknown})" >&2 + fi if [ -n "$MOUNT_DIR" ]; then - for sub in proc sys dev; do + echo "[build-custom-image] cleanup: unmounting $MOUNT_DIR (stage was: ${BUILD_STAGE:-unknown})" + for sub in var/lib/apt/lists var/cache/apt/archives proc sys dev; do mountpoint -q "$MOUNT_DIR/$sub" && sudo umount "$MOUNT_DIR/$sub" done mountpoint -q "$MOUNT_DIR" && sudo umount "$MOUNT_DIR" [ -d "$MOUNT_DIR" ] && rmdir "$MOUNT_DIR" 2>/dev/null fi sudo qemu-nbd --disconnect "$NBD_DEV" 2>/dev/null - if [ $rc -ne 0 ]; then - echo "[build-custom-image] FAILED with exit code $rc" - fi exit $rc } trap cleanup EXIT INT TERM HUP @@ -115,7 +183,10 @@ download_base_image() { # ============================================================================ # Pre-flight: ensure base image (download if missing) # ============================================================================ +log_stage "preflight" echo "[build-custom-image] Starting custom image build..." +echo "[build-custom-image] OUTPUT_DIR=$OUTPUT_DIR" +echo "[build-custom-image] CUSTOM_IMAGE=$CUSTOM_IMAGE" if [ ! -f "$BASE_IMAGE" ]; then download_base_image @@ -131,6 +202,7 @@ mkdir -p "$IMAGE_DIR" # ============================================================================ # Stage 1: Copy + resize qcow2 # ============================================================================ +log_stage "1-copy-resize" echo "[build-custom-image] Copying base image..." cp "$BASE_IMAGE" "$CUSTOM_IMAGE" @@ -148,9 +220,56 @@ else echo "[build-custom-image] Using cached Rust tarball: $(du -sh "$RUST_TARBALL" | cut -f1)" fi +# ============================================================================ +# Stage 2b: Download buck2 on host (avoids needing reliable network in chroot) +# ============================================================================ +# Reuse a manually pre-downloaded archive if present at $BUCK2_TARBALL. +if [ ! -f "$BUCK2_TARBALL" ]; then + echo "[build-custom-image] Downloading buck2 ${BUCK2_VERSION}..." + if ! curl -fL --connect-timeout 30 --retry 5 --retry-delay 3 --retry-all-errors \ + -C - -o "$BUCK2_TARBALL" "$BUCK2_URL"; then + rm -f "$BUCK2_TARBALL" + echo "[build-custom-image] ERROR: failed to download buck2 from $BUCK2_URL" >&2 + echo "[build-custom-image] Pre-download it manually then re-run, e.g.:" >&2 + echo "[build-custom-image] curl -fL -o $BUCK2_TARBALL \"$BUCK2_URL\"" >&2 + exit 56 + fi + echo "[build-custom-image] buck2 downloaded: $(du -sh "$BUCK2_TARBALL" | cut -f1)" +else + echo "[build-custom-image] Using cached buck2 archive: $(du -sh "$BUCK2_TARBALL" | cut -f1)" +fi + +# ============================================================================ +# Stage 2c: Download cpython_archive tarball on host (buckal / prelude python bootstrap) +# ============================================================================ +if [ ! -f "$CPYTHON_TARBALL" ]; then + echo "[build-custom-image] Downloading CPython ${CPYTHON_VERSION} (${CPYTHON_BUILD})..." + if ! curl -fL --connect-timeout 30 --retry 5 --retry-delay 3 --retry-all-errors \ + -C - -o "$CPYTHON_TARBALL" "$CPYTHON_TARBALL_URL"; then + rm -f "$CPYTHON_TARBALL" + echo "[build-custom-image] ERROR: failed to download CPython from $CPYTHON_TARBALL_URL" >&2 + echo "[build-custom-image] Pre-download it manually then re-run, e.g.:" >&2 + echo "[build-custom-image] curl -fL -o $CPYTHON_TARBALL \"$CPYTHON_TARBALL_URL\"" >&2 + exit 56 + fi + echo "[build-custom-image] CPython tarball downloaded: $(du -sh "$CPYTHON_TARBALL" | cut -f1)" +else + echo "[build-custom-image] Using cached CPython tarball: $(du -sh "$CPYTHON_TARBALL" | cut -f1)" +fi + +echo "[build-custom-image] Verifying CPython SHA256..." +actual_cpython_sha=$(sha256sum "$CPYTHON_TARBALL" | awk '{ print $1 }') +if [ "$CPYTHON_SHA256" != "$actual_cpython_sha" ]; then + echo "[build-custom-image] ERROR: CPython checksum mismatch" >&2 + echo "[build-custom-image] expected: $CPYTHON_SHA256" >&2 + echo "[build-custom-image] actual: $actual_cpython_sha" >&2 + exit 1 +fi + # ============================================================================ # Stage 3: NBD attach + filesystem grow + mount # ============================================================================ +log_stage "3-nbd-mount" wait_for() { # wait_for local path="$1" tries="${2:-40}" i @@ -217,17 +336,51 @@ sudo mount --bind /sys "$MOUNT_DIR/sys" sudo mount --bind /dev "$MOUNT_DIR/dev" sudo cp --remove-destination /etc/resolv.conf "$MOUNT_DIR/etc/resolv.conf" +# Bind-mount host apt caches so `apt-get update` / `apt-get install` reuse +# previously downloaded indexes and .deb files. Because these are bind mounts +# onto the host, cached data stays on the host and is unmounted before the +# image is compacted (Stage 5), so it doesn't bloat the final image. +echo "[build-custom-image] Using apt lists cache: $APT_LISTS_DIR" +echo "[build-custom-image] Using apt archive cache: $APT_CACHE_DIR" +sudo mkdir -p "$APT_LISTS_DIR" "$APT_CACHE_DIR/partial" +sudo mkdir -p "$MOUNT_DIR/var/lib/apt/lists" "$MOUNT_DIR/var/cache/apt/archives/partial" +sudo mount --bind "$APT_LISTS_DIR" "$MOUNT_DIR/var/lib/apt/lists" +sudo mount --bind "$APT_CACHE_DIR" "$MOUNT_DIR/var/cache/apt/archives" + # Copy Rust tarball into the image's /tmp before entering chroot sudo cp "$RUST_TARBALL" "$MOUNT_DIR/tmp/rust.tar.gz" +# Copy buck2 archive in too so the chroot reuses it instead of hitting GitHub. +sudo cp "$BUCK2_TARBALL" "$MOUNT_DIR/tmp/buck2.zst" +# Copy cpython_archive tarball for the in-image GitHub mirror (toolchains unchanged). +sudo cp "$CPYTHON_TARBALL" "$MOUNT_DIR/tmp/cpython.tar.gz" + +# Resolve github.com while the host still has normal DNS (chroot will point it at 127.0.0.1). +GITHUB_UPSTREAM_IP=$(getent ahostsv4 github.com 2>/dev/null | awk '{print $1; exit}') +if [ -z "$GITHUB_UPSTREAM_IP" ]; then + GITHUB_UPSTREAM_IP="140.82.121.3" + echo "[build-custom-image] WARNING: could not resolve github.com; using fallback ${GITHUB_UPSTREAM_IP}" +else + echo "[build-custom-image] github.com upstream IP for mirror proxy: ${GITHUB_UPSTREAM_IP}" +fi # ============================================================================ # Stage 4: Install everything inside the chroot (no VM boot needed) # ============================================================================ +log_stage "4-chroot-install" +CHROOT_LOG="/tmp/chroot_install.log" echo "[build-custom-image] Installing toolchain into image (chroot)..." +echo "[build-custom-image] chroot log: $CHROOT_LOG" +# Disable errexit and pipefail for chroot|tee: with pipefail off, set -e would +# still abort on tee(1) failure even when chroot succeeded; capture PIPESTATUS +# explicitly instead. +set +e +set +o pipefail sudo BUCK2_URL="$BUCK2_URL" BUCK2_VERSION="$BUCK2_VERSION" \ + CPYTHON_MIRROR_REL_PATH="$CPYTHON_MIRROR_REL_PATH" \ + GITHUB_UPSTREAM_IP="$GITHUB_UPSTREAM_IP" \ ROOT_SSH_KEY="$ROOT_SSH_KEY" \ - chroot "$MOUNT_DIR" /bin/bash <<'CHROOT_EOF' 2>&1 | tee /tmp/chroot_install.log -set -eo pipefail + chroot "$MOUNT_DIR" /bin/bash <<'CHROOT_EOF' 2>&1 | tee "$CHROOT_LOG" +set -eu export HOME=/root export DEBIAN_FRONTEND=noninteractive export PATH=/root/.cargo/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin @@ -254,8 +407,8 @@ echo "=== [chroot] Rust version: $(/root/.cargo/bin/rustc --version) ===" echo "=== [chroot] Installing apt packages ===" apt-get update apt-get install -y \ - clang lld pkg-config protobuf-compiler zstd fuse curl git \ - seccomp libseccomp-dev libpython3-dev openssl libssl-dev build-essential + clang lld pkg-config protobuf-compiler zstd fuse curl git nginx \ + seccomp libseccomp-dev libpython3-dev openssl libssl-dev build-essential ca-certificates echo "=== [chroot] Verifying installed tools ===" git --version @@ -265,64 +418,258 @@ protoc --version zstd --version | head -1 echo "=== [chroot] Installing buck2 (${BUCK2_VERSION}) ===" -curl -fsSL -o /tmp/buck2.zst "$BUCK2_URL" +# Prefer the archive copied in from the host ($MOUNT_DIR/tmp/buck2.zst). Only +# fall back to downloading inside the chroot if it's missing. GitHub release +# downloads frequently get TLS-reset (curl 56: unexpected eof) on flaky +# networks, so the fallback retries with resume. +if [ -s /tmp/buck2.zst ]; then + echo "=== [chroot] Using buck2 archive provided by host ===" +else + rm -f /tmp/buck2.zst + buck2_downloaded=0 + for attempt in 1 2 3 4 5; do + if curl -fL --connect-timeout 30 --retry 5 --retry-delay 3 --retry-all-errors \ + -C - -o /tmp/buck2.zst "$BUCK2_URL"; then + buck2_downloaded=1 + break + fi + echo "=== [chroot] buck2 download attempt ${attempt} failed; retrying in 5s ===" + sleep 5 + done + if [ "$buck2_downloaded" -ne 1 ]; then + echo "=== [chroot] ERROR: failed to download buck2 from $BUCK2_URL after retries ===" >&2 + exit 56 + fi +fi zstd -d /tmp/buck2.zst -o /usr/local/bin/buck2 chmod +x /usr/local/bin/buck2 rm -f /tmp/buck2.zst /usr/local/bin/buck2 --version +echo "=== [chroot] Installing local GitHub mirror for cpython_archive ===" +if [ ! -s /tmp/cpython.tar.gz ]; then + echo "=== [chroot] ERROR: missing /tmp/cpython.tar.gz (host should have copied it) ===" >&2 + exit 1 +fi +MIRROR_ROOT=/var/cache/orion-github-mirror +CERT_DIR=/etc/orion-github-mirror +mkdir -p "${MIRROR_ROOT}/$(dirname "${CPYTHON_MIRROR_REL_PATH}")" "${CERT_DIR}" +mv /tmp/cpython.tar.gz "${MIRROR_ROOT}/${CPYTHON_MIRROR_REL_PATH}" + +# buck2/rustls rejects openssl req -x509 leaf certs (CaUsedAsEndEntity). Use a +# small PKI: CA cert -> system trust store; leaf server cert -> nginx. +cat > "${CERT_DIR}/openssl-ca.cnf" <<'OPENSSL_CA_EOF' +[req] +distinguished_name = dn +x509_extensions = v3_ca +prompt = no + +[dn] +CN = Orion GitHub Mirror CA + +[v3_ca] +basicConstraints = critical, CA:true, pathlen:0 +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer +OPENSSL_CA_EOF + +cat > "${CERT_DIR}/openssl-leaf.cnf" <<'OPENSSL_LEAF_EOF' +[req] +distinguished_name = dn +req_extensions = v3_req +prompt = no + +[dn] +CN = github.com + +[v3_req] +basicConstraints = critical, CA:FALSE +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names + +[alt_names] +DNS.1 = github.com +OPENSSL_LEAF_EOF + +openssl genrsa -out "${CERT_DIR}/ca.key" 4096 +openssl req -x509 -new -nodes \ + -key "${CERT_DIR}/ca.key" \ + -days 3650 \ + -out "${CERT_DIR}/ca.crt" \ + -config "${CERT_DIR}/openssl-ca.cnf" + +openssl genrsa -out "${CERT_DIR}/github.com.key" 2048 +openssl req -new \ + -key "${CERT_DIR}/github.com.key" \ + -out "${CERT_DIR}/github.com.csr" \ + -config "${CERT_DIR}/openssl-leaf.cnf" + +openssl x509 -req \ + -in "${CERT_DIR}/github.com.csr" \ + -CA "${CERT_DIR}/ca.crt" \ + -CAkey "${CERT_DIR}/ca.key" \ + -CAcreateserial \ + -out "${CERT_DIR}/github.com.crt" \ + -days 3650 \ + -extensions v3_req \ + -extfile "${CERT_DIR}/openssl-leaf.cnf" + +# Trust the CA only (not the leaf); rustls validates the server leaf against it. +cp "${CERT_DIR}/ca.crt" /usr/local/share/ca-certificates/orion-github-mirror-ca.crt +update-ca-certificates + +# Verify leaf is not flagged as a CA (would trigger CaUsedAsEndEntity in rustls). +cert_text=$(openssl x509 -in "${CERT_DIR}/github.com.crt" -noout -text) +if echo "$cert_text" | grep -q "CA:TRUE"; then + echo "=== [chroot] ERROR: github.com leaf cert has CA:TRUE extension ===" >&2 + exit 1 +fi + +# Present leaf + CA chain to TLS clients. +cat "${CERT_DIR}/github.com.crt" "${CERT_DIR}/ca.crt" > "${CERT_DIR}/github.com-chain.crt" + +cat > /etc/nginx/sites-available/orion-github-mirror <> /etc/hosts +fi +systemctl enable nginx +echo "=== [chroot] cpython mirror path: https://github.com/${CPYTHON_MIRROR_REL_PATH} ===" + echo "=== [chroot] Installing SSH key for root ===" mkdir -p /root/.ssh chmod 700 /root/.ssh echo "$ROOT_SSH_KEY" > /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys -echo "=== [chroot] Cleaning apt + temp files ===" -apt-get clean -rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +echo "=== [chroot] Cleaning temp files (host apt caches kept) ===" +# NOTE: do NOT run `apt-get clean` or wipe /var/lib/apt/lists here. Both +# /var/lib/apt/lists and /var/cache/apt/archives are bind-mounted to the host +# cache, so cleaning them would wipe the cross-build cache. Cached data is +# unmounted (not copied) before the image is sealed, so it doesn't end up in +# the final image anyway. +rm -f /var/cache/apt/*.bin +rm -rf /tmp/* /var/tmp/* rm -f /usr/sbin/policy-rc.d echo "=== [chroot] Clearing cloud-init state ===" rm -rf /var/lib/cloud/data/* /var/lib/cloud/instance/* 2>/dev/null || true # Zero unused blocks so qemu-img convert below can drop them, producing a -# much smaller final qcow2. -echo "=== [chroot] Zeroing free space (helps qcow2 compact) ===" -dd if=/dev/zero of=/EMPTY bs=4M status=none || true -sync +# much smaller final qcow2. Hitting ENOSPC here is expected and harmless. +echo "=== [chroot] Zeroing free space (helps qcow2 compact; ENOSPC expected) ===" +dd if=/dev/zero of=/EMPTY bs=4M conv=fdatasync 2>/dev/null || true rm -f /EMPTY sync echo "=== [chroot] Disk space after install ===" df -h / +echo "=== [chroot] install finished (exit 0) ===" CHROOT_EOF +_pipe_status=("${PIPESTATUS[@]}") +CHROOT_RC="${_pipe_status[0]:-255}" +TEE_RC="${_pipe_status[1]:-255}" +PIPELINE_RC=$? +set -e +set -o pipefail + +echo "[build-custom-image] chroot pipeline done: pipeline_rc=$PIPELINE_RC chroot_rc=$CHROOT_RC tee_rc=$TEE_RC" +if [ -f "$CHROOT_LOG" ]; then + echo "[build-custom-image] chroot log: $(wc -l < "$CHROOT_LOG") lines, $(stat -c '%s bytes, owner %U:%G' "$CHROOT_LOG" 2>/dev/null || ls -la "$CHROOT_LOG")" +else + echo "[build-custom-image] WARNING: chroot log missing at $CHROOT_LOG" >&2 +fi + +if [ "$CHROOT_RC" -ne 0 ]; then + echo "[build-custom-image] ERROR: chroot install failed (exit $CHROOT_RC)" >&2 + echo "[build-custom-image] Last 20 lines of $CHROOT_LOG:" >&2 + tail -20 "$CHROOT_LOG" >&2 2>/dev/null || true + exit "$CHROOT_RC" +fi +if [ "$TEE_RC" -ne 0 ]; then + echo "[build-custom-image] WARNING: tee failed (exit $TEE_RC) but chroot succeeded; continuing" >&2 + echo "[build-custom-image] tee target: $CHROOT_LOG ($(ls -la "$CHROOT_LOG" 2>/dev/null || echo missing))" >&2 +fi +echo "[build-custom-image] chroot install OK" # ============================================================================ # Stage 5: Unmount, disconnect, then compact the qcow2 # ============================================================================ +log_stage "5-unmount-compact" echo "[build-custom-image] Unmounting chroot bind mounts..." -sudo umount "$MOUNT_DIR/proc" -sudo umount "$MOUNT_DIR/sys" -sudo umount "$MOUNT_DIR/dev" +for _sub in var/lib/apt/lists var/cache/apt/archives proc sys dev; do + log_cmd umount "$MOUNT_DIR/$_sub" + if ! sudo umount "$MOUNT_DIR/$_sub"; then + echo "[build-custom-image] ERROR: umount $MOUNT_DIR/$_sub failed (exit $?)" >&2 + mountpoint "$MOUNT_DIR/$_sub" 2>/dev/null || true + exit 1 + fi +done echo "[build-custom-image] Unmounting image..." -sudo umount "$MOUNT_DIR" +log_cmd umount "$MOUNT_DIR" +if ! sudo umount "$MOUNT_DIR"; then + echo "[build-custom-image] ERROR: umount $MOUNT_DIR failed (exit $?)" >&2 + mountpoint "$MOUNT_DIR" 2>/dev/null || true + exit 1 +fi rmdir "$MOUNT_DIR" MOUNT_DIR="" -sudo qemu-nbd --disconnect "$NBD_DEV" +log_cmd qemu-nbd --disconnect "$NBD_DEV" +if ! sudo qemu-nbd --disconnect "$NBD_DEV"; then + echo "[build-custom-image] ERROR: qemu-nbd --disconnect $NBD_DEV failed (exit $?)" >&2 + exit 1 +fi +echo "[build-custom-image] NBD disconnected" UNCOMPACT_SIZE=$(du -sh "$CUSTOM_IMAGE" | cut -f1) echo "[build-custom-image] Compacting qcow2 (size before: $UNCOMPACT_SIZE)..." -qemu-img convert -O qcow2 -c "$CUSTOM_IMAGE" "${CUSTOM_IMAGE}.compact" +NEED_BYTES=$(stat -c%s "$CUSTOM_IMAGE") +AVAIL_BYTES=$(df --output=avail -B1 "$(dirname "$CUSTOM_IMAGE")" | tail -1) +echo "[build-custom-image] qemu-img convert needs ~$((NEED_BYTES / 1024 / 1024))MiB, avail $((AVAIL_BYTES / 1024 / 1024))MiB on $(dirname "$CUSTOM_IMAGE")" +if [ "$AVAIL_BYTES" -lt "$NEED_BYTES" ]; then + echo "[build-custom-image] ERROR: need ~$((NEED_BYTES / 1024 / 1024))MiB free for qemu-img convert, only $((AVAIL_BYTES / 1024 / 1024))MiB available on $(dirname "$CUSTOM_IMAGE")" >&2 + exit 1 +fi +log_cmd qemu-img convert -O qcow2 -c "$CUSTOM_IMAGE" "${CUSTOM_IMAGE}.compact" +if ! qemu-img convert -O qcow2 -c "$CUSTOM_IMAGE" "${CUSTOM_IMAGE}.compact"; then + echo "[build-custom-image] ERROR: qemu-img convert failed (exit $?)" >&2 + exit 1 +fi mv "${CUSTOM_IMAGE}.compact" "$CUSTOM_IMAGE" echo "[build-custom-image] Final image size: $(du -sh "$CUSTOM_IMAGE" | cut -f1)" # ============================================================================ # Stage 6: Copy kernel + initrd, write checksums # ============================================================================ +log_stage "6-kernel-checksums" echo "[build-custom-image] Copying kernel and initrd..." +log_cmd cp "$KERNEL" "$IMAGE_DIR/" cp "$KERNEL" "$IMAGE_DIR/" +log_cmd cp "$INITRD" "$IMAGE_DIR/" cp "$INITRD" "$IMAGE_DIR/" echo "[build-custom-image] Calculating checksums..." @@ -330,10 +677,12 @@ echo "[build-custom-image] Calculating checksums..." cd "$IMAGE_DIR" sha256sum "$IMAGE_NAME.qcow2" > checksums ) +echo "[build-custom-image] checksums: $(cat "$IMAGE_DIR/checksums")" # ============================================================================ # Stage 7: Publish image to the path qlean expects (parent-dir flat layout) # ============================================================================ +log_stage "7-publish" # qlean reads $OUTPUT_DIR/$IMAGE_NAME.json which carries `path` pointing at # $OUTPUT_DIR/$IMAGE_NAME.qcow2 (flat). The subdir build artifacts above are # kept for record but qlean won't read them directly. @@ -341,23 +690,44 @@ PUBLISHED_IMAGE="$OUTPUT_DIR/$IMAGE_NAME.qcow2" PUBLISHED_JSON="$OUTPUT_DIR/$IMAGE_NAME.json" # Refuse to overwrite if a VM has the file locked (qemu holds a write lock). -if [ -f "$PUBLISHED_IMAGE" ] && ! qemu-img info "$PUBLISHED_IMAGE" >/dev/null 2>&1; then - echo "[build-custom-image] WARNING: $PUBLISHED_IMAGE appears locked (VM running?)." - echo "[build-custom-image] Skipping publish. Shut down any VMs using it and re-run, or manually:" - echo "[build-custom-image] cp $CUSTOM_IMAGE $PUBLISHED_IMAGE" -else - echo "[build-custom-image] Publishing to $PUBLISHED_IMAGE..." - cp "$CUSTOM_IMAGE" "$PUBLISHED_IMAGE" +if [ -f "$PUBLISHED_IMAGE" ]; then + if qemu-img info "$PUBLISHED_IMAGE" >/dev/null 2>&1; then + echo "[build-custom-image] published image exists and is readable: $PUBLISHED_IMAGE" + else + echo "[build-custom-image] WARNING: $PUBLISHED_IMAGE appears locked (VM running?)." + echo "[build-custom-image] qemu-img info exit=$?; skipping publish." + echo "[build-custom-image] Shut down any VMs using it and re-run, or manually:" + echo "[build-custom-image] cp $CUSTOM_IMAGE $PUBLISHED_IMAGE" + fi +fi +if [ ! -f "$PUBLISHED_IMAGE" ] || qemu-img info "$PUBLISHED_IMAGE" >/dev/null 2>&1; then + if [ -f "$PUBLISHED_IMAGE" ]; then + echo "[build-custom-image] Publishing to $PUBLISHED_IMAGE (overwrite)..." + else + echo "[build-custom-image] Publishing to $PUBLISHED_IMAGE (new file)..." + fi + log_cmd cp "$CUSTOM_IMAGE" "$PUBLISHED_IMAGE" + if ! cp "$CUSTOM_IMAGE" "$PUBLISHED_IMAGE"; then + echo "[build-custom-image] ERROR: failed to copy image to $PUBLISHED_IMAGE (exit $?; disk full or file locked?)" >&2 + df -h "$(dirname "$PUBLISHED_IMAGE")" >&2 + echo "[build-custom-image] Build artifact remains at: $CUSTOM_IMAGE" >&2 + exit 1 + fi NEW_DIGEST=$(sha256sum "$PUBLISHED_IMAGE" | awk '{print $1}') + echo "[build-custom-image] published digest: sha256:$NEW_DIGEST" if [ -f "$PUBLISHED_JSON" ]; then echo "[build-custom-image] Updating digest in $PUBLISHED_JSON..." TMP_JSON=$(mktemp) - jq --arg d "$NEW_DIGEST" \ + if ! jq --arg d "$NEW_DIGEST" \ --arg p "$PUBLISHED_IMAGE" \ '.path = $p | .digest = ["Sha256", $d]' \ - "$PUBLISHED_JSON" > "$TMP_JSON" + "$PUBLISHED_JSON" > "$TMP_JSON"; then + echo "[build-custom-image] ERROR: jq failed updating $PUBLISHED_JSON (exit $?)" >&2 + rm -f "$TMP_JSON" + exit 1 + fi mv "$TMP_JSON" "$PUBLISHED_JSON" else echo "[build-custom-image] Creating $PUBLISHED_JSON..." @@ -371,8 +741,16 @@ else } JSON_EOF fi + echo "[build-custom-image] JSON updated: $(cat "$PUBLISHED_JSON")" +fi + +if [ -z "${NEW_DIGEST:-}" ]; then + NEW_DIGEST=$(sha256sum "$CUSTOM_IMAGE" | awk '{print $1}') fi +fix_qlean_ownership + +log_stage "done" echo "" echo "[build-custom-image] ===============================================" echo "[build-custom-image] Custom image build complete!" diff --git a/orion-scheduler/src/config.rs b/orion-scheduler/src/config.rs index e388005fc..3c3a9a5bd 100644 --- a/orion-scheduler/src/config.rs +++ b/orion-scheduler/src/config.rs @@ -34,6 +34,22 @@ pub struct Config { ssh_public_key_path: String, } +/// Expand a leading `~` or `~/` to `$HOME`. Other paths are returned unchanged. +pub fn expand_tilde(path: impl AsRef) -> PathBuf { + let path = path.as_ref(); + if path == "~" { + return std::env::var_os("HOME") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(path)); + } + if let Some(rest) = path.strip_prefix("~/") + && let Some(home) = std::env::var_os("HOME") + { + return PathBuf::from(home).join(rest); + } + PathBuf::from(path) +} + impl Config { /// Create a new Config with the given log directory and empty targets #[cfg(test)] @@ -210,3 +226,24 @@ struct ConfigFile { /// Global configuration state pub type SharedConfig = Arc>; + +#[cfg(test)] +mod tests { + use super::expand_tilde; + + #[test] + fn expand_tilde_home_prefix() { + let home = std::env::var("HOME").expect("HOME must be set in test env"); + let p = expand_tilde("~/.local/share/qlean/images/debian-13-buck2.qcow2"); + assert_eq!( + p, + std::path::PathBuf::from(home).join(".local/share/qlean/images/debian-13-buck2.qcow2") + ); + } + + #[test] + fn expand_tilde_absolute_unchanged() { + let abs = "/home/orion/image.qcow2"; + assert_eq!(expand_tilde(abs), std::path::PathBuf::from(abs)); + } +} diff --git a/orion-scheduler/src/handlers.rs b/orion-scheduler/src/handlers.rs index fc26d76ac..050dffc39 100644 --- a/orion-scheduler/src/handlers.rs +++ b/orion-scheduler/src/handlers.rs @@ -378,11 +378,11 @@ impl LogCursor { // First non-empty fetch: show recent activity without spamming. lines.len().saturating_sub(INITIAL_TAIL_LINES) } else { - // Resume right after the previous tail. If the source rolled - // past our entire fingerprint (huge burst, restart, rotation), - // silently re-anchor and wait for new lines on the next tick - // instead of replaying the whole window. - self.find_resume_index(lines).unwrap_or(lines.len()) + // Resume right after the previous tail. If the source rolled past our + // fingerprint (burst faster than the poll window), emit a recent tail + // so the stream stays live instead of going silent until the burst ends. + self.find_resume_index(lines) + .unwrap_or_else(|| lines.len().saturating_sub(INITIAL_TAIL_LINES)) }; self.refresh_fingerprint(lines); @@ -428,29 +428,27 @@ pub async fn logs_stream_handler( State(state): State>, ) -> Sse>> { let stream = async_stream::stream! { - let mut ticker = interval(std::time::Duration::from_secs(2)); + let mut ticker = interval(std::time::Duration::from_secs(1)); let mut journal_cursor = LogCursor::default(); - let mut orion_cursor = LogCursor::default(); + let mut orion_log_offset: u64 = 0; loop { ticker.tick().await; - let full_logs = match orion_deployer::get_live_logs(&state).await { - Ok(logs) => logs, + let snapshot = match orion_deployer::get_live_logs_since(&state, orion_log_offset).await { + Ok(snapshot) => snapshot, Err(e) => { yield Ok(Event::default().data(format!("Error: {}", e))); continue; } }; + orion_log_offset = snapshot.orion_log_offset; - let (journal_part, orion_part) = split_logs(&full_logs); - let journal_lines: Vec<&str> = journal_part.lines().collect(); - let orion_lines: Vec<&str> = orion_part.lines().collect(); - + let journal_lines: Vec<&str> = snapshot.journal_window.lines().collect(); let new_j = journal_cursor.advance(&journal_lines); - let new_o = orion_cursor.advance(&orion_lines); + let orion_lines: Vec<&str> = snapshot.orion_log_delta.lines().collect(); - if new_j.is_empty() && new_o.is_empty() { + if new_j.is_empty() && orion_lines.is_empty() { continue; } @@ -458,8 +456,8 @@ pub async fn logs_stream_handler( if !new_j.is_empty() { append_logs_section(&mut output, "SYSTEM LOGS", new_j); } - if !new_o.is_empty() { - append_logs_section(&mut output, "ORION LOGS", new_o); + if !orion_lines.is_empty() { + append_logs_section(&mut output, "ORION LOGS", &orion_lines); } yield Ok(Event::default().comment("---").data(output)); @@ -469,18 +467,6 @@ pub async fn logs_stream_handler( Sse::new(stream).keep_alive(axum::response::sse::KeepAlive::default()) } -/// Split combined logs into systemd journal and Orion log file sections -/// The separator is "========== Orion Log" -fn split_logs(full_logs: &str) -> (&str, &str) { - // Find the separator "========== Orion Log" - if let Some(pos) = full_logs.find("========== Orion Log") { - let journal = &full_logs[..pos]; - let orion = &full_logs[pos..]; - return (journal, orion); - } - (full_logs, "") -} - /// Append a log section with a title header and colored log lines to `output`. fn append_logs_section(output: &mut String, title: &str, lines: &[&str]) { use std::fmt::Write; diff --git a/orion-scheduler/src/orion_deployer.rs b/orion-scheduler/src/orion_deployer.rs index fb49e2829..50a333873 100644 --- a/orion-scheduler/src/orion_deployer.rs +++ b/orion-scheduler/src/orion_deployer.rs @@ -5,6 +5,7 @@ use tokio::fs; use tracing::info; use crate::{ + config::expand_tilde, handlers::ImageParams, keep_alive::{ImageSpec, KeepAliveMachine}, state::{AppState, VmInfo}, @@ -94,9 +95,25 @@ pub async fn handle_update( }) } (None, Some(path), Some(digest)) => { - info!("[orion-deploy] Using image from path: {}", path); + let expanded = expand_tilde(path); + let path_str = expanded.to_string_lossy().into_owned(); + if path != &path_str { + info!( + "[orion-deploy] Using image from path: {} (expanded from {})", + path_str, path + ); + } else { + info!("[orion-deploy] Using image from path: {}", path_str); + } + if !expanded.is_file() { + return Err(anyhow::anyhow!( + "image file does not exist: {} (from image_path: {})", + path_str, + path + )); + } Some(ImageSpec { - source: Some(path.clone()), + source: Some(path_str), digest: Some(digest.clone()), }) } @@ -150,38 +167,69 @@ pub async fn handle_update( Ok(vm_name) } -/// Get live Orion logs from the running VM (journalctl + orion.log) -pub async fn get_live_logs(state: &AppState) -> Result { +const ORION_LOG_PATH: &str = "/home/orion/orion-runner/log/orion.log"; +/// On the first SSE tick, only bootstrap the tail of orion.log instead of the +/// entire file (build logs can grow to hundreds of MB). +const ORION_LOG_BOOTSTRAP_BYTES: u64 = 65536; + +/// Incremental snapshot for `/logs/orion/stream`. +pub struct LiveLogSnapshot { + pub journal_window: String, + pub orion_log_delta: String, + pub orion_log_offset: u64, +} + +/// Get live Orion logs from the running VM (journalctl window + orion.log delta). +pub async fn get_live_logs_since( + state: &AppState, + orion_log_offset: u64, +) -> Result { let machine = state .get_machine() .await .ok_or_else(|| anyhow::anyhow!("No VM is currently running"))?; - info!("[orion-deploy] Fetching live Orion logs"); - - // Get recent journalctl logs + // Get recent journalctl logs (sliding window; deduped by the SSE handler). let output = machine - .exec("journalctl -u orion-runner --no-pager -n 100 2>&1") + .exec("journalctl -u orion-runner --no-pager -n 200 2>&1") .await?; - // Get Orion log file content - let orion_log_output = machine.exec("tail -100 /home/orion/orion-runner/log/orion.log 2>/dev/null || echo 'Orion log not found'").await?; - - // Get process info - let process_output = machine - .exec("pgrep -a orion || echo 'Orion process not found'") + let size_output = machine + .exec(&format!("stat -c%s {ORION_LOG_PATH} 2>/dev/null || echo 0")) .await?; + let file_size: u64 = String::from_utf8_lossy(&size_output.stdout) + .trim() + .parse() + .unwrap_or(0); + + let (orion_log_delta, new_offset) = if file_size < orion_log_offset { + // Log rotated/truncated — re-read from the start. + let out = machine + .exec(&format!("cat {ORION_LOG_PATH} 2>/dev/null")) + .await?; + (String::from_utf8_lossy(&out.stdout).into_owned(), file_size) + } else if file_size > orion_log_offset { + let read_from = if orion_log_offset == 0 { + file_size.saturating_sub(ORION_LOG_BOOTSTRAP_BYTES) + } else { + orion_log_offset + }; + let cmd = if read_from == 0 { + format!("cat {ORION_LOG_PATH} 2>/dev/null") + } else { + format!("tail -c +{} {ORION_LOG_PATH} 2>/dev/null", read_from + 1) + }; + let out = machine.exec(&cmd).await?; + (String::from_utf8_lossy(&out.stdout).into_owned(), file_size) + } else { + (String::new(), orion_log_offset) + }; - let journal_logs = String::from_utf8_lossy(&output.stdout); - let orion_logs = String::from_utf8_lossy(&orion_log_output.stdout); - let process_info = String::from_utf8_lossy(&process_output.stdout); - - let logs = format!( - "{}\n\n========== Orion Log (/home/orion/orion-runner/log/orion.log) ==========\n{}\n\n[Orion Process Info]\n{}", - journal_logs, orion_logs, process_info - ); - - Ok(logs) + Ok(LiveLogSnapshot { + journal_window: String::from_utf8_lossy(&output.stdout).into_owned(), + orion_log_delta, + orion_log_offset: new_offset, + }) } /// Get current VM status diff --git a/orion-scheduler/src/vm_manager.rs b/orion-scheduler/src/vm_manager.rs index 44b72cb61..984210836 100644 --- a/orion-scheduler/src/vm_manager.rs +++ b/orion-scheduler/src/vm_manager.rs @@ -3,7 +3,10 @@ use std::path::PathBuf; use anyhow::Result; use tracing::info; -use crate::{config::TargetConfig, keep_alive::KeepAliveMachine}; +use crate::{ + config::{TargetConfig, expand_tilde}, + keep_alive::KeepAliveMachine, +}; /// The target directory inside the VM guest OS where Orion is deployed const ORION_TARGET_DIR: &str = "/home/orion/orion-runner"; @@ -214,7 +217,7 @@ pub async fn inject_ssh_keys(machine: &KeepAliveMachine, ssh_public_key_path: &s info!("[ssh] Injecting SSH keys for debugging access"); // Read the extra public key from a file - let extra_key_path = std::path::Path::new(ssh_public_key_path); + let extra_key_path = expand_tilde(ssh_public_key_path); let extra_key = if extra_key_path.exists() { tokio::fs::read_to_string(extra_key_path) .await? diff --git a/orion/Cargo.toml b/orion/Cargo.toml index f427eb478..4f7b7f356 100644 --- a/orion/Cargo.toml +++ b/orion/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "orion" -version = "0.1.1" +version = "0.1.2" edition = "2024" [[bin]] @@ -29,8 +29,6 @@ parse-display = "0.11.0" audit = { path = "./audit" } td_util = { path = "./td_util" } td_util_buck = { path = "./buck" } -thiserror = { workspace = true } -utoipa.workspace = true common = { path = "../common" } tokio-util = { workspace = true } @@ -39,4 +37,4 @@ serial_test = { workspace = true } tempfile = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] -scorpiofs = { git = "https://github.com/web3infra-foundation/scorpiofs.git", rev = "900309f0abc397ccf8b9565ae0da2e3ef1e65618" } +scorpiofs = { git = "https://github.com/web3infra-foundation/scorpiofs.git" } diff --git a/orion/buck/discovery_scope.rs b/orion/buck/discovery_scope.rs new file mode 100644 index 000000000..3cde630ca --- /dev/null +++ b/orion/buck/discovery_scope.rs @@ -0,0 +1,416 @@ +//! Narrow `buck2 targets` discovery to subtrees touched by the change list (scheme A), +//! with optional `buck2 uquery rdeps` expansion over the full cell universe (scheme D). + +use std::{ + collections::{HashMap, HashSet}, + path::{Path, PathBuf}, +}; + +use api_model::buck2::{status::Status, types::ProjectRelativePath}; + +use crate::{ + cells::CellInfo, + types::{CellName, CellPath}, +}; + +/// Result of computing which target patterns to pass to `buck2 targets`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DiscoveryScope { + /// Patterns such as `root//rk8s/...` for narrowed discovery. + pub query_patterns: Vec, + /// `true` when patterns are a strict subset of the full cell scan. + pub narrow: bool, +} + +/// A monorepo subdirectory that carries its own `.buckconfig` (e.g. `rk8s/`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SubprojectBuckRoot { + pub buck_root: PathBuf, + pub strip_prefix: String, +} + +/// When every changed path lives under one directory that has its own `.buckconfig`, +/// run Buck2 discovery from that directory so `PACKAGE` / cell config match the sub-project. +pub fn detect_subproject_buck_root( + project_root: &Path, + changes: &[Status], +) -> Option { + if changes.is_empty() { + return None; + } + + let first = changes.first()?.get().as_str(); + let prefix = first + .split('/') + .next() + .filter(|segment| !segment.is_empty())?; + let prefix_with_slash = format!("{prefix}/"); + + for change in changes { + let path = change.get().as_str(); + if path != prefix && !path.starts_with(&prefix_with_slash) { + return None; + } + } + + let buck_root = project_root.join(prefix); + if !buck_root.join(".buckconfig").is_file() { + return None; + } + + Some(SubprojectBuckRoot { + buck_root, + strip_prefix: prefix.to_owned(), + }) +} + +pub fn strip_subproject_path_prefix( + path: &ProjectRelativePath, + prefix: &str, +) -> ProjectRelativePath { + let path_str = path.as_str(); + if path_str == prefix { + return ProjectRelativePath::new(""); + } + if let Some(rest) = path_str + .strip_prefix(prefix) + .and_then(|s| s.strip_prefix('/')) + { + ProjectRelativePath::new(rest) + } else { + path.clone() + } +} + +pub fn strip_subproject_changes( + changes: &[Status], + prefix: &str, +) -> Vec> { + changes + .iter() + .map(|change| { + change + .clone() + .into_map(|path| strip_subproject_path_prefix(&path, prefix)) + }) + .collect() +} + +/// Whether path-scoped discovery is enabled (scheme A + D). +/// +/// Disable with `ORION_DISCOVERY_SCOPE=0`, `false`, `no`, or `off`. +pub fn discovery_scope_enabled() -> bool { + match std::env::var("ORION_DISCOVERY_SCOPE") { + Ok(value) => { + let normalized = value.trim().to_ascii_lowercase(); + !(normalized.is_empty() + || normalized == "0" + || normalized == "false" + || normalized == "no" + || normalized == "off") + } + Err(_) => true, + } +} + +/// Compute narrowed `buck2 targets` query patterns from the change list. +/// +/// When narrowing applies, only first-level directory segments under each touched cell +/// are queried (e.g. `root//rk8s/...` instead of `root//...`). Repo-root `.buckconfig` +/// changes still trigger a full scan. +pub fn compute_discovery_scope( + cells: &CellInfo, + project_root: &Path, + changes: &[Status], +) -> DiscoveryScope { + compute_discovery_scope_inner(cells, project_root, changes, discovery_scope_enabled()) +} + +fn compute_discovery_scope_inner( + cells: &CellInfo, + project_root: &Path, + changes: &[Status], + enabled: bool, +) -> DiscoveryScope { + let full_patterns = cells.get_all_cell_patterns(project_root); + if !enabled || changes.is_empty() { + return DiscoveryScope { + query_patterns: full_patterns.clone(), + narrow: false, + }; + } + + let mut full_cell: HashSet = HashSet::new(); + let mut segments: HashMap> = HashMap::new(); + let mut resolved_any = false; + let mut force_full_scan = false; + + for change in changes { + let path = change.get(); + let cell_path = match cells.unresolve(path) { + Ok(cell_path) => cell_path, + Err(_) => continue, + }; + resolved_any = true; + + if is_repo_root_config(&cell_path) { + force_full_scan = true; + break; + } + + if is_cell_wide_scan_path(&cell_path) { + full_cell.insert(cell_path.cell()); + continue; + } + + let path = cell_path.path(); + let rel = path.as_str(); + if let Some(segment) = first_path_segment(rel) { + segments + .entry(cell_path.cell()) + .or_default() + .insert(segment.to_owned()); + } else { + full_cell.insert(cell_path.cell()); + } + } + + if !resolved_any || force_full_scan { + return DiscoveryScope { + query_patterns: full_patterns.clone(), + narrow: false, + }; + } + + let mut query_patterns = Vec::new(); + for cell_name in cells_with_changes(&full_cell, &segments) { + let cell = cell_name.as_str(); + if full_cell.contains(&cell_name) { + query_patterns.push(format!("{cell}//...")); + continue; + } + if let Some(segs) = segments.get(&cell_name) { + let mut sorted: Vec<_> = segs.iter().cloned().collect(); + sorted.sort(); + for segment in sorted { + query_patterns.push(format!("{cell}//{segment}/...")); + } + } + } + + if query_patterns.is_empty() { + return DiscoveryScope { + query_patterns: full_patterns.clone(), + narrow: false, + }; + } + + query_patterns.sort(); + query_patterns.dedup(); + + let narrow = query_patterns != full_patterns; + DiscoveryScope { + query_patterns, + narrow, + } +} + +fn cells_with_changes( + full_cell: &HashSet, + segments: &HashMap>, +) -> Vec { + let mut cells: HashSet<_> = full_cell.iter().cloned().collect(); + cells.extend(segments.keys().cloned()); + let mut sorted: Vec<_> = cells.into_iter().collect(); + sorted.sort_by_key(|cell| cell.as_str().to_owned()); + sorted +} + +fn is_repo_root_config(cell_path: &CellPath) -> bool { + let path = cell_path.path(); + cell_path.cell().as_str() == "root" + && matches!(path.as_str(), ".buckconfig" | ".buckroot" | ".buckversion") +} + +/// Paths that affect an entire cell (top-level BUCK / cell config), but not repo-root config. +fn is_cell_wide_scan_path(cell_path: &CellPath) -> bool { + let path = cell_path.path(); + let rel = path.as_str(); + if rel.is_empty() { + return true; + } + if rel.contains('/') { + return false; + } + matches!( + rel, + ".buckconfig" | ".buckroot" | ".buckversion" | "BUCK" | "TARGETS" | "BUCK.v2" + ) +} + +fn first_path_segment(rel: &str) -> Option<&str> { + rel.split('/').next().filter(|segment| !segment.is_empty()) +} + +#[cfg(test)] +mod tests { + use std::{env, fs, path::PathBuf}; + + use super::*; + + fn test_cells(project_root: &Path) -> CellInfo { + let cell_json = serde_json::json!({ + "root": project_root.to_str().unwrap(), + "toolchains": project_root.join("toolchains").to_str().unwrap(), + "prelude": project_root.join("prelude").to_str().unwrap(), + }); + CellInfo::parse(&serde_json::to_string(&cell_json).unwrap()).unwrap() + } + + fn temp_project_root() -> PathBuf { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = env::temp_dir().join(format!("discovery_scope_test_{nanos}")); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + fs::create_dir_all(dir.join("toolchains")).unwrap(); + fs::create_dir_all(dir.join("prelude")).unwrap(); + fs::create_dir_all(dir.join("rk8s")).unwrap(); + dir + } + + #[test] + fn test_rk8s_only_changes_narrow_to_subtree() { + let root = temp_project_root(); + let cells = test_cells(&root); + let changes = vec![Status::Added(ProjectRelativePath::new( + "rk8s/project/libfuse-fs/src/foo.rs", + ))]; + + let scope = compute_discovery_scope(&cells, &root, &changes); + assert!(scope.narrow); + assert_eq!(scope.query_patterns, vec!["root//rk8s/...".to_string()]); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn test_repo_root_buckconfig_forces_full_scan() { + let root = temp_project_root(); + let cells = test_cells(&root); + let changes = vec![Status::Modified(ProjectRelativePath::new(".buckconfig"))]; + + let scope = compute_discovery_scope(&cells, &root, &changes); + assert!(!scope.narrow); + assert!(scope.query_patterns.contains(&"root//...".to_string())); + assert!(scope + .query_patterns + .contains(&"toolchains//...".to_string())); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn test_subproject_buckconfig_narrows_to_segment() { + let root = temp_project_root(); + let cells = test_cells(&root); + let changes = vec![Status::Modified(ProjectRelativePath::new( + "rk8s/.buckconfig", + ))]; + + let scope = compute_discovery_scope(&cells, &root, &changes); + assert!(scope.narrow); + assert_eq!(scope.query_patterns, vec!["root//rk8s/...".to_string()]); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn test_toolchains_cell_change_includes_toolchains_pattern() { + let root = temp_project_root(); + let cells = test_cells(&root); + let changes = vec![Status::Modified(ProjectRelativePath::new( + "toolchains/BUCK", + ))]; + + let scope = compute_discovery_scope(&cells, &root, &changes); + assert!(scope.narrow); + assert_eq!(scope.query_patterns, vec!["toolchains//...".to_string()]); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn test_multiple_segments_under_root() { + let root = temp_project_root(); + let cells = test_cells(&root); + let changes = vec![ + Status::Added(ProjectRelativePath::new("rk8s/a.rs")), + Status::Added(ProjectRelativePath::new("third-party/b.rs")), + ]; + + let scope = compute_discovery_scope(&cells, &root, &changes); + assert!(scope.narrow); + assert_eq!( + scope.query_patterns, + vec![ + "root//rk8s/...".to_string(), + "root//third-party/...".to_string(), + ] + ); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn test_disabled_via_env_uses_full_scan() { + let root = temp_project_root(); + let cells = test_cells(&root); + let changes = vec![Status::Added(ProjectRelativePath::new( + "rk8s/project/foo.rs", + ))]; + + let scope = compute_discovery_scope_inner(&cells, &root, &changes, false); + assert!(!scope.narrow); + assert!(scope.query_patterns.contains(&"root//...".to_string())); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn test_empty_changes_use_full_scan() { + let root = temp_project_root(); + let cells = test_cells(&root); + + let scope = compute_discovery_scope(&cells, &root, &[]); + assert!(!scope.narrow); + assert!(scope.query_patterns.contains(&"root//...".to_string())); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn test_detect_subproject_buck_root_for_rk8s() { + let root = temp_project_root(); + fs::write(root.join("rk8s/.buckconfig"), "[build]\n").unwrap(); + + let changes = vec![Status::Added(ProjectRelativePath::new( + "rk8s/project/common/src/lib.rs", + ))]; + let sub = detect_subproject_buck_root(&root, &changes).expect("rk8s subproject"); + assert_eq!(sub.buck_root, root.join("rk8s")); + assert_eq!(sub.strip_prefix, "rk8s"); + + let stripped = strip_subproject_changes(&changes, "rk8s"); + assert_eq!(stripped[0].get().as_str(), "project/common/src/lib.rs"); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn test_detect_subproject_rejects_mixed_prefixes() { + let root = temp_project_root(); + fs::write(root.join("rk8s/.buckconfig"), "[build]\n").unwrap(); + + let changes = vec![ + Status::Added(ProjectRelativePath::new("rk8s/a.rs")), + Status::Added(ProjectRelativePath::new("third-party/b.rs")), + ]; + assert!(detect_subproject_buck_root(&root, &changes).is_none()); + let _ = fs::remove_dir_all(&root); + } +} diff --git a/orion/buck/lib.rs b/orion/buck/lib.rs index 89b3765fa..a59b02431 100644 --- a/orion/buck/lib.rs +++ b/orion/buck/lib.rs @@ -33,11 +33,13 @@ impl ExitStatusExt for ExitStatus { pub mod cells; pub mod config; +pub mod discovery_scope; pub mod glob; pub mod ignore_set; pub mod labels; pub mod owners; pub mod package_resolver; +pub mod platform; pub mod run; pub mod target_graph; pub mod target_map; diff --git a/orion/buck/platform.rs b/orion/buck/platform.rs new file mode 100644 index 000000000..39fba463b --- /dev/null +++ b/orion/buck/platform.rs @@ -0,0 +1,52 @@ +//! Scheme C2: ensure every cell resolves a target platform instead of ``. +//! +//! Orion C1 removed CLI `--target-platforms`; platform must come from `.buckconfig`. +//! Repos with an incomplete `target_platform_detector_spec` (e.g. only `root//...`) +//! leave `buckal//tool:manifest_parse` unconfigured and fail analysis with +//! `manifest_parse ()`. + +use std::process::Command; + +/// Global default when a target has no per-target `default_target_platform`. +pub const DEFAULT_TARGET_PLATFORM: &str = "prelude//platforms:default"; + +/// Maps all monorepo cells to the shared default platform. +pub const TARGET_PLATFORM_DETECTOR_SPEC: &str = "\ +target:root//...->prelude//platforms:default \ +target:prelude//...->prelude//platforms:default \ +target:toolchains//...->prelude//platforms:default \ +target:buckal//...->prelude//platforms:default"; + +/// Append Buck2 `--config` overrides so platform resolution works even when the +/// checked-in `.buckconfig` is incomplete (read-only on Antares mounts). +pub fn append_platform_config(command: &mut Command) { + command + .arg("--config") + .arg(format!( + "build.default_target_platforms={DEFAULT_TARGET_PLATFORM}" + )) + .arg("--config") + .arg(format!( + "parser.target_platform_detector_spec={TARGET_PLATFORM_DETECTOR_SPEC}" + )); +} + +/// Config key/value pairs for async command builders (e.g. `tokio::process::Command`). +pub fn platform_config_flags() -> [String; 4] { + [ + "--config".to_owned(), + format!("build.default_target_platforms={DEFAULT_TARGET_PLATFORM}"), + "--config".to_owned(), + format!("parser.target_platform_detector_spec={TARGET_PLATFORM_DETECTOR_SPEC}"), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detector_spec_covers_buckal_cell() { + assert!(TARGET_PLATFORM_DETECTOR_SPEC.contains("target:buckal//...")); + } +} diff --git a/orion/buck/run.rs b/orion/buck/run.rs index 7fce3b499..e8bd6a555 100644 --- a/orion/buck/run.rs +++ b/orion/buck/run.rs @@ -21,7 +21,8 @@ use tracing::info; use crate::{ cells::CellInfo, - types::{Package, TargetPattern}, + platform::append_platform_config, + types::{Package, TargetLabel, TargetPattern}, ExitStatusExt, }; @@ -150,6 +151,7 @@ impl Buck2 { .arg(output) .arg(at_file.clone()) .args(extra_args); + append_platform_config(&mut command); command })?; res.status.exit_result().context("buck2 targets failed")?; @@ -181,6 +183,78 @@ impl Buck2 { info!("Running owners query"); Ok(String::from_utf8(res.stdout)?) } + + /// Reverse-deps of `seeds` within `universe_patterns`, via `buck2 uquery rdeps`. + /// + /// Buck2 defines `rdeps(universe, targets, ...)`: search for reverse dependencies + /// of `targets` only within `universe`. Use `%Ss` so each `@` file is expanded + /// into a single `set(...)` for one query (not `%s`, which runs one query per line + /// and groups JSON output by input literal). + pub fn uquery_rdeps( + &mut self, + seeds: &[TargetLabel], + universe_patterns: &[String], + ) -> anyhow::Result> { + if seeds.is_empty() || universe_patterns.is_empty() { + return Ok(Vec::new()); + } + + let seed_exprs: Vec = seeds + .iter() + .map(|label| label.as_str().to_owned()) + .collect(); + let (_seed_file, at_seeds) = create_at_file_arg(&seed_exprs, "\n")?; + let (_universe_file, at_universe) = create_at_file_arg(universe_patterns, "\n")?; + + let root = self.root()?; + let res = self.run_output_with_retry(|| { + let mut command = self.command(); + command + .arg("uquery") + .arg("--json") + .arg("rdeps(%Ss, %Ss)") + .arg(&at_universe) + .arg(&at_seeds); + append_platform_config(&mut command); + command.current_dir(&root); + command + })?; + + parse_uquery_rdeps_labels(&String::from_utf8(res.stdout)?) + } +} + +fn parse_uquery_rdeps_labels(json_str: &str) -> anyhow::Result> { + let raw: serde_json::Value = serde_json::from_str(json_str)?; + let mut labels = match raw { + serde_json::Value::Array(items) => items + .into_iter() + .filter_map(|v| v.as_str().map(TargetLabel::new)) + .collect(), + serde_json::Value::Object(map) => { + // Multi-query `%s` groups results by input literal; collect targets from + // values, not keys (keys may be universe patterns like `root//...`). + if map.values().all(|v| v.is_array()) { + map.into_values() + .flat_map(|v| { + v.as_array() + .map(|arr| { + arr.iter() + .filter_map(|item| item.as_str().map(TargetLabel::new)) + .collect::>() + }) + .unwrap_or_default() + }) + .collect() + } else { + map.keys().map(|key| TargetLabel::new(key)).collect() + } + } + _ => Vec::new(), + }; + labels.sort(); + labels.dedup(); + Ok(labels) } fn should_retry_buck2_daemon(stderr: &str) -> bool { @@ -193,8 +267,6 @@ fn should_retry_buck2_daemon(stderr: &str) -> bool { pub fn targets_arguments() -> &'static [&'static str] { &[ "targets", - "--target-platforms", - "prelude//platforms:default", "--streaming", "--keep-going", "--no-cache", @@ -205,3 +277,41 @@ pub fn targets_arguments() -> &'static [&'static str] { "--package-values-regex=^citadel\\.labels$|^test_config_unification\\.rollout$", ] } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_uquery_rdeps_labels_accepts_target_array() { + let json = r#"["root//rk8s/lib:foo", "root//rk8s/bin:bar"]"#; + let labels = parse_uquery_rdeps_labels(json).unwrap(); + assert_eq!(labels.len(), 2); + assert!(labels.contains(&TargetLabel::new("root//rk8s/lib:foo"))); + assert!(labels.contains(&TargetLabel::new("root//rk8s/bin:bar"))); + } + + #[test] + fn parse_uquery_rdeps_labels_flattens_multi_query_grouped_output() { + let json = r#"{ + "root//rk8s/...": ["root//rk8s/lib:foo", "root//rk8s/bin:bar"], + "root//other/...": ["root//other:lib"] + }"#; + let labels = parse_uquery_rdeps_labels(json).unwrap(); + assert_eq!(labels.len(), 3); + assert!(!labels.iter().any(|l| l.as_str().contains("..."))); + assert!(labels.contains(&TargetLabel::new("root//rk8s/lib:foo"))); + assert!(labels.contains(&TargetLabel::new("root//other:lib"))); + } + + #[test] + fn parse_uquery_rdeps_labels_reads_target_keys_from_attribute_map() { + let json = r#"{ + "root//rk8s/lib:foo": {"name": "foo"}, + "root//rk8s/bin:bar": {"name": "bar"} + }"#; + let labels = parse_uquery_rdeps_labels(json).unwrap(); + assert_eq!(labels.len(), 2); + assert!(labels.contains(&TargetLabel::new("root//rk8s/lib:foo"))); + } +} diff --git a/orion/docs/FUSE_MOUNT_ISSUES.md b/orion/docs/FUSE_MOUNT_ISSUES.md new file mode 100644 index 000000000..be2f63b7e --- /dev/null +++ b/orion/docs/FUSE_MOUNT_ISSUES.md @@ -0,0 +1,174 @@ +# Antares FUSE 挂载问题 + +Orion Worker 长构建（Buck2 + Antares overlay）在 `/data/scorpio/antares/mnt/-N/` 上曾出现 **FUSE ENOENT**，导致全量 `rustc` 构建失败（exit 3）。Build **#45** 在 discovery 修复后首次在同一 CL 上 **0× ENOENT、exit 0** 完成全量编译；本文档记录现象、与 discovery 的交互、已落地补丁与验收结果。 + +Target discovery 收窄与 all-added 子项目逻辑见 [TARGET_DISCOVERY_SCOPE.md](./TARGET_DISCOVERY_SCOPE.md)。 + +--- + +## 构建记录 + +### 早期（discovery 未收窄 + JVM toolchain） + +| 构建 | 特点 | 结果 | +|------|------|------| +| #21–#23 | ~35–49 min；`linker_wrapper` ENOENT 5–7% | FAILED | + +### CL `UYXIYYNJ`（`rk8s/**` 子项目） + +| 构建 | Task ID（前缀） | Action / Commands | `os error 2` | 结果 | +|------|----------------|-------------------|--------------|------| +| #39 | `019ef246` | ~39k | — | 失败（discovery 范围过大） | +| #40 | `019ef252` | ~18k，真 `rustc` | 有 | **FAILED**（FUSE ENOENT） | +| #42 | `019ef325` | ~113，仅 `:vendor` | 0 | exit 0，**未编译**（discovery 误选，已修） | +| **#45** | `019ef333` | **23,270** action；**8,035** local commands | **0** | **exit 0，全量 `rustc` 通过** | + +#### Build #45 摘要（`019ef333-db97-70e2-8960-40bcc5cc2496`，VM `192.168.221.108`） + +- **总时长** ~73 min（其中 buck2 build ~72 min）；discovery 28 targets，自 `rk8s/` 子项目根构建。 +- **FUSE 构建期**：`os error 2` **0**、`Action failed` **0**、`` **0**（日志共 17,697 行含该 task）。 +- **编译证据**：日志中 **3,025** 处 `rustc` 相关 action；收尾为 `root//project/rkl:rkl`、`rkforge`、`slayerfs` 的 `rustc link [pic]`。 +- **缓存**：`Cache hits: 0%`；`Commands: 8035 (cached: 0, remote: 0, local: 8035)`（`--no-remote-cache` + 新 isolation-dir）。 +- **收尾**：构建成功后 `fusermount -u` 报 **Device or resource busy**，fallback 后 unmount 成功（不影响 exit 0）。 + +#23 指标（历史参考）：`os error 2` **80**、`Action failed` **80**。 + +--- + +## 问题描述 + +### 现象 A：写后立刻 stat ENOENT + +buck2 写出 `linker_wrapper.sh` 后，紧接 `metadata()` / `set_permissions()` 报 ENOENT： + +``` +`write_file` setting executable `.../buck-out/buck-isolation-.../linker_wrapper.sh`: + metadata(.../linker_wrapper.sh): No such file or directory (os error 2) +``` + +- 路径：`buck-out/buck-isolation-.../art/root/third-party/rust/crates/**` 或 `project/**`（upper passthrough） +- 多出现在高并发 action burst 初期 + +### 现象 B：延迟访问 shim ENOENT + +`write __cc_shim.sh` / `__cxx_shim.sh` 在数分钟至数十分钟后，cc-rs 执行时报： + +``` +error occurred in cc-rs: failed to find tool ".../__cc_shim.sh": No such file or directory (os error 2) +``` + +典型 crate：`libsqlite3-sys`、`ring`、`zstd-sys` 等。写出与执行间隔可达 **~40 min**。可用 `ORION_RETAIN_ANTARES_MOUNTS=1` 对比 upper 与 mnt 是否「磁盘有、FUSE 无」。 + +### 根因假设（未完全证实） + +`/data/scorpio/antares/mnt/...` = dicfuse lower + passthrough upper。FORGET 后内存 inode 与 upper 磁盘可能脱节；补丁将早期 ~72 次 ENOENT/构建降至个位数百分比，但未清零： + +1. 高并发 `linker_wrapper` burst 下 overlay lookup / dentry 仍偶发 ENOENT。 +2. shim 长间隔 `open`/`access`，`materialize_node_by_path` 可能未覆盖实际路径。 +3. 隔离压测（仅写 buck-out）0 失败；全量构建含 dicfuse 读压 + buck2 daemon + 大量 action。 + +--- + +## Discovery 与 FUSE 的交互 + +| Discovery 行为 | 对 FUSE 压力的影响 | +|----------------|-------------------| +| 全 cell + `SelectAll`（#39） | 数万 vendor unpack action，放大写 burst | +| `ScopedNew` / 真 `rustc`（#40） | action 减少但仍大量编译，ENOENT 暴露 | +| owner → `:vendor` only（#42） | action 极少，**绕过**编译，不能验证 FUSE + rustc | +| `project/` only + rust 映射（#45） | 28 个 `rust_*` 根 + 传递 `third-party` 依赖；23k action、~72 min；**本次 0× ENOENT** | + +收窄 discovery **不能**单独解决 ENOENT，但可减少无关 action。#45 表明在 discovery 修复 + 现有 unionfs 补丁下，**单次全量 `rustc` 构建可以不在 FUSE 上触发 ENOENT**；仍须更多构建观察是否偶发回归。 + +--- + +## 已落地补丁 + +| 改动 | 位置 | 说明 | +|------|------|------| +| `materialize_child_from_layers` | `rk8s/.../unionfs/mod.rs` | ENOENT 部分缓解 | +| `do_lookup` 在 `stat64` 前 pin | 同上 | 同上 | +| `materialize_node_by_path` + `inode_paths` | `unionfs/mod.rs` + `inode_store.rs` | 延迟 shim 仍可能失败 | +| discovery 后卸载 `old` mount | `orion/src/buck_controller.rs` | 释放双挂载；不消除 ENOENT | +| `writeback=false`、TTL 5s / dicfuse 60s | passthrough + `scorpio.toml` | EBADF 已解决 | +| Scheme C2 platform | `orion/buck/platform.rs` `--config` | `` 已解决 | +| Target discovery A/B/子项目 | `discovery_scope.rs`、`buck_controller.rs` | 减少无关 action；见姊妹文档 | + +--- + +## 待验证方向 + +1. `ORION_RETAIN_ANTARES_MOUNTS=1` 在**失败**构建后对比 upper 与 mnt 上 shim / `linker_wrapper`（#45 已成功，暂无失败样本）。 +2. unionfs/passthrough 在 `open`/`getattr` 路径强制回落 upper backing layer。 +3. 多次复跑 `UYXIYYNJ` / 其他 CL，观察 ENOENT 是否偶发回归（#45 为单点成功）。 +4. `ORION_BUCK_REMOTE_CACHE=1` 对重复构建耗时与 FUSE 读压的影响（#45 为 0% cache，~73 min）。 +5. 构建后 `fusermount` **EBUSY** 与 fuse task 5s 超时（#45 出现，unmount 仍成功）。 + +--- + +## 构建验收 + +Worker 日志：`/home/orion/orion-runner/log/orion.log`（或 `GET /logs/orion/stream`）。 + +```bash +TASK= +LOG=/home/orion/orion-runner/log/orion.log + +# ENOENT / 失败 action +grep "$TASK" "$LOG" | grep -c "os error 2" +grep "$TASK" "$LOG" | grep -c "Action failed" + +# 是否真在编译（不应只有 :vendor symlink） +grep "$TASK" "$LOG" | grep "buck2 stderr" | grep -E "rustc|rust_library|rust_binary" | head + +# discovery 目标（应为 rust_* 而非 :vendor） +grep "$TASK" "$LOG" | grep -E "Target discovery|owner_seed|in-graph rdeps" +``` + +--- + +## 回归压测（`scorpiofs/tests/antares_test.rs`） + +需 root/`/dev/fuse`；VM 上 `systemctl stop orion-runner` 后运行。 + +| 测试 | 说明 | +|------|------| +| `test_fuse_write_then_metadata` | 写后立即 stat | + +--- + +## 相关文件 + +| 路径 | 说明 | +|------|------| +| `rk8s/project/libfuse-fs/src/unionfs/mod.rs` | materialize、`do_lookup` | +| `rk8s/project/libfuse-fs/src/unionfs/inode_store.rs` | `inode_paths` | +| `scorpiofs/src/antares/fuse.rs` | Antares FUSE 集成 | +| `orion/src/buck_controller.rs` | 双挂载、卸 old、`ORION_RETAIN_ANTARES_MOUNTS` | +| `scorpiofs/tests/antares_test.rs` | FUSE 压测 | +| [orion/docs/TARGET_DISCOVERY_SCOPE.md](./TARGET_DISCOVERY_SCOPE.md) | discovery 收窄与 #39–#42 | + +--- + +## 已关闭问题 + +| 问题 | 验证 | +|------|------| +| `event.jsonl` EBADF | #21–#23：0× `os error 9` | +| `manifest_parse ()` | C2 platform 注入后：0× `` | +| `buck2 build --config` CLI | 已修 | +| TTL=0 吞吐劣化 | 勿将 TTL 设为 0 | +| 从 monorepo 根跑 rk8s buck2 | 子项目 `.buckconfig` 检测（#37） | +| #42 浅层成功（仅 `:vendor`） | `normalize_owner_targets_to_rust` | +| #45 全量 `rustc` on FUSE | `019ef333`：0× ENOENT，exit 0（单点验证） | + +--- + +## 修订历史 + +| 日期 | 说明 | +|------|------| +| 2026-06-15 | 初稿 | +| 2026-06-17–18 | ENOENT 根因、补丁、压测；#22/#23 | +| 2026-06-23 | 补充 #39–#42、discovery 与 FUSE 关系、验收命令 | +| 2026-06-23 | **#45**：全量 rustc 首次 exit 0、0× ENOENT；fusermount EBUSY 收尾 | diff --git a/orion/docs/TARGET_DISCOVERY_SCOPE.md b/orion/docs/TARGET_DISCOVERY_SCOPE.md new file mode 100644 index 000000000..162d2a174 --- /dev/null +++ b/orion/docs/TARGET_DISCOVERY_SCOPE.md @@ -0,0 +1,220 @@ +# Target Discovery 扫描范围过大问题分析 + +本文档记录 Orion worker 在做 Buck2 增量构建时，**target discovery（目标发现）与影响传播范围过大**的问题：一个纯 Rust 改动（如 `rk8s`）曾拉起无关 JVM/Android toolchain，或扫入整个 `third-party/**`，导致构建失败或耗时过长。 + +涉及代码主要在 `orion` crate（worker 端）。本文作为问题归档、修改方案与**落地进展**说明。 + +> 重要澄清：`project/buck2_test/toolchains:jdk_system_image`、`__android_sdk_tools__` 等 **不代表 `buck2_test` 业务代码依赖 JVM/Android**。它们只是 JVM/Android toolchain / platform helper target 的**定义位置**恰好在 `project/buck2_test/toolchains` 这个包下。这些 helper 被拉进来，通常是因为默认 platform 为 `prelude//platforms:default`，且影响传播曾不过滤 toolchain/platform 节点——而不是 `rk8s` 真的需要 Java/Android。 + +--- + +## 进展摘要（截至 2026-06-23） + +| 方案 | 内容 | 状态 | 说明 | +|------|------|------|------| +| **B** | 过滤 toolchain/platform helper 的传播与 build 选集 | **已落地** | `buck_controller.rs`：`is_toolchain_or_platform_*` | +| **C1** | Orion 不再 CLI 强制 `--target-platforms` | **已落地** | platform 由 `.buckconfig` + Orion `--config` 决定 | +| **C2** | 补全 platform 映射 + Orion `--config` 兜底 | **已落地** | `orion/buck/platform.rs`；挂载只读时覆盖不完整 `.buckconfig` | +| **A** | 按改动路径收窄 `buck2 targets` 查询范围 | **已落地** | `orion/buck/discovery_scope.rs`；`ORION_DISCOVERY_SCOPE=0` 可关 | +| **D** | A + rdeps 补齐反向依赖 | **部分落地** | narrow scope 用 `uquery rdeps`；all-added 子项目用**图 rdeps** | +| **子项目根** | 检测 `rk8s/.buckconfig` 等，从子目录跑 buck2 | **已落地** | `detect_subproject_buck_root()` | +| **All-added 子项目** | 全新增 CL 只 build `project/` crate | **已落地** | owner 播种 + `normalize_owner_targets_to_rust` | +| **E** | 收敛 all-added 空 base 的 SelectAll | **已落地** | 任意 all-added CL 跳过图 `SelectAll`，改 `owner()` 播种 | + +### CL `UYXIYYNJ`（`rk8s/**` 全量导入）构建对照 + +| Build | Task ID（前缀） | Discovery | Build 规模 | 结果 | +|-------|----------------|-----------|------------|------| +| #35–#36 | — | 0 target | 跳过 build | 假成功（`finish_without_build` exit 0） | +| #37 | — | 重试 | — | 从 monorepo 根跑 buck2，`set_cfg_constructor` 失败 | +| #39 | `019ef246` | `SelectAll` 全图 | ~39k action | 失败（范围过大） | +| #40 | `019ef252` | `ScopedNew` | ~18k action，真 `rustc` | 失败（FUSE ENOENT） | +| #42 | `019ef325` | owner → **17× `:vendor`** | ~113 action，无 `rustc` | exit 0，**浅层成功** | +| **#45** | `019ef333` | **28× `rust_*`**（owner 归一化 + 图 rdeps） | **23,270 action**；**8,035** local commands；**0%** cache | **exit 0，真编译通过** | + +#### Build #45 时间线（`019ef333-db97-70e2-8960-40bcc5cc2496`） + +| 阶段 | 时间 (UTC) | 耗时 | +|------|------------|------| +| 收到任务 | 06:38:42 | — | +| CL overlay 就绪 | 06:39:33 | ~51s | +| Discovery 完成（28 targets，子项目 `rk8s/`） | 06:39:56 | ~23s | +| `buck2 build` 开始 | 06:39:56 | — | +| `BUILD SUCCEEDED` | 07:51:54 | **~72 min** | +| 上报 exit 0 | 07:52:00 | 总计 **~73 min** | + +日志要点：从 `mount/.../rk8s` 跑 buck2；discovery 后卸载 `old` mount；末尾为 `root//project/rkl:rkl`、`rkforge`、`slayerfs` 等 **`rustc link`**；**无** `uquery rdeps` / `cxx_no_default_deps` 报错（all-added 走图 rdeps）。 + +--- + +## 1. 问题现象（历史） + +CL（例如 `#UYXIYYNJ`）只新增 `rk8s` 下 Rust 代码与 `third-party/**`，早期构建曾出现： + +``` +Action failed: root//project/buck2_test/toolchains:jdk_system_image (create_jdk_system_image) + FileNotFoundError: ... jlink +``` + +以及 `__android_sdk_tools__` 缺失、`` platform 等。 + +几个关键事实： + +- monorepo 根 `.buckconfig` 下，`rk8s` 与 `project/buck2_test` 同在 **`root` cell**。 +- 仅「按 cell 收窄」不够：扫 `root//...` 仍会带入无关 `project/**` 与 `third-party/**`。 +- `rk8s` 作为**子项目**有自己的 `.buckconfig`（`root = .`），必须从 `mount/.../rk8s` 跑 buck2，不能从 monorepo 根跑。 + +--- + +## 2. 根因分析 + +worker 端流程：`get_build_targets()` → `collect_impacted_targets()` / owner 回退 → `buck2 build`（见 [buck_controller.rs](../src/buck_controller.rs)）。 + +### 2.1 发现阶段曾扫全 cell（已由方案 A 缓解） + +`get_repo_targets()` 默认通过 `get_all_cell_patterns()` 查询 `root//...`、`toolchains//...` 等。方案 A 根据改动路径收窄为例如 `root//rk8s/...`（见 `compute_discovery_scope()`）。 + +### 2.2 默认 platform + toolchain 传播（已由 B + C 缓解） + +1. 默认 platform `prelude//platforms:default` 会解析多语言 toolchain。 +2. 历史上 `recursive_target_changes(..., |_| true)` 不过滤 toolchain/platform。 +3. **B**：传播与 build 选集跳过 toolchain/platform helper。 +4. **C2**：Orion 对每次 `buck2 targets` / `buck2 build` 注入 `default_target_platforms` 与完整 `target_platform_detector_spec`（含 `buckal//...`），避免 FUSE 只读挂载上旧 `.buckconfig` 导致 ``。 + +### 2.3 All-added + 空 base 曾 SelectAll（#39） + +`base` 图为空时，`EmptyBasePolicy::SelectAll` 会把 diff 图里**所有** target 标为 impacted，包括 `third-party/**` 下每个 crate 的 vendor 根，action 爆炸。 + +**任意 all-added CL**（含子项目导入与普通「只新增文件」CL）现统一： + +1. **跳过** 图 `collect_impacted_targets` + `SelectAll`（避免空 base 全选）。 +2. **`owner()` 播种**：改动中的源文件路径（排除 `BUCK`、`Cargo.toml`、`vendor/` 等）。 + +**All-added 子项目**（改动全在带 `.buckconfig` 的子目录，如 `rk8s/**`）在此基础上还有： + +1. **发现范围**：仅 `root//project/...`（不含 `third-party/`）。 +2. **`normalize_owner_targets_to_rust`**：buckal 的 `filegroup :vendor` 拥有包内几乎所有文件，`owner()` 常返回 `:vendor`；映射为同 package 的 `rust_library` / `rust_binary` / `rust_test`。 +3. **图 rdeps**：在 `project/` 内用 `diff::recursive_target_changes` 扩展反向依赖，**不用** `buck2 uquery rdeps`（遍历 `aardvark-dns` 等会碰到缺失的 `toolchains//:cxx_no_default_deps`）。 + +### 2.4 owner 误选 `:vendor`（#42） + +#42 在 ~30s 内 exit 0，但 buck2 只执行了 17 个 `symlinked_dir vendor`，**无 `rustc`**。根因是 build 列表为 `root//project/*:vendor` 而非 `root//project/*:`。`normalize_owner_targets_to_rust` 已修复。 + +### 2.5 0 target 假成功（未修复） + +`finish_without_build_if_no_targets()` 在 0 target 时仍返回 exit 0（#35–#36）。与扫描范围无关，但会掩盖发现失败。 + +--- + +## 3. 当前 discovery 流程（简图） + +```mermaid +flowchart TD + CL["CL 改动列表"] --> Sub{"全在子目录且含\n.buckconfig？"} + Sub -->|是| Strip["strip 前缀\n从子项目根跑 buck2"] + Sub -->|否| Root["monorepo 根"] + Strip --> Scope + Root --> Scope + + Scope["compute_discovery_scope (A)\nORION_DISCOVERY_SCOPE"] + Scope --> AllAdded{"all-added？"} + + AllAdded -->|是| AllAddedKind{"子项目\n.buckconfig？"} + AllAdded -->|否| Patterns["收窄或全 cell 模式"] + + AllAddedKind -->|是| ProjOnly["buck2 targets\nroot//project/..."] + AllAddedKind -->|否| Patterns + + ProjOnly --> Owner["owner() 播种\n+ normalize → rust_*"] + Patterns --> GraphCheck{"all-added？"} + GraphCheck -->|是| Owner + GraphCheck -->|否| Graph["collect_impacted_targets (B)"] + + Owner --> RdepsG["图 rdeps (project/)"] + Graph --> RdepsU{"narrow？"} + RdepsU -->|是| RdepsTry["uquery rdeps\n失败则图 rdeps fallback"] + RdepsU -->|否| Build + + RdepsG --> Build["buck2 build\n+ platform --config (C2)"] + RdepsTry --> Build +``` + +--- + +## 4. 方案说明与状态 + +### 方案 A：按改动路径收窄发现（**已落地**） + +- 落点：`orion/buck/discovery_scope.rs` → `compute_discovery_scope()`。 +- 行为：改动在 `rk8s/**` 时查询 `root//rk8s/...`，而非整个 `root//...`。 +- 环境变量：`ORION_DISCOVERY_SCOPE=0|false|no|off` 关闭。 + +### 方案 B：过滤 toolchain / platform helper（**已落地**） + +见 `is_toolchain_or_platform_*` 与 `collect_impacted_targets()`；单元测试 `test_toolchain_helper_targets_are_excluded`。 + +### 方案 C：收敛默认 platform（**C1 + C2 已落地**） + +| 子项 | 内容 | 状态 | +|------|------|------| +| **C1** | 移除 Orion CLI `--target-platforms` | 已落地 | +| **C2** | `platform.rs` 注入 `default_target_platforms` + `target_platform_detector_spec` | 已落地 | + +仓库根 [.buckconfig](../../.buckconfig) 可与 Orion 注入不一致；**以 Orion `--config` 为准**保证 worker 行为一致。完整裁剪 `prelude//platforms:default`（按语言缩 toolchain）仍未做。 + +### 方案 D：A + rdeps（**部分落地**） + +| 场景 | rdeps 实现 | +|------|------------| +| 普通 narrow scope | `buck2 uquery rdeps(seeds, universe)`；失败时 fallback 图 rdeps | +| all-added 子项目 | 仅用图 rdeps，universe = `root//project/...` | + +### 子项目 buck 根（**已落地**） + +`detect_subproject_buck_root()`：当所有改动路径在同一含 `.buckconfig` 的目录下（如 `rk8s/`），discovery 与 build 的 `current_dir` 设为 `mount/.../rk8s`，路径去掉 `rk8s/` 前缀。 + +### 环境变量 + +| 变量 | 默认 | 作用 | +|------|------|------| +| `ORION_DISCOVERY_SCOPE` | 开启 | `0`/`false`/`no`/`off` 关闭方案 A | +| `ORION_BUCK_REMOTE_CACHE` | 关闭 | `1` 时 buck2 build 允许读 remote cache；否则 `--no-remote-cache` | + +--- + +## 5. 现状小结 + +| 维度 | 当前行为 | 进展 | +|------|----------|------| +| 发现范围 | 方案 A 按改动子树；all-added 子项目限 `project/` | **已收窄** | +| 子项目 buck 根 | `rk8s/.buckconfig` 等 | **已落地** | +| toolchain 传播 / build 选集 | 过滤 helper | **B 已落地** | +| platform | Orion `--config` + `.buckconfig` | **C2 已落地** | +| all-added 子项目 | owner + rust 映射 + 图 rdeps | **已落地** | +| 0 target | 仍 exit 0 | **待修** | +| FUSE + 真 `rustc` | #45 单点通过（0× ENOENT） | 见 [FUSE_MOUNT_ISSUES.md](./FUSE_MOUNT_ISSUES.md) | + +--- + +## 6. 相关代码索引 + +| 文件 | 内容 | +|------|------| +| [orion/src/buck_controller.rs](../src/buck_controller.rs) | `get_build_targets()`、B、all-added 子项目、owner 归一化、图 rdeps、`ORION_BUCK_REMOTE_CACHE` | +| [orion/buck/discovery_scope.rs](../buck/discovery_scope.rs) | 方案 A、子项目检测、`ORION_DISCOVERY_SCOPE` | +| [orion/buck/run.rs](../buck/run.rs) | `uquery_rdeps`、`owners` | +| [orion/buck/platform.rs](../buck/platform.rs) | Scheme C2 `--config` | +| [orion/src/repo/diff.rs](../src/repo/diff.rs) | `EmptyBasePolicy`、`recursive_target_changes` | +| [.buckconfig](../../.buckconfig) | 仓库侧 platform（可被 Orion 覆盖） | + +--- + +## 修订历史 + +| 日期 | 说明 | +|------|------| +| 2026-06-15 | 初稿：根因分析与方案 A–E | +| 2026-06-17 | 补充 B 与 Build #17 验证 | +| 2026-06-18 | 落地 C2 | +| 2026-06-23 | 更新：A/B/C1/D/子项目/all-added 已落地；#39–#42 对照；owner→rust 与图 rdeps | +| 2026-06-23 | **#45** 验证：28 targets、23k action、~73 min、exit 0 真编译 | diff --git a/orion/docs/deployment.md b/orion/docs/deployment.md index a8a7d47ba..f7ec1f4f6 100644 --- a/orion/docs/deployment.md +++ b/orion/docs/deployment.md @@ -165,3 +165,4 @@ sudo umount -lf /workspace/mount - Ensure `user_allow_other` is enabled in `/etc/fuse.conf`. - Ensure service keeps `AmbientCapabilities=CAP_SYS_ADMIN` and `/dev/fuse` is readable/writable. - `preflight.sh` will now block startup early with explicit errors if capability/device checks fail. +4. **Buck2 `linker_wrapper.sh` ENOENT / `event.jsonl` EBADF on Antares mount**: See [FUSE_MOUNT_ISSUES.md](FUSE_MOUNT_ISSUES.md). diff --git a/orion/runner-config/.env.prod b/orion/runner-config/.env.prod index eab2a8c66..ad6409598 100644 --- a/orion/runner-config/.env.prod +++ b/orion/runner-config/.env.prod @@ -15,3 +15,6 @@ INITIAL_POLL_INTERVAL_SECS="2" # 临时 buck-out 目录 TMP_BUCKOUT_DIR="/data/scorpio/tmp_build" + +# Buck2 remote action cache：0 关闭（默认，本地必编译）；1 允许读 remote cache +ORION_BUCK_REMOTE_CACHE="0" diff --git a/orion/runner-config/README.md b/orion/runner-config/README.md index a4ac77c49..581698790 100644 --- a/orion/runner-config/README.md +++ b/orion/runner-config/README.md @@ -2,7 +2,9 @@ 此目录包含生产环境下 Orion Worker 的运行配置。 -**详细部署文档**: [docs/deployment.md](../docs/deployment.md) +**详细部署文档**: [docs/deployment.md](../docs/deployment.md) +**FUSE 问题分析与修复**: [docs/FUSE_MOUNT_ISSUES.md](../docs/FUSE_MOUNT_ISSUES.md) +**Target Discovery 范围问题**: [docs/TARGET_DISCOVERY_SCOPE.md](../docs/TARGET_DISCOVERY_SCOPE.md) ## 文件说明 diff --git a/orion/src/buck_controller.rs b/orion/src/buck_controller.rs index 10fe67b4e..5d310158f 100644 --- a/orion/src/buck_controller.rs +++ b/orion/src/buck_controller.rs @@ -17,10 +17,15 @@ use once_cell::sync::Lazy; use td_util::{command::spawn, file_io::file_writer, file_tail::tail_compressed_buck2_events}; use td_util_buck::{ cells::CellInfo, + discovery_scope::{ + compute_discovery_scope, detect_subproject_buck_root, strip_subproject_changes, + }, + owners::Owners, + platform::{append_platform_config, platform_config_flags}, run::{Buck2, targets_arguments}, target_status::{BuildState, EVENT_LOG_FILE, Event, LogicalActionId, TargetBuildStatusUpdate}, - targets::Targets, - types::TargetLabel, + targets::{BuckTarget, Targets}, + types::{RuleType, TargetLabel}, }; use tokio::{ io::AsyncBufReadExt, @@ -51,6 +56,21 @@ struct AntaresMountPair { old_mount_point: String, new_mount_id: String, new_mount_point: String, + old_unmounted: bool, +} + +/// Enable buck2 remote cache when set to `1`, `true`, `yes`, or `on`. +/// +/// Default: disabled (`--no-remote-cache`) so incremental builds always compile +/// locally and catch syntax errors immediately. +fn buck_remote_cache_enabled() -> bool { + match std::env::var("ORION_BUCK_REMOTE_CACHE") { + Ok(value) => { + let normalized = value.trim().to_ascii_lowercase(); + matches!(normalized.as_str(), "1" | "true" | "yes" | "on") + } + Err(_) => false, + } } fn retain_antares_mounts() -> bool { @@ -131,20 +151,44 @@ async fn cleanup_antares_mount( } async fn cleanup_antares_mount_pair(task_id: &str, mounts: &AntaresMountPair, reason: &str) { + if !mounts.old_unmounted { + cleanup_antares_mount( + task_id, + &mounts.old_mount_id, + Some(&mounts.old_mount_point), + reason, + ) + .await; + } cleanup_antares_mount( task_id, - &mounts.old_mount_id, - Some(&mounts.old_mount_point), + &mounts.new_mount_id, + Some(&mounts.new_mount_point), reason, ) .await; +} + +/// The old-repo mount is only needed for target discovery; release it before buck2 build. +async fn unmount_discovery_old_mount(task_id: &str, mounts: &mut AntaresMountPair) { + if mounts.old_unmounted { + return; + } + + tracing::info!( + "[Task {}] Unmounting old-repo Antares view after target discovery (mount_id={}, mountpoint={})", + task_id, + mounts.old_mount_id, + mounts.old_mount_point, + ); cleanup_antares_mount( task_id, - &mounts.new_mount_id, - Some(&mounts.new_mount_point), - reason, + &mounts.old_mount_id, + Some(&mounts.old_mount_point), + "old-repo mount no longer needed after target discovery", ) .await; + mounts.old_unmounted = true; } /// Mount an Antares overlay filesystem for a build job. @@ -350,10 +394,83 @@ fn resolve_config_path() -> Option { /// populates the Dicfuse backing store which makes subsequent FUSE reads /// fast, and `preheat_shallow()` in `get_build_targets()` handles the /// per-mount VFS cache warming. +fn all_changes_are_added(changes: &[Status]) -> bool { + !changes.is_empty() + && changes + .iter() + .all(|change| matches!(change, Status::Added(_))) +} + +/// Subproject import (e.g. all-added `rk8s/`): only consider paths under `project/`. +fn filter_changes_under_prefix( + changes: &[Status], + prefix: &str, +) -> Vec> { + let prefix_slash = format!("{prefix}/"); + changes + .iter() + .filter(|change| { + let path = change.get().as_str(); + path == prefix || path.starts_with(&prefix_slash) + }) + .cloned() + .collect() +} + +/// Paths suitable for `buck2 uquery owner()` when seeding all-added subproject builds. +/// Skips package manifests and vendored trees so we do not select every crate in `project/`. +fn is_owner_seed_path(path: &str) -> bool { + if path.is_empty() { + return false; + } + let file_name = path.rsplit('/').next().unwrap_or(path); + if matches!( + file_name, + "BUCK" | "TARGETS" | "BUCK.v2" | "Cargo.toml" | "Cargo.lock" | ".buckconfig" + ) { + return false; + } + !(path.contains("/vendor/") || path.ends_with("/vendor")) +} + +fn filter_owner_seed_changes( + changes: &[Status], +) -> Vec> { + changes + .iter() + .filter(|change| is_owner_seed_path(change.get().as_str())) + .cloned() + .collect() +} + +/// Paths passed to `owner()` when the CL is all-added. +/// +/// All-added sets use an empty base graph (old-repo `buck2 targets` is skipped). +/// Running graph diff with `EmptyBasePolicy::SelectAll` would mark every target in +/// the diff graph as impacted; seed from changed source paths instead. +fn owner_seed_changes_for_discovery( + all_added_subproject: bool, + all_added: bool, + changes: &[Status], +) -> Vec> { + match (all_added_subproject, all_added) { + (true, _) => { + let under_project = + filter_changes_under_prefix(changes, ALL_ADDED_SUBPROJECT_BUILD_PREFIX); + filter_owner_seed_changes(&under_project) + } + (false, true) => filter_owner_seed_changes(changes), + (false, false) => changes.to_vec(), + } +} + +const ALL_ADDED_SUBPROJECT_BUILD_PREFIX: &str = "project"; + fn get_repo_targets( file_name: &str, repo_path: &Path, cells: Option<&CellInfo>, + query_patterns: Option<&[String]>, ) -> anyhow::Result { const MAX_ATTEMPTS: usize = 2; let jsonl_path = PathBuf::from(repo_path).join(file_name); @@ -375,10 +492,14 @@ fn get_repo_targets( // Add base targets arguments command.args(targets_arguments()); + append_platform_config(&mut command); // If cells info is provided, query all cells; otherwise just query root cell if let Some(cells_info) = cells { - let cell_patterns = cells_info.get_all_cell_patterns(repo_path); + let cell_patterns = match query_patterns { + Some(patterns) if !patterns.is_empty() => patterns.to_vec(), + _ => cells_info.get_all_cell_patterns(repo_path), + }; tracing::debug!("Querying targets for cells: {:?}", cell_patterns); command.args(&cell_patterns); } else { @@ -495,23 +616,279 @@ fn fallback_targets_for_errored_packages( fallback } -fn collect_impacted_targets(base: &Targets, diff: &Targets, changes: &Changes) -> Vec { - let immediate = diff::immediate_target_changes(base, diff, changes, false); - let recursive = diff::recursive_target_changes(diff, changes, &immediate, None, |_| true); +const OWNER_QUERY_BATCH_SIZE: usize = 500; + +/// When the `buck2 targets` graph has no impacted nodes (e.g. all-added CL with only +/// package errors + imports in jsonl), resolve owners directly via `buck2 uquery owner()`. +fn fallback_targets_from_owners( + buck2: &mut Buck2, + changes: &[Status], +) -> anyhow::Result> { + let paths: Vec = changes + .iter() + .map(|change| change.get().clone()) + .filter(|path| !path.as_str().is_empty()) + .collect(); + if paths.is_empty() { + return Ok(Vec::new()); + } + + let mut seen = HashSet::new(); + let mut targets = Vec::new(); + let mut batches = 0usize; + + for chunk in paths.chunks(OWNER_QUERY_BATCH_SIZE) { + batches += 1; + let json = match buck2.owners(&[], chunk) { + Ok(json) => json, + Err(err) => { + tracing::warn!( + error = %err, + owner_batch = batches, + "buck2 uquery owner() batch failed; continuing with remaining paths." + ); + continue; + } + }; + let owners = Owners::from_json(&json)?; + for label in owners.all_targets() { + if label_is_toolchain_or_platform(label) { + continue; + } + if seen.insert(label.clone()) { + targets.push(label.clone()); + } + } + } + + if targets.is_empty() { + tracing::warn!( + change_paths = paths.len(), + owner_batches = batches, + "buck2 uquery owner() returned no build targets for the changed paths." + ); + } else { + tracing::info!( + change_paths = paths.len(), + owner_batches = batches, + recovered_targets = targets.len(), + "Recovered impacted Buck targets via buck2 uquery owner()." + ); + } + + Ok(targets) +} + +/// Rule types that define toolchains, platforms, or configuration nodes. +/// +/// These are build-system plumbing, not business targets. We never want to +/// propagate impact *through* them (a pure-Rust change must not fan out into +/// JVM/Android/CXX toolchain helpers), nor select them as explicit `buck2 +/// build` targets (e.g. `jdk_system_image`, `__android_sdk_tools__`). +fn is_toolchain_or_platform_rule(rule_type: &RuleType) -> bool { + let short = rule_type.short(); + + const EXACT: &[&str] = &[ + "platform", + "execution_platform", + "execution_platforms", + "constraint_setting", + "constraint_value", + "config_setting", + "configuration", + "configured_alias", + "toolchain_alias", + ]; + + EXACT.contains(&short) + || short == "toolchain" + || short.ends_with("_toolchain") + || short.starts_with("toolchain_") +} + +/// Whether a package (cell-qualified, e.g. `root//project/buck2_test/toolchains`) +/// is a toolchain or platform definition package. +fn package_is_toolchain_or_platform(package: &str) -> bool { + let (cell, rel) = package.split_once("//").unwrap_or(("", package)); + + if cell == "toolchains" { + return true; + } + + for plumbing in ["toolchains", "platforms"] { + if rel == plumbing + || rel.starts_with(&format!("{plumbing}/")) + || rel.ends_with(&format!("/{plumbing}")) + || rel.contains(&format!("/{plumbing}/")) + { + return true; + } + } + + false +} + +/// Whether a concrete target is a toolchain/platform helper that should be +/// excluded from the explicit build set. +fn is_toolchain_or_platform_target(target: &BuckTarget) -> bool { + is_toolchain_or_platform_rule(&target.rule_type) + || package_is_toolchain_or_platform(target.package.as_str()) + || matches!( + target.name.as_str(), + "jdk_system_image" | "__android_sdk_tools__" + ) +} + +/// Heuristic for fallback labels (we only have the label string, not the +/// `BuckTarget`), e.g. `root//project/buck2_test/toolchains:jdk_system_image`. +fn label_is_toolchain_or_platform(label: &TargetLabel) -> bool { + let label = label.as_str(); + let package = label.rsplit_once(':').map(|(pkg, _)| pkg).unwrap_or(label); + let name = label.rsplit_once(':').map(|(_, name)| name).unwrap_or(""); + + package_is_toolchain_or_platform(package) + || matches!(name, "jdk_system_image" | "__android_sdk_tools__") +} + +fn is_rust_build_rule(rule_type: &RuleType) -> bool { + matches!( + rule_type.short(), + "rust_library" | "rust_binary" | "rust_test" + ) +} + +/// Buckal-generated helper targets that must not be passed to `buck2 build`. +fn label_is_buckal_plumbing_name(name: &str) -> bool { + matches!(name, "vendor" | "manifest") || name.starts_with("build-script") +} + +fn is_buckal_plumbing_target(target: &BuckTarget) -> bool { + label_is_buckal_plumbing_name(target.name.as_str()) + || matches!( + target.rule_type.short(), + "cargo_manifest" | "filegroup" | "buildscript_run" + ) +} + +fn target_package_under_prefix(package: &str, package_prefix: &str) -> bool { + let qualified = format!("root//{package_prefix}"); + let qualified_slash = format!("{qualified}/"); + package == qualified || package.starts_with(&qualified_slash) +} + +fn rust_build_targets_in_package<'a>(diff: &'a Targets, package: &str) -> Vec<&'a BuckTarget> { + diff.targets() + .filter(|t| t.package.as_str() == package && is_rust_build_rule(&t.rule_type)) + .collect() +} + +/// `buck2 uquery owner()` on buckal trees often returns `:vendor` filegroups. +/// Map those to the real `rust_library` / `rust_binary` targets in the same package. +fn normalize_owner_targets_to_rust(diff: &Targets, seeds: Vec) -> Vec { + let mut seen = HashSet::new(); + let mut out = Vec::new(); + + for label in seeds { + if label_is_toolchain_or_platform(&label) { + continue; + } + let label_str = label.as_str(); + let package = label_str + .rsplit_once(':') + .map(|(pkg, _)| pkg) + .unwrap_or(label_str); + let name = label_str.rsplit_once(':').map(|(_, n)| n).unwrap_or(""); + + let push_rust_in_package = |out: &mut Vec<_>, seen: &mut HashSet<_>| { + for target in rust_build_targets_in_package(diff, package) { + let rust_label = target.label(); + if seen.insert(rust_label.clone()) { + out.push(rust_label); + } + } + }; + + if label_is_buckal_plumbing_name(name) { + push_rust_in_package(&mut out, &mut seen); + continue; + } + + if let Some(target) = diff.targets().find(|t| t.label() == label) + && (is_buckal_plumbing_target(target) || !is_rust_build_rule(&target.rule_type)) + { + push_rust_in_package(&mut out, &mut seen); + continue; + } + + if seen.insert(label.clone()) { + out.push(label); + } + } + out +} + +fn package_prefix_from_universe_patterns(patterns: &[String]) -> Option { + for pattern in patterns { + let stripped = pattern.strip_prefix("root//")?; + let prefix = stripped.strip_suffix("/...")?; + if prefix.is_empty() || prefix.contains("...") { + continue; + } + return Some(prefix.to_owned()); + } + None +} + +fn collect_impacted_targets( + base: &Targets, + diff: &Targets, + changes: &Changes, + empty_base_policy: diff::EmptyBasePolicy, +) -> Vec { + let immediate = + diff::immediate_target_changes_with_policy(base, diff, changes, false, empty_base_policy); + // Do not propagate impact *through* toolchain/platform/config nodes: a + // change to (or near) a toolchain definition must not drag in every target + // that resolves that toolchain. + let recursive = diff::recursive_target_changes(diff, changes, &immediate, None, |rule_type| { + !is_toolchain_or_platform_rule(rule_type) + }); + + let mut excluded_helpers = 0usize; let mut targets: Vec<_> = recursive .into_iter() .flatten() + .filter(|(target, _)| { + // Never build toolchain/platform helpers as explicit targets; if a + // real target needs them, buck2 still builds them transitively. + let keep = !is_toolchain_or_platform_target(target); + if !keep { + excluded_helpers += 1; + } + keep + }) .map(|(target, _)| target.label()) .collect(); let mut seen: HashSet<_> = targets.iter().cloned().collect(); for label in fallback_targets_for_errored_packages(base, diff, changes) { + if label_is_toolchain_or_platform(&label) { + excluded_helpers += 1; + continue; + } if seen.insert(label.clone()) { targets.push(label); } } + if excluded_helpers > 0 { + tracing::info!( + excluded_helpers, + "Excluded toolchain/platform helper targets from the build set." + ); + } + if targets.is_empty() { tracing::info!( changes_count = changes.cell_paths().count(), @@ -531,6 +908,162 @@ fn collect_impacted_targets(base: &Targets, diff: &Targets, changes: &Changes) - targets } +/// Expand impacted targets with reverse-deps from the full cell universe (scheme D). +fn expand_impacted_with_rdeps( + buck2: &mut Buck2, + seeds: &[TargetLabel], + universe_patterns: &[String], +) -> anyhow::Result> { + if seeds.is_empty() { + return Ok(Vec::new()); + } + + let rdeps = buck2.uquery_rdeps(seeds, universe_patterns)?; + let mut seen: HashSet = seeds.iter().cloned().collect(); + let mut expanded = seeds.to_vec(); + + for label in rdeps { + if label_is_toolchain_or_platform(&label) { + continue; + } + if seen.insert(label.clone()) { + expanded.push(label); + } + } + + Ok(expanded) +} + +/// Reverse-deps expansion using the in-memory `buck2 targets` graph. +/// +/// Avoids `buck2 uquery rdeps`, which can fail when a dependency references a +/// missing toolchain cell (e.g. `toolchains//:cxx_no_default_deps`). +fn expand_impacted_with_graph_rdeps( + diff: &Targets, + changes: &Changes, + seeds: &[TargetLabel], + package_prefix: &str, +) -> Vec { + if seeds.is_empty() { + return Vec::new(); + } + + let target_by_label: HashMap = + diff.targets().map(|t| (t.label(), t)).collect(); + + let mut recursive_seeds = Vec::new(); + for label in seeds { + let Some(target) = target_by_label.get(label) else { + continue; + }; + if !target_package_under_prefix(target.package.as_str(), package_prefix) { + continue; + } + recursive_seeds.push(( + *target, + diff::ImpactTraceData::new(target, diff::RootImpactKind::Inputs), + )); + } + + if recursive_seeds.is_empty() { + return seeds.to_vec(); + } + + let impact = diff::GraphImpact::from_recursive(recursive_seeds); + let layers = diff::recursive_target_changes(diff, changes, &impact, None, |rule_type| { + !is_toolchain_or_platform_rule(rule_type) + }); + + let mut seen: HashSet = seeds.iter().cloned().collect(); + let mut expanded = seeds.to_vec(); + + for layer in layers { + for (target, _) in layer { + if !target_package_under_prefix(target.package.as_str(), package_prefix) { + continue; + } + if is_toolchain_or_platform_target(target) || is_buckal_plumbing_target(target) { + continue; + } + if !is_rust_build_rule(&target.rule_type) { + continue; + } + let label = target.label(); + if seen.insert(label.clone()) { + expanded.push(label); + } + } + } + + expanded +} + +fn maybe_expand_narrow_targets( + buck2: &mut Buck2, + diff: &Targets, + changes: &Changes, + targets: Vec, + narrow: bool, + universe_patterns: &[String], + graph_rdeps_prefix: Option<&str>, +) -> Vec { + if !narrow || targets.is_empty() { + return targets; + } + + if let Some(prefix) = graph_rdeps_prefix { + let expanded = expand_impacted_with_graph_rdeps(diff, changes, &targets, prefix); + if expanded.len() > targets.len() { + tracing::info!( + seeds = targets.len(), + after_rdeps = expanded.len(), + package_prefix = prefix, + "Expanded narrowed discovery seeds with in-graph rdeps." + ); + } + return expanded; + } + + match expand_impacted_with_rdeps(buck2, &targets, universe_patterns) { + Ok(expanded) => { + if expanded.len() > targets.len() { + tracing::info!( + seeds = targets.len(), + after_rdeps = expanded.len(), + "Expanded narrowed discovery seeds with buck2 uquery rdeps." + ); + } + expanded + } + Err(err) => { + if let Some(prefix) = package_prefix_from_universe_patterns(universe_patterns) { + tracing::warn!( + error = %err, + seed_count = targets.len(), + package_prefix = %prefix, + "buck2 uquery rdeps failed; falling back to in-graph rdeps." + ); + let expanded = expand_impacted_with_graph_rdeps(diff, changes, &targets, &prefix); + if expanded.len() > targets.len() { + tracing::info!( + seeds = targets.len(), + after_rdeps = expanded.len(), + "Expanded narrowed discovery seeds with in-graph rdeps fallback." + ); + } + expanded + } else { + tracing::warn!( + error = %err, + seed_count = targets.len(), + "buck2 uquery rdeps failed; keeping narrowed impacted targets only." + ); + targets + } + } + } +} + fn has_path_component_suffix(candidate: &str, suffix: &str) -> bool { candidate == suffix || candidate @@ -621,22 +1154,44 @@ async fn get_build_targets( tracing::debug!("Analyzing changes {mega_changes:?}"); - preheat_shallow(&mount_path, preheat_shallow_depth())?; + let subproject = detect_subproject_buck_root(&mount_path, &mega_changes); + let buck2_root = subproject + .as_ref() + .map(|sp| sp.buck_root.clone()) + .unwrap_or_else(|| mount_path.clone()); + let old_buck_root = subproject + .as_ref() + .map(|sp| old_repo.join(&sp.strip_prefix)) + .unwrap_or_else(|| old_repo.clone()); + let discovery_changes = subproject + .as_ref() + .map(|sp| strip_subproject_changes(&mega_changes, &sp.strip_prefix)) + .unwrap_or_else(|| mega_changes.clone()); + + if let Some(sp) = &subproject { + tracing::info!( + buck_root = %sp.buck_root.display(), + strip_prefix = %sp.strip_prefix, + "Using sub-project .buckconfig for target discovery." + ); + } + + preheat_shallow(&buck2_root, preheat_shallow_depth())?; // DEBUG: Log Buck2 initialization - tracing::debug!(buck2_root = %mount_path.display(), "DEBUG: Initializing Buck2 with root"); - let mut buck2 = Buck2::with_root("buck2".to_string(), mount_path.clone()); + tracing::debug!(buck2_root = %buck2_root.display(), "DEBUG: Initializing Buck2 with root"); + let mut buck2 = Buck2::with_root("buck2".to_string(), buck2_root.clone()); // DEBUG: Log before buck2 cells() call - tracing::debug!(buck2_root = %mount_path.display(), "DEBUG: About to call buck2 cells()"); + tracing::debug!(buck2_root = %buck2_root.display(), "DEBUG: About to call buck2 cells()"); let cells_result = buck2.cells(); match &cells_result { Ok(_cells_info) => { - tracing::debug!(buck2_root = %mount_path.display(), "DEBUG: buck2 cells() succeeded"); + tracing::debug!(buck2_root = %buck2_root.display(), "DEBUG: buck2 cells() succeeded"); } Err(e) => { - tracing::warn!(buck2_root = %mount_path.display(), error = %e, "DEBUG: buck2 cells() failed"); + tracing::warn!(buck2_root = %buck2_root.display(), error = %e, "DEBUG: buck2 cells() failed"); } } @@ -650,36 +1205,163 @@ async fn get_build_targets( .map_err(|err| anyhow!("Fail to get config: {}", err))?, )?; - let base = get_repo_targets("base.jsonl", &old_repo, Some(&cells))?; - let diff = get_repo_targets("diff.jsonl", &mount_path, Some(&cells))?; - let changes = Changes::new(&cells, mega_changes.clone())?; + let full_patterns = cells.get_all_cell_patterns(&buck2_root); + let all_added = all_changes_are_added(&mega_changes); + let all_added_subproject = all_added && subproject.is_some(); + + let scope = compute_discovery_scope(&cells, &buck2_root, &discovery_changes); + let project_only_patterns = vec![format!("root//{ALL_ADDED_SUBPROJECT_BUILD_PREFIX}/...")]; + let (discovery_patterns, rdeps_universe) = if all_added_subproject { + tracing::info!( + patterns = ?project_only_patterns, + "All-added subproject import; limiting discovery and build to project/ only." + ); + (project_only_patterns.clone(), project_only_patterns) + } else if scope.narrow { + tracing::info!( + patterns = ?scope.query_patterns, + "Using narrowed target discovery scope." + ); + (scope.query_patterns.clone(), full_patterns.clone()) + } else { + (full_patterns.clone(), full_patterns.clone()) + }; + let query_patterns = &discovery_patterns; + let use_rdeps_expansion = all_added_subproject || scope.narrow; + let graph_rdeps_prefix = if all_added_subproject { + Some(ALL_ADDED_SUBPROJECT_BUILD_PREFIX) + } else { + None + }; + + let base = if all_added { + tracing::info!( + change_count = mega_changes.len(), + "All-added change set; skipping old-repo buck2 targets query." + ); + Targets::new(Vec::new()) + } else { + match get_repo_targets( + "base.jsonl", + &old_buck_root, + Some(&cells), + Some(query_patterns), + ) { + Ok(base) => base, + Err(err) => return Err(err), + } + }; + let diff = get_repo_targets( + "diff.jsonl", + &buck2_root, + Some(&cells), + Some(query_patterns), + )?; + let changes = Changes::new(&cells, discovery_changes.clone())?; tracing::debug!("Changes {changes:?}"); tracing::debug!("Base targets number: {}", base.len_targets_upperbound()); tracing::debug!("Diff targets number: {}", diff.len_targets_upperbound()); - let targets = collect_impacted_targets(&base, &diff, &changes); + let owner_seed_changes = + owner_seed_changes_for_discovery(all_added_subproject, all_added, &discovery_changes); + + if all_added_subproject { + tracing::info!( + owner_seed_paths = owner_seed_changes.len(), + "All-added subproject import; seeding targets via owner() on project source paths, then mapping to rust_library/rust_binary." + ); + } else if all_added { + tracing::info!( + owner_seed_paths = owner_seed_changes.len(), + "All-added change set; skipping graph SelectAll on empty base, seeding targets via owner()." + ); + } + + let graph_targets = if all_added { + Vec::new() + } else { + collect_impacted_targets(&base, &diff, &changes, diff::EmptyBasePolicy::SelectAll) + }; + + let targets = maybe_expand_narrow_targets( + &mut buck2, + &diff, + &changes, + graph_targets, + use_rdeps_expansion, + &rdeps_universe, + graph_rdeps_prefix, + ); if !targets.is_empty() { return Ok(targets); } - let (remapped_changes, remapped_count) = - remap_repo_local_change_paths(&mount_path, &diff, &mega_changes); - if remapped_count == 0 { - return Ok(targets); + let owner_targets = { + let seeds = fallback_targets_from_owners(&mut buck2, &owner_seed_changes)?; + let seeds = normalize_owner_targets_to_rust(&diff, seeds); + maybe_expand_narrow_targets( + &mut buck2, + &diff, + &changes, + seeds, + use_rdeps_expansion, + &rdeps_universe, + graph_rdeps_prefix, + ) + }; + if !owner_targets.is_empty() { + return Ok(owner_targets); } - let remapped = Changes::new(&cells, remapped_changes)?; - let remapped_targets = collect_impacted_targets(&base, &diff, &remapped); - if !remapped_targets.is_empty() { - tracing::info!( - remapped_count, - recovered_target_count = remapped_targets.len(), - "Recovered impacted Buck targets after remapping repo-local change paths." + let (remapped_changes, remapped_count) = + remap_repo_local_change_paths(&buck2_root, &diff, &owner_seed_changes); + if remapped_count > 0 { + let remapped = Changes::new(&cells, remapped_changes.clone())?; + let remapped_graph_targets = if all_added { + Vec::new() + } else { + collect_impacted_targets(&base, &diff, &remapped, diff::EmptyBasePolicy::SelectAll) + }; + let remapped_targets = maybe_expand_narrow_targets( + &mut buck2, + &diff, + &remapped, + remapped_graph_targets, + use_rdeps_expansion, + &rdeps_universe, + graph_rdeps_prefix, ); + if !remapped_targets.is_empty() { + tracing::info!( + remapped_count, + recovered_target_count = remapped_targets.len(), + "Recovered impacted Buck targets after remapping repo-local change paths." + ); + return Ok(remapped_targets); + } + + let remapped_owner_seeds = + owner_seed_changes_for_discovery(all_added_subproject, all_added, &remapped_changes); + let owner_remapped = { + let seeds = fallback_targets_from_owners(&mut buck2, &remapped_owner_seeds)?; + let seeds = normalize_owner_targets_to_rust(&diff, seeds); + maybe_expand_narrow_targets( + &mut buck2, + &diff, + &remapped, + seeds, + use_rdeps_expansion, + &rdeps_universe, + graph_rdeps_prefix, + ) + }; + if !owner_remapped.is_empty() { + return Ok(owner_remapped); + } } - Ok(remapped_targets) + Ok(targets) } fn validate_project_roots( @@ -1049,6 +1731,7 @@ pub async fn build( old_mount_point: old_repo_mount_point.clone(), new_mount_id: id_for_repo, new_mount_point: repo_mount_point.clone(), + old_unmounted: false, }; tracing::info!( @@ -1142,7 +1825,7 @@ pub async fn build( } } - let mounts = match selected_mounts { + let mut mounts = match selected_mounts { Some(value) => value, None => { let err = last_targets_error @@ -1166,6 +1849,10 @@ pub async fn build( }; let mount_point = mounts.new_mount_point.clone(); + if !retain_mounts { + unmount_discovery_old_mount(&id, &mut mounts).await; + } + if finish_without_build_if_no_targets(&id, &targets, &sender)? { tracing::info!( "[Task {}] No impacted Buck targets detected; skipping buck2 build.", @@ -1185,9 +1872,16 @@ pub async fn build( } let build_result = async { - // Run buck2 build from the sub-project directory, not the monorepo root. - // This ensures buck2 uses the sub-project's .buckconfig and PACKAGE files. - let project_root = PathBuf::from(&mount_point).join(repo_prefix); + // Run buck2 build from the sub-project directory when it has its own `.buckconfig`. + let mut project_root = PathBuf::from(&mount_point).join(repo_prefix); + if let Some(sp) = detect_subproject_buck_root(&project_root, &changes) { + tracing::info!( + buck_root = %sp.buck_root.display(), + strip_prefix = %sp.strip_prefix, + "Using sub-project .buckconfig for buck2 build." + ); + project_root = sp.buck_root; + } tracing::info!( "[Task {}] Starting buck2 build. project_root={}, targets={}", id, @@ -1195,12 +1889,11 @@ pub async fn build( targets.len() ); - // Disable both remote and local cache to ensure syntax errors are detected: - // 1. Kill daemon to clear local action cache - // 2. Use unique isolation-dir to prevent cache sharing between builds - // 3. Use --no-remote-cache to prevent remote cache usage + // Kill daemon + unique isolation-dir limit cross-build local cache sharing. + // Remote cache is off by default; set ORION_BUCK_REMOTE_CACHE=1 to enable it. // Note: Buck2 requires isolation-dir to be a simple directory name without path separators let isolation_dir = format!("buck-isolation-{}", id); + let remote_cache = buck_remote_cache_enabled(); let mut kill_cmd = Command::new("buck2"); kill_cmd .arg("kill") @@ -1229,21 +1922,26 @@ pub async fn build( // FUSE-backed repos may trigger lazy loading during daemon init, which // can be slow on cold caches — allow up to 1200s for the daemon to start. cmd.env("BUCKD_STARTUP_TIMEOUT", "30") - .env("BUCKD_STARTUP_INIT_TIMEOUT", "1200"); - let cmd = cmd - .arg("build") - .args(["--event-log", EVENT_LOG_FILE]) + .env("BUCKD_STARTUP_INIT_TIMEOUT", "1200") + .arg("build"); + for flag in platform_config_flags() { + cmd.arg(flag); + } + cmd.args(["--event-log", EVENT_LOG_FILE]) .args(&targets) - .arg("--target-platforms") - .arg("prelude//platforms:default") // Avoid failing the whole build when a target is explicitly incompatible // with the selected platform (e.g., macOS-only crates on Linux builders). .arg("--skip-incompatible-targets") - .arg("--verbose=2") - // Disable remote cache to ensure we always build with the latest code changes - // and detect syntax errors immediately in incremental builds - .arg("--no-remote-cache") - .arg("--isolation-dir") + .arg("--verbose=2"); + if remote_cache { + tracing::info!( + "[Task {}] ORION_BUCK_REMOTE_CACHE enabled; buck2 may read remote action cache.", + id + ); + } else { + cmd.arg("--no-remote-cache"); + } + cmd.arg("--isolation-dir") .arg(&isolation_dir) .current_dir(&project_root) .stdout(Stdio::piped()) @@ -1287,8 +1985,8 @@ pub async fn build( result = stdout_reader.next_line() => { match result { Ok(Some(line)) => { - // Log buck2 stdout for debugging - tracing::debug!("[Task {}] buck2 stdout: {}", id, line); + // Log at info so run.sh/orion.log and scheduler SSE can tail build output. + tracing::info!("[Task {}] buck2: {}", id, line); if sender.send(WSMessage::TaskBuildOutput { build_id: id.clone(), output: line }).is_err() { child.kill().await?; return Err("WebSocket connection lost during build.".into()); @@ -1405,13 +2103,19 @@ mod tests { use api_model::buck2::{status::Status, types::ProjectRelativePath}; use serial_test::serial; - use td_util_buck::types::TargetLabel; + use td_util_buck::{ + targets::{BuckTarget, Targets, TargetsEntry}, + types::{RuleType, TargetLabel}, + }; use tempfile::TempDir; use tokio::sync::mpsc; use super::{ - antares_unmount_grace_duration, finish_without_build_if_no_targets, get_build_targets, - get_repo_targets, remap_repo_local_change_paths, retain_antares_mounts, + all_changes_are_added, antares_unmount_grace_duration, buck_remote_cache_enabled, + filter_owner_seed_changes, finish_without_build_if_no_targets, get_build_targets, + get_repo_targets, is_toolchain_or_platform_rule, is_toolchain_or_platform_target, + label_is_toolchain_or_platform, normalize_owner_targets_to_rust, + owner_seed_changes_for_discovery, remap_repo_local_change_paths, retain_antares_mounts, validate_project_root_exists, }; @@ -1502,6 +2206,80 @@ mod tests { } } + #[test] + fn test_toolchain_and_platform_rules_are_detected() { + for rule in [ + "prelude//toolchains/jdk.bzl:system_jdk_toolchain", + "prelude//rules.bzl:toolchain", + "prelude//platforms.bzl:platform", + "prelude//config.bzl:config_setting", + "prelude//config.bzl:constraint_value", + ] { + assert!( + is_toolchain_or_platform_rule(&RuleType::new(rule)), + "expected {rule} to be treated as toolchain/platform plumbing" + ); + } + + for rule in [ + "prelude//rules.bzl:rust_library", + "prelude//rules.bzl:rust_binary", + "prelude//rules.bzl:genrule", + ] { + assert!( + !is_toolchain_or_platform_rule(&RuleType::new(rule)), + "expected {rule} to be treated as a business rule" + ); + } + } + + #[test] + fn test_toolchain_helper_targets_are_excluded() { + let jdk = BuckTarget::testing( + "jdk_system_image", + "root//project/buck2_test/toolchains", + "prelude//toolchains/jdk.bzl:create_jdk_system_image", + ); + assert!(is_toolchain_or_platform_target(&jdk)); + + let android = BuckTarget::testing( + "__android_sdk_tools__", + "root//rk8s/foo", + "prelude//rules.bzl:genrule", + ); + assert!(is_toolchain_or_platform_target(&android)); + + let rk8s_toolchain = BuckTarget::testing( + "rust-platform", + "root//rk8s/toolchains", + "prelude//rules.bzl:platform", + ); + assert!(is_toolchain_or_platform_target(&rk8s_toolchain)); + + let rust_lib = BuckTarget::testing( + "jni", + "root//rk8s/third-party/rust/crates/jni/0.21.1", + "prelude//rules.bzl:rust_library", + ); + assert!( + !is_toolchain_or_platform_target(&rust_lib), + "a real rust crate must not be filtered even if it binds to the JVM" + ); + } + + #[test] + fn test_label_toolchain_detection_for_fallback() { + assert!(label_is_toolchain_or_platform(&TargetLabel::new( + "root//project/buck2_test/toolchains:jdk_system_image" + ))); + assert!(label_is_toolchain_or_platform(&TargetLabel::new( + "toolchains//:cxx" + ))); + assert!(!label_is_toolchain_or_platform(&TargetLabel::new( + "root//rk8s/src:lib" + ))); + } + #[test] #[serial] fn test_retain_antares_mounts_defaults_to_false() { @@ -1665,6 +2443,19 @@ mod tests { ); } + #[test] + fn test_all_changes_are_added_requires_non_empty_all_added_set() { + assert!(all_changes_are_added(&[ + Status::Added(ProjectRelativePath::new(".buckconfig")), + Status::Added(ProjectRelativePath::new("src/main.rs")), + ])); + assert!(!all_changes_are_added(&[])); + assert!(!all_changes_are_added(&[ + Status::Added(ProjectRelativePath::new("src/main.rs")), + Status::Modified(ProjectRelativePath::new("src/lib.rs")), + ])); + } + #[tokio::test] #[serial] async fn test_get_build_targets_detects_modified_tracked_file_in_standalone_repo() { @@ -1909,7 +2700,8 @@ edition = "2024" let (_tempdir, _old_root, new_root) = isolated_buck_scope_fixture(); let jsonl_cleanup = JsonlCleanupGuard::new([new_root.join("diff.jsonl"), new_root.join("base.jsonl")]); - let diff = get_repo_targets("diff.jsonl", &new_root, None).expect("load diff targets"); + let diff = + get_repo_targets("diff.jsonl", &new_root, None, None).expect("load diff targets"); let (remapped, remapped_count) = remap_repo_local_change_paths( &new_root, @@ -1993,25 +2785,107 @@ edition = "2024" ); } + fn set_buck_remote_cache_env(value: Option<&str>) { + // SAFETY: these tests are marked serial and only mutate process env inside the test. + unsafe { + match value { + Some(value) => std::env::set_var("ORION_BUCK_REMOTE_CACHE", value), + None => std::env::remove_var("ORION_BUCK_REMOTE_CACHE"), + } + } + } + #[test] - fn test_build_command_includes_no_remote_cache_flag() { - // Read the source code file to verify the flag exists - let source = include_str!("buck_controller.rs"); + #[serial] + fn test_buck_remote_cache_disabled_by_default() { + set_buck_remote_cache_env(None); + assert!(!buck_remote_cache_enabled()); + } - // Verify build function includes --no-remote-cache - assert!( - source.contains(r#".arg("--no-remote-cache")"#), - "buck2 build command must include --no-remote-cache flag. \ - This flag ensures incremental builds always use the latest code changes \ - and detect syntax errors immediately." - ); + #[test] + #[serial] + fn test_buck_remote_cache_enabled_with_one() { + set_buck_remote_cache_env(Some("1")); + assert!(buck_remote_cache_enabled()); + set_buck_remote_cache_env(None); + } - // Verify the comment exists to ensure future maintainers understand why - assert!( - source.contains( - "Disable remote cache to ensure we always build with the latest code changes" - ), - "The --no-remote-cache flag must have a comment explaining why it's needed" + #[test] + fn test_filter_owner_seed_changes_skips_buck_and_vendor() { + let changes = vec![ + Status::Added(ProjectRelativePath::new("project/common/src/lib.rs")), + Status::Added(ProjectRelativePath::new("project/common/BUCK")), + Status::Added(ProjectRelativePath::new("project/common/vendor/foo.rs")), + ]; + let filtered = filter_owner_seed_changes(&changes); + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].get().as_str(), "project/common/src/lib.rs"); + } + + #[test] + fn test_owner_seed_changes_for_all_added_non_subproject_skips_select_all_path() { + let changes = vec![ + Status::Added(ProjectRelativePath::new("orion/src/lib.rs")), + Status::Added(ProjectRelativePath::new("orion/BUCK")), + ]; + let seeds = owner_seed_changes_for_discovery(false, true, &changes); + assert_eq!(seeds.len(), 1); + assert_eq!(seeds[0].get().as_str(), "orion/src/lib.rs"); + } + + #[test] + fn test_owner_seed_changes_for_modified_cl_uses_full_change_list() { + let changes = vec![ + Status::Modified(ProjectRelativePath::new("orion/src/lib.rs")), + Status::Added(ProjectRelativePath::new("orion/BUCK")), + ]; + let seeds = owner_seed_changes_for_discovery(false, false, &changes); + assert_eq!(seeds.len(), 2); + } + + #[test] + fn test_normalize_owner_targets_maps_vendor_to_rust_library() { + let diff = Targets::new(vec![ + TargetsEntry::Target(BuckTarget::testing( + "vendor", + "root//project/common", + "filegroup", + )), + TargetsEntry::Target(BuckTarget::testing( + "common", + "root//project/common", + "rust_library", + )), + ]); + let seeds = vec![ + TargetLabel::new("root//project/common:vendor"), + TargetLabel::new("root//project/common:common"), + ]; + let normalized = normalize_owner_targets_to_rust(&diff, seeds); + assert_eq!(normalized.len(), 1); + assert_eq!(normalized[0].as_str(), "root//project/common:common"); + } + + #[test] + fn test_normalize_owner_targets_maps_vendor_to_rust_binary() { + let diff = Targets::new(vec![ + TargetsEntry::Target(BuckTarget::testing( + "vendor", + "root//project/aardvark-dns", + "filegroup", + )), + TargetsEntry::Target(BuckTarget::testing( + "aardvark-dns", + "root//project/aardvark-dns", + "rust_binary", + )), + ]); + let seeds = vec![TargetLabel::new("root//project/aardvark-dns:vendor")]; + let normalized = normalize_owner_targets_to_rust(&diff, seeds); + assert_eq!(normalized.len(), 1); + assert_eq!( + normalized[0].as_str(), + "root//project/aardvark-dns:aardvark-dns" ); } } diff --git a/orion/src/main.rs b/orion/src/main.rs index f4757ab5f..1ea72e60e 100644 --- a/orion/src/main.rs +++ b/orion/src/main.rs @@ -6,6 +6,8 @@ pub mod repo; mod util; mod ws; +use std::io::{LineWriter, stderr}; + use tracing_subscriber::EnvFilter; use uuid::Uuid; @@ -15,12 +17,15 @@ async fn main() { // log filters can be configured via RUST_LOG in local dev and deployments. dotenvy::dotenv().ok(); - // Initialize structured logging + // LineWriter flushes on every newline so redirected stderr (run.sh → orion.log) + // stays live for remote tail/SSE consumers instead of block-buffering. tracing_subscriber::fmt() .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new("info,rfuse3::raw::session=error")), ) .with_target(true) + .with_writer(|| LineWriter::new(stderr())) .init(); // Configure WebSocket server address diff --git a/orion/src/repo/diff.rs b/orion/src/repo/diff.rs index 1fa64c5d3..9621ba356 100644 --- a/orion/src/repo/diff.rs +++ b/orion/src/repo/diff.rs @@ -239,12 +239,82 @@ pub enum RootImpactKind { SelectAll, } +/// How to treat targets when the base graph is empty (no old-repo snapshot). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EmptyBasePolicy { + /// Entire new repository: every target in the diff graph is impacted. + SelectAll, + /// Subdirectory import into a monorepo (e.g. all-added `rk8s/`): only targets + /// whose package lives under `root///...`. + ScopedNew { package_prefix: String }, +} + +impl EmptyBasePolicy { + pub fn scoped_new(prefix: impl Into) -> Self { + Self::ScopedNew { + package_prefix: prefix.into(), + } + } +} + +fn target_package_under_prefix(package: &str, package_prefix: &str) -> bool { + let qualified = format!("root//{package_prefix}"); + let qualified_slash = format!("{qualified}/"); + package == qualified || package.starts_with(&qualified_slash) +} + pub fn immediate_target_changes<'a>( base: &'a Targets, diff: &'a Targets, changes: &Changes, track_prelude_changes: bool, ) -> GraphImpact<'a> { + immediate_target_changes_with_policy( + base, + diff, + changes, + track_prelude_changes, + EmptyBasePolicy::SelectAll, + ) +} + +pub fn immediate_target_changes_with_policy<'a>( + base: &'a Targets, + diff: &'a Targets, + changes: &Changes, + track_prelude_changes: bool, + empty_base_policy: EmptyBasePolicy, +) -> GraphImpact<'a> { + // If there is no base graph, then everything is new. This must happen before + // universal `.buckconfig` handling so initial imports select the full graph + // even when the change list includes project-level configuration files. + if base.len_targets_upperbound() == 0 { + return match empty_base_policy { + EmptyBasePolicy::SelectAll => { + tracing::info!("All targets are new"); + let all_targets = diff + .targets() + .map(|t| (t, ImpactTraceData::new(t, RootImpactKind::SelectAll))) + .collect(); + GraphImpact::from_non_recursive(all_targets) + } + EmptyBasePolicy::ScopedNew { package_prefix } => { + tracing::info!( + package_prefix = %package_prefix, + "All-added import with empty base; selecting new targets under scoped prefix only." + ); + let mut res = GraphImpact::default(); + for target in diff.targets() { + if target_package_under_prefix(target.package.as_str(), &package_prefix) { + res.recursive + .push((target, ImpactTraceData::new(target, RootImpactKind::New))); + } + } + res + } + }; + } + if changes.cell_paths().any(is_buckconfig_change) { let mut ret = GraphImpact::from_non_recursive( diff.targets() @@ -262,16 +332,6 @@ pub fn immediate_target_changes<'a>( tracing::debug!("Finding changes"); - // If there is no base graph, then everything is new. - if base.len_targets_upperbound() == 0 { - tracing::info!("All targets are new"); - let all_targets = diff - .targets() - .map(|t| (t, ImpactTraceData::new(t, RootImpactKind::SelectAll))) - .collect(); - return GraphImpact::from_non_recursive(all_targets); - } - // Find those targets which are different let mut old = base.targets_by_label_key(); @@ -848,6 +908,81 @@ mod tests { ); } + #[test] + fn test_empty_base_selects_all_targets_before_buckconfig_handling() { + let base = Targets::new(vec![]); + let diff = Targets::new(vec![ + TargetsEntry::Target(BuckTarget::testing( + "aaa", + "foo//bar", + "prelude//rules.bzl:cxx_library", + )), + TargetsEntry::Target(BuckTarget::testing( + "bbb", + "foo//baz", + "prelude//rules.bzl:cxx_library", + )), + ]); + + let res = immediate_target_changes( + &base, + &diff, + &Changes::testing(&[ + Status::Added(CellPath::new("foo//.buckconfig")), + Status::Added(CellPath::new("foo//bar/BUCK")), + ]), + false, + ); + + assert!(res.recursive.is_empty()); + assert!(res.removed.is_empty()); + let mut selected = res + .non_recursive + .iter() + .map(|(target, trace)| (target.label().to_string(), trace.root_cause_reason)) + .collect::>(); + selected.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + selected, + vec![ + ("foo//bar:aaa".to_owned(), RootImpactKind::SelectAll), + ("foo//baz:bbb".to_owned(), RootImpactKind::SelectAll), + ] + ); + } + + #[test] + fn test_empty_base_scoped_new_limits_to_package_prefix() { + let base = Targets::new(vec![]); + let diff = Targets::new(vec![ + TargetsEntry::Target(BuckTarget::testing( + "aaa", + "root//project/lib", + "prelude//rules.bzl:rust_library", + )), + TargetsEntry::Target(BuckTarget::testing( + "vendor", + "root//third-party/rust/crates/foo/1.0.0", + "prelude//rules.bzl:genrule", + )), + ]); + + let res = immediate_target_changes_with_policy( + &base, + &diff, + &Changes::testing(&[Status::Added(CellPath::new("root//project/lib/src/lib.rs"))]), + false, + EmptyBasePolicy::scoped_new("project"), + ); + + assert_eq!(res.recursive.len(), 1); + assert_eq!( + res.recursive[0].0.label().to_string(), + "root//project/lib:aaa" + ); + assert!(res.non_recursive.is_empty()); + } + #[test] fn test_immediate_changes_with_removed() { fn target( diff --git a/scripts/demo/README.md b/scripts/demo/README.md index 898c90fc7..474a23dee 100644 --- a/scripts/demo/README.md +++ b/scripts/demo/README.md @@ -119,7 +119,7 @@ set TARGET_PLATFORMS=linux/amd64 |------------|----------------|---------------|-----| | `mega/mono-engine` | `mono/Dockerfile` | `.` (repo root) | `latest` | | `mega/orion-server` | `orion-server/Dockerfile` | `.` (repo root) | `latest` | -| `mega/mega-ui` | `moon/apps/web/Dockerfile` | `moon` | `demo-latest` | +| `mega/mega-ui` | `moon/apps/web/Dockerfile` | `moon` | `latest` | ## Image Tags diff --git a/scripts/demo/build-demo-images-local.bat b/scripts/demo/build-demo-images-local.bat index ccc7fe26a..5fba54755 100644 --- a/scripts/demo/build-demo-images-local.bat +++ b/scripts/demo/build-demo-images-local.bat @@ -71,7 +71,7 @@ set "TAGS_mono-engine=latest" set "IMAGES_mega-ui_DOCKERFILE=moon/apps/web/Dockerfile" set "IMAGES_mega-ui_CONTEXT=moon" -set "TAGS_mega-ui=demo-latest" +set "TAGS_mega-ui=latest" set "IMAGES_orion-server_DOCKERFILE=orion-server/Dockerfile" set "IMAGES_orion-server_CONTEXT=." @@ -263,10 +263,6 @@ rem ============================================================================ rem Build command set "BUILD_CMD=docker buildx build --builder mega-builder --platform %TARGET_PLATFORMS% --file !dockerfile! --tag !full_image! --progress=plain --build-arg BUILDKIT_INLINE_CACHE=1 --load" - if "!img_name!"=="mega-ui" ( - set "BUILD_CMD=!BUILD_CMD! --build-arg APP_ENV=demo" - ) - if defined CACHE_ARGS ( set "BUILD_CMD=!BUILD_CMD! !CACHE_ARGS!" ) diff --git a/scripts/demo/build-demo-images-local.sh b/scripts/demo/build-demo-images-local.sh index 7b8989f46..6f3f7df7a 100644 --- a/scripts/demo/build-demo-images-local.sh +++ b/scripts/demo/build-demo-images-local.sh @@ -92,7 +92,7 @@ get_image_config() { get_image_tag() { case "$1" in "mono-engine") echo "latest" ;; - "mega-ui") echo "demo-latest" ;; + "mega-ui") echo "latest" ;; "orion-server") echo "latest" ;; esac } @@ -403,10 +403,6 @@ build_and_push() { # Load before docker push so the pushed artifact is a single-architecture image manifest. build_args+=(--load) fi - - if [ "$image_name" = "mega-ui" ]; then - build_args+=(--build-arg APP_ENV=demo) - fi # Add cache arguments if [ ${#cache_from_args[@]} -gt 0 ]; then