diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 6afee9c54..a6912cd79 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -35,6 +35,7 @@ jobs: with: submodules: recursive + - name: Install nightly toolchain run: | rustup toolchain install nightly --profile minimal @@ -58,6 +59,7 @@ jobs: with: submodules: recursive + - name: Install system dependencies uses: ./.github/install-dep with: @@ -109,6 +111,7 @@ jobs: submodules: recursive lfs: true + - name: Install system dependencies uses: ./.github/install-dep with: diff --git a/.github/workflows/services.yml b/.github/workflows/services.yml index a6d3adda6..434a1c5a3 100644 --- a/.github/workflows/services.yml +++ b/.github/workflows/services.yml @@ -51,6 +51,7 @@ jobs: with: submodules: recursive + # Install system dependencies - name: Install system dependencies uses: ./.github/install-dep diff --git a/Cargo.lock b/Cargo.lock index 97bbd2159..ef6ed9918 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4440,8 +4440,7 @@ dependencies = [ [[package]] name = "libfuse-fs" version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61be2a0235d6a5ba30064c172f12d5465da0f013281a0e6675604d0781496e9" +source = "git+https://github.com/rk8s-dev/rk8s.git?branch=codex%2Fstabilize-libfuse-buck2-races#66ce9a6c95e27bb7b06bfec0a6f12bdd74a3565f" dependencies = [ "async-trait", "bitflags 2.13.0", @@ -7051,8 +7050,7 @@ dependencies = [ [[package]] name = "rfuse3" version = "0.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d7bea55817b58bd3a706ac0671f9aedc06d2d91e2f68272bd4fbe62f817d2b" +source = "git+https://github.com/rk8s-dev/rk8s.git?branch=codex%2Fstabilize-libfuse-buck2-races#66ce9a6c95e27bb7b06bfec0a6f12bdd74a3565f" dependencies = [ "aligned_box", "async-notify", @@ -7699,7 +7697,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "scorpiofs" version = "0.2.2" -source = "git+https://github.com/web3infra-foundation/scorpiofs.git#230f62a28598100b2bb94379de20bdec9d4566c5" +source = "git+https://github.com/web3infra-foundation/scorpiofs.git?branch=codex%2Fantares-fuse-local-libfuse#286147cd9295f68b1fad56994fc59adaca6ff50d" dependencies = [ "async-recursion", "async-trait", diff --git a/orion-scheduler/TESTING.md b/orion-scheduler/TESTING.md index 0c96a2001..5f7b95b48 100644 --- a/orion-scheduler/TESTING.md +++ b/orion-scheduler/TESTING.md @@ -91,6 +91,8 @@ VM_IP=$(curl -s http://localhost:8080/status | jq -r .vm_ip) ssh -i ~/.ssh/orion_vm_access root@$VM_IP ``` +部署时 scheduler 会在 guest 内创建 **8 GB** swap 文件(`/swapfile`,写入 `/etc/fstab`),减轻全量 buck2 编译时的内存压力。可用 `swapon --show` / `free -h` 确认。 + ### 日志 | 端点 | 格式 | 说明 | diff --git a/orion-scheduler/src/vm_manager.rs b/orion-scheduler/src/vm_manager.rs index 984210836..7330b358e 100644 --- a/orion-scheduler/src/vm_manager.rs +++ b/orion-scheduler/src/vm_manager.rs @@ -11,6 +11,80 @@ use crate::{ /// The target directory inside the VM guest OS where Orion is deployed const ORION_TARGET_DIR: &str = "/home/orion/orion-runner"; +/// Swap file path and size bounds for Orion build VMs (16 GB RAM by default). +const VM_SWAP_FILE: &str = "/swapfile"; +const VM_SWAP_TARGET_SIZE_GB: u32 = 8; +const VM_SWAP_MIN_SIZE_GB: u32 = 1; +const VM_SWAP_FREE_SPACE_RESERVE_GB: u32 = 1; + +/// Create and enable a swap file if not already active. +async fn setup_vm_swap(machine: &KeepAliveMachine) -> Result<()> { + info!( + "[orion-deploy] Configuring up to {} GB swap at {}", + VM_SWAP_TARGET_SIZE_GB, VM_SWAP_FILE + ); + let cmd = format!( + r#"set -e +swap_active=0 +if swapon --show --noheadings 2>/dev/null | grep -q '{swap_file}'; then + echo 'swap already active' + swap_active=1 +elif [ -f '{swap_file}' ]; then + swapon '{swap_file}' + swap_active=1 +else + target_kb=$(( {target_gb} * 1024 * 1024 )) + min_kb=$(( {min_gb} * 1024 * 1024 )) + reserve_kb=$(( {reserve_gb} * 1024 * 1024 )) + avail_kb=$(df -Pk "$(dirname '{swap_file}')" | awk 'NR == 2 {{ print $4 }}') + if [ -z "$avail_kb" ]; then + echo 'unable to determine available disk space for swap' >&2 + exit 1 + fi + usable_kb=$(( avail_kb - reserve_kb )) + if [ "$usable_kb" -lt "$min_kb" ]; then + echo "skipping swap: only ${{avail_kb}} KiB available, reserving ${{reserve_kb}} KiB" + else + if [ "$usable_kb" -gt "$target_kb" ]; then + swap_kb="$target_kb" + else + swap_kb="$usable_kb" + fi + echo "creating ${{swap_kb}} KiB swap file" + fallocate -l "${{swap_kb}}K" '{swap_file}' 2>/dev/null \ + || dd if=/dev/zero of='{swap_file}' bs=1024 count="$swap_kb" status=none + chmod 600 '{swap_file}' + mkswap '{swap_file}' + swapon '{swap_file}' + swap_active=1 + fi +fi +if [ "$swap_active" -eq 1 ]; then + grep -qF '{swap_file}' /etc/fstab 2>/dev/null \ + || echo '{swap_file} none swap sw 0 0' >> /etc/fstab +fi +swapon --show +free -h | head -2 +"#, + swap_file = VM_SWAP_FILE, + target_gb = VM_SWAP_TARGET_SIZE_GB, + min_gb = VM_SWAP_MIN_SIZE_GB, + reserve_gb = VM_SWAP_FREE_SPACE_RESERVE_GB, + ); + let output = machine.exec(&cmd).await?; + info!( + "[orion-deploy] Swap setup output:\n{}", + String::from_utf8_lossy(&output.stdout).trim() + ); + if !output.status.success() { + anyhow::bail!( + "swap setup failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(()) +} + /// Upload a single file to the VM via SFTP async fn upload_file( machine: &KeepAliveMachine, @@ -143,6 +217,8 @@ pub async fn deploy_orion_in_vm( pub async fn start_orion_in_vm(machine: &KeepAliveMachine) -> Result { info!("[orion-deploy] Starting Orion service"); + setup_vm_swap(machine).await?; + // Step 1: Create Scorpio directories info!("[orion-deploy] Creating Scorpio directories"); machine.exec("mkdir -p /data/scorpio/store").await?; diff --git a/orion/Cargo.toml b/orion/Cargo.toml index 4f7b7f356..671dfa16b 100644 --- a/orion/Cargo.toml +++ b/orion/Cargo.toml @@ -37,4 +37,4 @@ serial_test = { workspace = true } tempfile = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] -scorpiofs = { git = "https://github.com/web3infra-foundation/scorpiofs.git" } +scorpiofs = { git = "https://github.com/web3infra-foundation/scorpiofs.git", branch = "codex/antares-fuse-local-libfuse" }